From noreply at buildbot.pypy.org  Mon Dec  1 07:27:08 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  1 Dec 2014 07:27:08 +0100 (CET)
Subject: [pypy-commit] pypy default: two more unroll safes
Message-ID: <20141201062708.C3E791D2865@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74770:e6a84d9a3a34
Date: 2014-12-01 08:26 +0200
http://bitbucket.org/pypy/pypy/changeset/e6a84d9a3a34/

Log:	two more unroll safes

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -104,6 +104,7 @@
 
     track_index = True
 
+    @jit.unroll_safe
     def __init__(self, array, size, shape, strides, backstrides, op_flags=OpFlag()):
         from pypy.module.micronumpy import concrete
         assert len(shape) == len(strides) == len(backstrides)
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -455,6 +455,7 @@
     def descr_len(self, space):
         space.wrap(len(self.iters))
 
+    @jit.unroll_safe
     def descr_next(self, space):
         for it, st in self.iters:
             if not it.done(st):

From noreply at buildbot.pypy.org  Mon Dec  1 07:33:05 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  1 Dec 2014 07:33:05 +0100 (CET)
Subject: [pypy-commit] pypy default: one missing
Message-ID: <20141201063305.6F5D81D2865@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74771:d14911aabae2
Date: 2014-12-01 08:32 +0200
http://bitbucket.org/pypy/pypy/changeset/d14911aabae2/

Log:	one missing

diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -270,7 +270,7 @@
                 shape = shape_agreement(space, shape, arr)
     return shape
 
-
+ at jit.unroll_safe
 def _shape_agreement(shape1, shape2):
     """ Checks agreement about two shapes with respect to broadcasting. Returns
     the resulting shape.

From noreply at buildbot.pypy.org  Mon Dec  1 13:19:12 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Dec 2014 13:19:12 +0100 (CET)
Subject: [pypy-commit] creflect default: prepare the tests for more
Message-ID: <20141201121912.2634E1C04B4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r134:36bdfd223942
Date: 2014-12-01 11:39 +0100
http://bitbucket.org/cffi/creflect/changeset/36bdfd223942/

Log:	prepare the tests for more

diff --git a/zeffir/test/support.py b/zeffir/test/support.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/support.py
@@ -0,0 +1,34 @@
+import sys, os
+from udir import udir, include_dir
+
+zeffir_dir = os.path.join(os.path.dirname(__file__), '..')
+zeffir_lib_path = str(udir.join('zeffir.so'))
+
+
+def run(cmd):
+    print cmd
+    err = os.system(cmd)
+    if err:
+        raise OSError("%r\n==> error code %s" % (cmd, err))
+
+def compile_and_open(modname):
+    if not os.path.exists(zeffir_lib_path):
+        run("cd '%s' && "
+            "gcc -g -I'%s' -Wall -Werror -fPIC -shared zeffir.c -o '%s'" %
+            (zeffir_dir, include_dir, zeffir_lib_path))
+        assert os.path.exists(zeffir_lib_path)
+
+    sys.path.insert(0, str(udir))
+    import zeffir
+    del sys.path[0]
+
+    mod_c_path = str(udir.join('%s.c' % modname))
+    mod_lib_path = str(udir.join('lib%s.so' % modname))
+    if not os.path.exists(mod_lib_path):
+        run("cd '%s'/test && "
+            "PYTHONPATH=../.. ../../bin/creflect %s.crx '%s' && "
+            "gcc -g -I../../creflect -fPIC -shared '%s' -o '%s'" %
+            (zeffir_dir, modname, mod_c_path, mod_c_path, mod_lib_path))
+        assert os.path.exists(mod_lib_path)
+
+    return zeffir.open(modname, relative_to=zeffir_lib_path)
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -1,39 +1,19 @@
-import os, sys
-from udir import udir, include_dir
-
-def setup_module(mod):
-    global zeffir, zeffir_lib_path
-    zeffir_dir = os.path.join(os.path.dirname(__file__), '..')
-    zeffir_lib_path = str(udir.join('zeffir.so'))
-    err = os.system("cd '%s' && "
-            "gcc -g -I'%s' -Wall -Werror -fPIC -shared zeffir.c -o '%s'" %
-            (zeffir_dir, include_dir, zeffir_lib_path))
-    assert not err
-    #
-    sys.path.insert(0, str(udir))
-    import zeffir
-    #
-    basic_c_path = str(udir.join('basic.c'))
-    basic_lib_path = str(udir.join('libbasic.so'))
-    err = os.system("cd '%s'/test && "
-            "PYTHONPATH=../.. ../../bin/creflect basic.crx '%s' && "
-            "gcc -g -I../../creflect -fPIC -shared '%s' -o '%s'" %
-            (zeffir_dir, basic_c_path, basic_c_path, basic_lib_path))
-    assert not err
+import os
+import support
 
 
 def test_ffi_type():
-    ffi, lib = zeffir.open('basic', relative_to=zeffir_lib_path)
+    ffi, lib = support.compile_and_open('basic')
     assert type(ffi).__module__ == 'zeffir'
     assert type(ffi).__name__ == 'FFI'
     assert repr(ffi) == "<zeffir.FFI object for '%s/libbasic.so'>" % (
-        os.path.dirname(zeffir_lib_path),)
+        os.path.dirname(support.zeffir_lib_path),)
 
 def test_forty_two():
-    ffi, lib = zeffir.open('basic', relative_to=zeffir_lib_path)
+    ffi, lib = support.compile_and_open('basic')
     assert lib.forty_two == 42
     assert type(lib.forty_two) is int
 
 def test_dir():
-    ffi, lib = zeffir.open('basic', relative_to=zeffir_lib_path)
+    ffi, lib = support.compile_and_open('basic')
     assert dir(lib) == ['forty_two']

From noreply at buildbot.pypy.org  Mon Dec  1 13:19:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Dec 2014 13:19:13 +0100 (CET)
Subject: [pypy-commit] creflect default: in-progress: simple function calls
Message-ID: <20141201121913.830491C04B4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r135:083a62ba609e
Date: 2014-12-01 13:19 +0100
http://bitbucket.org/cffi/creflect/changeset/083a62ba609e/

Log:	in-progress: simple function calls

diff --git a/zeffir/test/function.crx b/zeffir/test/function.crx
new file mode 100644
--- /dev/null
+++ b/zeffir/test/function.crx
@@ -0,0 +1,11 @@
+// CREFLECT: start
+
+int simple_function(int);
+
+// CREFLECT: end
+
+
+int simple_function(int x)
+{
+    return x + 1;
+}
diff --git a/zeffir/test/test_function.py b/zeffir/test/test_function.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_function.py
@@ -0,0 +1,8 @@
+import support
+
+
+def test_simple_function():
+    ffi, lib = support.compile_and_open('function')
+    res = lib.simple_function(42)
+    assert type(res) is int
+    assert res == 43
diff --git a/zeffir/zef_builder.c b/zeffir/zef_builder.c
--- a/zeffir/zef_builder.c
+++ b/zeffir/zef_builder.c
@@ -4,9 +4,9 @@
 
 typedef struct {
     _crx_builder_t cb;
-    PyLibObject *lib;
-    PyFFIObject *ffi;
-    PyObject *dict;
+    ZefLibObject *lib;
+    ZefFFIObject *ffi;
+    PyObject *l_dict;
 } zeffir_builder_t;
 
 static PyObject *zef_cache_primitive_signed;
@@ -157,13 +157,22 @@
                             _crx_type_t *ret, _crx_qual_type args[], int nargs,
                             _crx_trampoline0_fn trampl, void *directcall)
 {
-    abort();
+    PyObject *l_dict = ((zeffir_builder_t *)cb)->l_dict;
+    ZefFFIObject *ffi = ((zeffir_builder_t *)cb)->ffi;
+
+    assert(trampl != NULL);
+    PyObject *x = make_builtin_func(ffi, name, ret, args, nargs, trampl);
+    if (x == NULL)
+        return;
+
+    PyDict_SetItemString(l_dict, name, x);
+    Py_DECREF(x);
 }
 
 static void zef_define_num_const(_crx_builder_t *cb, const char *name,
                                  _crx_type_t *ct, _crx_num_const_t *value)
 {
-    PyObject *dict = ((zeffir_builder_t *)cb)->dict;
+    PyObject *l_dict = ((zeffir_builder_t *)cb)->l_dict;
 
     assert(ct->ct_flags & CT_PRIMITIVE_ANY);
 
@@ -171,7 +180,7 @@
     if (x == NULL)
         return;
 
-    PyDict_SetItemString(dict, name, x);
+    PyDict_SetItemString(l_dict, name, x);
     Py_DECREF(x);
 }
 
@@ -180,26 +189,27 @@
     abort();
 }
 
-static int load_creflect_main(PyLibObject *lib)
+static int load_creflect_main(ZefLibObject *lib)
 {
     const char *creflect_main = "_creflect_main";   /* XXX fixed for now */
     int (*crxmain)(_crx_builder_t *);
-    crxmain = (int(*)(_crx_builder_t *))dlsym(lib->ffi->dl_lib, creflect_main);
+    crxmain = (int(*)(_crx_builder_t *))dlsym(lib->ffi->f_dl_lib,
+                                              creflect_main);
     if (crxmain == NULL) {
         PyErr_Format(PyExc_OSError, "%s: symbol '%s' not found",
-                     lib->ffi->libname, creflect_main);
+                     lib->ffi->f_libname, creflect_main);
         return -1;
     }
 
     int result = crxmain(NULL);
     if ((result & ~0xFF) != MAGIC_CREFLECT_TAG) {
         PyErr_Format(PyExc_OSError, "%s: %s() is not a creflect entry point",
-                     lib->ffi->libname, creflect_main);
+                     lib->ffi->f_libname, creflect_main);
         return -1;
     }
     if ((result & 0xFF) != CREFLECT_VERSION) {
         PyErr_Format(PyExc_OSError, "%s was made with creflect version %d, "
-                     "expected %d", lib->ffi->libname, result & 0xFF,
+                     "expected %d", lib->ffi->f_libname, result & 0xFF,
                      CREFLECT_VERSION);
         return -1;
     }
@@ -230,11 +240,12 @@
             zef_define_num_const,
             zef_error,
         },
-        lib,        /* lib */
-        lib->ffi,   /* ffi */
-        lib->dict,  /* dict */
+        lib,          /* lib */
+        lib->ffi,     /* ffi */
+        lib->l_dict,  /* l_dict */
     };
     crxmain(&builder.cb);
+
     return PyErr_Occurred() ? -1 : 0;
 }
 
diff --git a/zeffir/zef_cdata.c b/zeffir/zef_cdata.c
new file mode 100644
--- /dev/null
+++ b/zeffir/zef_cdata.c
@@ -0,0 +1,1 @@
+
diff --git a/zeffir/zef_cfunc.c b/zeffir/zef_cfunc.c
new file mode 100644
--- /dev/null
+++ b/zeffir/zef_cfunc.c
@@ -0,0 +1,96 @@
+
+typedef struct {
+    PyObject_HEAD
+
+    ZefFFIObject       *zfs_ffi;
+    int                 zfs_nargs;
+    _crx_trampoline0_fn zfs_trampl;
+    PyMethodDef         zfs_md;
+    char                zfs_funcname[1];
+
+} ZefFuncSupportObject;
+
+static void zef_builtin_support_dealloc(ZefFuncSupportObject *zfs)
+{
+    Py_DECREF(zfs->zfs_ffi);
+    PyObject_Del(zfs);
+}
+
+static PyTypeObject ZefFuncSupport_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.BuiltinSupport",
+    sizeof(ZefFuncSupportObject),
+    0,
+    (destructor)zef_builtin_support_dealloc,    /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    0,                                          /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
+};
+
+static PyObject *zfs_call(PyObject *self, PyObject *args)
+{
+    ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)self;
+    int i, nargs = zfs->zfs_nargs;
+    Py_ssize_t actualnargs;
+    void *llargs[nargs];
+    int llinput[nargs], lloutput;   /* temp */
+
+    actualnargs = PyTuple_Size(args);
+    if (actualnargs != nargs) {
+        if (!PyErr_Occurred())
+            PyErr_Format(PyExc_TypeError, "'%s' expects %d arguments, got %zd",
+                         zfs->zfs_funcname, nargs, actualnargs);
+        return NULL;
+    }
+
+    for (i = 0; i < nargs; i++) {
+        llinput[i] = PyInt_AsLong(PyTuple_GET_ITEM(args, i));
+        if (PyErr_Occurred())
+            return NULL;
+        llargs[i] = &llinput[i];
+    }
+
+    zfs->zfs_trampl(llargs, &lloutput);
+
+    return PyInt_FromLong(lloutput);
+}
+
+static PyObject *make_builtin_func(ZefFFIObject *ffi, const char *funcname,
+                                   _crx_type_t *ret,
+                                   _crx_qual_type args[], int nargs,
+                                   _crx_trampoline0_fn trampl)
+{
+    size_t size = sizeof(ZefFuncSupportObject) + strlen(funcname);
+    ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)PyObject_Malloc(size);
+    if (zfs == NULL)
+        return PyErr_NoMemory();
+    PyObject_Init((PyObject *)zfs, &ZefFuncSupport_Type);
+
+    Py_INCREF(ffi);
+    zfs->zfs_ffi = ffi;
+    zfs->zfs_nargs = nargs;
+    zfs->zfs_trampl = trampl;
+
+    memset(&zfs->zfs_md, 0, sizeof(PyMethodDef));
+    zfs->zfs_md.ml_name = strcpy(zfs->zfs_funcname, funcname);
+    zfs->zfs_md.ml_meth = &zfs_call;
+    zfs->zfs_md.ml_flags = METH_VARARGS;
+    /*zfs->zfs_md.ml_doc = ... */
+
+    PyObject *res = PyCFunction_NewEx(&zfs->zfs_md, (PyObject *)zfs,
+                                      ffi->f_libname_obj);
+    Py_DECREF(zfs);
+    return res;
+}
diff --git a/zeffir/zef_ffi_obj.c b/zeffir/zef_ffi_obj.c
--- a/zeffir/zef_ffi_obj.c
+++ b/zeffir/zef_ffi_obj.c
@@ -1,27 +1,28 @@
 
-struct PyFFIObject_s {
+struct ZefFFIObject_s {
     PyObject_HEAD
-    void *dl_lib;
-    char *libname;
+    void *f_dl_lib;      /* keep the library returned by 'dlopen()' */
+    char *f_libname;     /* the original argument to 'dlopen()' */
+    PyObject *f_libname_obj;   /* same, as a PyObject */
 };
 
-static void ffi_dealloc(PyFFIObject *ffi)
+static void ffi_dealloc(ZefFFIObject *ffi)
 {
-    if (ffi->dl_lib != NULL)
-        dlclose(ffi->dl_lib);
-    free(ffi->libname);
+    if (ffi->f_dl_lib != NULL)
+        dlclose(ffi->f_dl_lib);
+    Py_DECREF(ffi->f_libname_obj);
     PyObject_Del(ffi);
 }
 
-static PyObject *ffi_repr(PyFFIObject *ffi)
+static PyObject *ffi_repr(ZefFFIObject *ffi)
 {
-    return PyString_FromFormat("<zeffir.FFI object for '%s'>", ffi->libname);
+    return PyString_FromFormat("<zeffir.FFI object for '%s'>", ffi->f_libname);
 }
 
-static PyTypeObject FFI_Type = {
+static PyTypeObject ZefFFI_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "zeffir.FFI",
-    sizeof(PyFFIObject),
+    sizeof(ZefFFIObject),
     0,
     (destructor)ffi_dealloc,                    /* tp_dealloc */
     0,                                          /* tp_print */
diff --git a/zeffir/zef_funcs.c b/zeffir/zef_funcs.c
deleted file mode 100644
--- a/zeffir/zef_funcs.c
+++ /dev/null
@@ -1,77 +0,0 @@
-
-static PyObject *b_open(PyObject *self, PyObject *args, PyObject *kwds)
-{
-    char *keywords[] = {"libname", "relative_to", NULL};
-    char *libname, *relative_to;
-    char *path;
-
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "ss", keywords,
-                                     &libname, &relative_to))
-        return NULL;
-
-    char *last_slash = strrchr(relative_to, '/');
-    if (last_slash == NULL) {
-        relative_to = "./";
-        last_slash = relative_to + 1;
-    }
-    size_t n = last_slash + 1 - relative_to;
-    path = malloc(n + strlen(libname) + 16);
-    if (path == NULL)
-        return PyErr_NoMemory();
-    memcpy(path, relative_to, n);
-    strcpy(path + n, "lib");
-    strcat(path + n, libname);
-    strcat(path + n, ".so");
-
-    PyFFIObject *ffi = NULL;
-    PyLibObject *lib = NULL;
-    PyObject *dict = NULL;
-
-    dlerror();   /* clear errors */
-    void *dl_lib = dlopen(path, RTLD_LAZY);
-    if (dl_lib == NULL) {
-        char *error = dlerror();
-        if (error == NULL)
-            error = "failed to open";
-        PyErr_Format(PyExc_OSError, "%s: %s", path, error);
-        goto error;
-    }
-
-    ffi = PyObject_New(PyFFIObject, &FFI_Type);
-    if (ffi == NULL)
-        goto error;
-    ffi->dl_lib = dl_lib;  dl_lib = NULL;
-    ffi->libname = path;   path = NULL;
-
-    dict = PyDict_New();
-    if (dict == NULL)
-        goto error;
-
-    lib = PyObject_New(PyLibObject, &Lib_Type);
-    if (lib == NULL)
-        goto error;
-    lib->ffi = ffi;    ffi = NULL;
-    lib->dict = dict;  dict = NULL;
-
-    if (load_creflect_main(lib) < 0)
-        goto error;
-
-    PyObject *result = Py_BuildValue("OO", lib->ffi, lib);
-    Py_DECREF(lib);
-    return result;
-
- error:
-    Py_XDECREF(dict);
-    Py_XDECREF(lib);
-    Py_XDECREF(ffi);
-    if (dl_lib != NULL)
-        dlclose(dl_lib);
-    free(path);
-    return NULL;
-}
-
-
-static PyMethodDef ZeffirMethods[] = {
-    {"open",   (PyCFunction)b_open, METH_KEYWORDS},
-    {NULL,     NULL}    /* Sentinel */
-};
diff --git a/zeffir/zef_lib_obj.c b/zeffir/zef_lib_obj.c
--- a/zeffir/zef_lib_obj.c
+++ b/zeffir/zef_lib_obj.c
@@ -1,26 +1,26 @@
 
-struct PyLibObject_s {
+struct ZefLibObject_s {
     PyObject_HEAD
-    PyObject *dict;
-    PyFFIObject *ffi;
+    PyObject *l_dict;
+    ZefFFIObject *ffi;
 };
 
-static void lib_dealloc(PyLibObject *lib)
+static void lib_dealloc(ZefLibObject *lib)
 {
-    Py_DECREF(lib->dict);
+    Py_DECREF(lib->l_dict);
     Py_DECREF(lib->ffi);
     PyObject_Del(lib);
 }
 
-static PyObject *lib_repr(PyLibObject *lib)
+static PyObject *lib_repr(ZefLibObject *lib)
 {
     return PyString_FromFormat("<zeffir.Lib object for '%s'>",
-                               lib->ffi->libname);
+                               lib->ffi->f_libname);
 }
 
 static PyObject *lib_dir(PyObject *lib, PyObject *noarg)
 {
-    return PyDict_Keys(((PyLibObject *)lib)->dict);
+    return PyDict_Keys(((ZefLibObject *)lib)->l_dict);
 }
 
 static PyMethodDef lib_methods[] = {
@@ -29,14 +29,14 @@
 };
 
 static PyMemberDef lib_members[] = {
-    {"__dict__", T_OBJECT, offsetof(PyLibObject, dict), READONLY},
+    {"__dict__", T_OBJECT, offsetof(ZefLibObject, l_dict), READONLY},
     {NULL}
 };
 
-static PyTypeObject Lib_Type = {
+static PyTypeObject ZefLib_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "zeffir.Lib",
-    sizeof(PyLibObject),
+    sizeof(ZefLibObject),
     0,
     (destructor)lib_dealloc,                    /* tp_dealloc */
     0,                                          /* tp_print */
@@ -68,5 +68,5 @@
     0,                                          /* tp_dict */
     0,                                          /* tp_descr_get */
     0,                                          /* tp_descr_set */
-    offsetof(PyLibObject, dict),                /* tp_dictoffset */
+    offsetof(ZefLibObject, l_dict),              /* tp_dictoffset */
 };
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -6,8 +6,8 @@
 
 
 typedef struct _crx_type_s CTypeDescrObject;
-typedef struct PyLibObject_s PyLibObject;
-typedef struct PyFFIObject_s PyFFIObject;
+typedef struct ZefLibObject_s ZefLibObject;
+typedef struct ZefFFIObject_s ZefFFIObject;
 
 
 /************************************************************/
@@ -15,13 +15,91 @@
 /* Allows all function and global symbols to remain static. */
 
 #include "zef_ctype.c"
+#include "zef_cdata.c"
 #include "zef_ffi_obj.c"
 #include "zef_lib_obj.c"
+#include "zef_cfunc.c"
 #include "zef_builder.c"
-#include "zef_funcs.c"
 
 /************************************************************/
 
+
+static PyObject *z_open(PyObject *self, PyObject *args, PyObject *kwds)
+{
+    char *keywords[] = {"libname", "relative_to", NULL};
+    char *libname, *relative_to, *relative_dir;
+    PyObject *path;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "ss", keywords,
+                                     &libname, &relative_to))
+        return NULL;
+
+    char *last_slash = strrchr(relative_to, '/');
+    if (last_slash == NULL)
+        relative_dir = strdup(".");
+    else
+        relative_dir = strndup(relative_to, last_slash - relative_to);
+    if (relative_dir == NULL)
+        return PyErr_NoMemory();
+
+    path = PyString_FromFormat("%s/lib%s.so", relative_dir, libname);
+    free(relative_dir);
+    if (path == NULL)
+        return NULL;
+
+    ZefFFIObject *ffi = NULL;
+    ZefLibObject *lib = NULL;
+    PyObject *dict = NULL;
+
+    dlerror();   /* clear errors */
+    void *dl_lib = dlopen(PyString_AS_STRING(path), RTLD_LAZY);
+    if (dl_lib == NULL) {
+        char *error = dlerror();
+        if (error == NULL)
+            error = "failed to open";
+        PyErr_Format(PyExc_OSError, "%s: %s", PyString_AS_STRING(path), error);
+        goto error;
+    }
+
+    ffi = PyObject_New(ZefFFIObject, &ZefFFI_Type);
+    if (ffi == NULL)
+        goto error;
+    ffi->f_dl_lib = dl_lib;     dl_lib = NULL;
+    ffi->f_libname = PyString_AS_STRING(path);
+    ffi->f_libname_obj = path;  path = NULL;
+
+    dict = PyDict_New();
+    if (dict == NULL)
+        goto error;
+
+    lib = PyObject_New(ZefLibObject, &ZefLib_Type);
+    if (lib == NULL)
+        goto error;
+    lib->ffi = ffi;      ffi = NULL;
+    lib->l_dict = dict;  dict = NULL;
+
+    if (load_creflect_main(lib) < 0)
+        goto error;
+
+    PyObject *result = Py_BuildValue("OO", lib->ffi, lib);
+    Py_DECREF(lib);
+    return result;
+
+ error:
+    Py_XDECREF(dict);
+    Py_XDECREF(lib);
+    Py_XDECREF(ffi);
+    if (dl_lib != NULL)
+        dlclose(dl_lib);
+    Py_XDECREF(path);
+    return NULL;
+}
+
+static PyMethodDef ZeffirMethods[] = {
+    {"open",   (PyCFunction)z_open, METH_KEYWORDS},
+    {NULL,     NULL}    /* Sentinel */
+};
+
 PyMODINIT_FUNC
 initzeffir(void)
 {
@@ -34,8 +112,12 @@
     if (PyErr_Occurred())
         return;
 
-    if (PyType_Ready(&FFI_Type) < 0)
+    if (PyType_Ready(&ZefFFI_Type) < 0)
         return;
-    if (PyType_Ready(&Lib_Type) < 0)
+    if (PyType_Ready(&ZefLib_Type) < 0)
+        return;
+    if (PyType_Ready(&CTypeDescr_Type) < 0)
+        return;
+    if (PyType_Ready(&ZefFuncSupport_Type) < 0)
         return;
 }

From noreply at buildbot.pypy.org  Mon Dec  1 13:38:31 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Dec 2014 13:38:31 +0100 (CET)
Subject: [pypy-commit] creflect default: add comment
Message-ID: <20141201123831.484151D2CCC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r136:96dd4b191e06
Date: 2014-12-01 13:38 +0100
http://bitbucket.org/cffi/creflect/changeset/96dd4b191e06/

Log:	add comment

diff --git a/zeffir/test/support.py b/zeffir/test/support.py
--- a/zeffir/test/support.py
+++ b/zeffir/test/support.py
@@ -12,6 +12,8 @@
         raise OSError("%r\n==> error code %s" % (cmd, err))
 
 def compile_and_open(modname):
+    """Process with creflect 'modname.crx', compile the result, and load it."""
+
     if not os.path.exists(zeffir_lib_path):
         run("cd '%s' && "
             "gcc -g -I'%s' -Wall -Werror -fPIC -shared zeffir.c -o '%s'" %

From noreply at buildbot.pypy.org  Mon Dec  1 14:46:46 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  1 Dec 2014 14:46:46 +0100 (CET)
Subject: [pypy-commit] pypy default: kill this hack for now - makes matters
 worse without a deeper thought
Message-ID: <20141201134646.332521C04B4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74772:70275c5679a9
Date: 2014-12-01 15:46 +0200
http://bitbucket.org/pypy/pypy/changeset/70275c5679a9/

Log:	kill this hack for now - makes matters worse without a deeper
	thought

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -155,7 +155,7 @@
         index = state.index
         if self.track_index:
             index += 1
-        indices = state.indices[:]
+        indices = state.indices
         offset = state.offset
         if self.contiguous:
             offset += self.array.dtype.elsize

From noreply at buildbot.pypy.org  Mon Dec  1 15:05:04 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  1 Dec 2014 15:05:04 +0100 (CET)
Subject: [pypy-commit] pypy default: this is unroll safe
Message-ID: <20141201140504.58CF31C34C4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74773:ff9f32344c01
Date: 2014-12-01 16:04 +0200
http://bitbucket.org/pypy/pypy/changeset/ff9f32344c01/

Log:	this is unroll safe

diff --git a/pypy/module/micronumpy/flagsobj.py b/pypy/module/micronumpy/flagsobj.py
--- a/pypy/module/micronumpy/flagsobj.py
+++ b/pypy/module/micronumpy/flagsobj.py
@@ -1,3 +1,5 @@
+from rpython.rlib import jit
+
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import interp2app
@@ -13,6 +15,7 @@
     arr.flags &= ~flags
 
 
+ at jit.unroll_safe
 def _update_contiguous_flags(arr):
     shape = arr.shape
     strides = arr.strides

From noreply at buildbot.pypy.org  Mon Dec  1 15:11:38 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  1 Dec 2014 15:11:38 +0100 (CET)
Subject: [pypy-commit] pypy default: try some unrolling strategy
Message-ID: <20141201141138.5A4B21C34C4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74774:bf9b7211befc
Date: 2014-12-01 16:11 +0200
http://bitbucket.org/pypy/pypy/changeset/bf9b7211befc/

Log:	try some unrolling strategy

diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py
--- a/rpython/rtyper/rlist.py
+++ b/rpython/rtyper/rlist.py
@@ -983,6 +983,14 @@
 #
 #  Comparison.
 
+def listeq_unroll_case(l1, l2, eqfn):
+    if jit.isvirtual(l1) and len(l1) < 10:
+        return True
+    if jit.isvirtual(l2) and len(l2) < 10:
+        return True
+    return False
+
+ at jit.unroll_iff(listeq_unroll_case)
 def ll_listeq(l1, l2, eqfn):
     if not l1 and not l2:
         return True

From noreply at buildbot.pypy.org  Mon Dec  1 15:13:08 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  1 Dec 2014 15:13:08 +0100 (CET)
Subject: [pypy-commit] pypy default: I never get those right
Message-ID: <20141201141308.BB35D1C34C4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74775:739ecaa8a1c0
Date: 2014-12-01 16:12 +0200
http://bitbucket.org/pypy/pypy/changeset/739ecaa8a1c0/

Log:	I never get those right

diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py
--- a/rpython/rtyper/rlist.py
+++ b/rpython/rtyper/rlist.py
@@ -984,13 +984,13 @@
 #  Comparison.
 
 def listeq_unroll_case(l1, l2, eqfn):
-    if jit.isvirtual(l1) and len(l1) < 10:
+    if jit.isvirtual(l1) and l1.ll_length() < 10:
         return True
-    if jit.isvirtual(l2) and len(l2) < 10:
+    if jit.isvirtual(l2) and l2.ll_length() < 10:
         return True
     return False
 
- at jit.unroll_iff(listeq_unroll_case)
+ at jit.look_inside_iff(listeq_unroll_case)
 def ll_listeq(l1, l2, eqfn):
     if not l1 and not l2:
         return True

From noreply at buildbot.pypy.org  Mon Dec  1 16:59:19 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon,  1 Dec 2014 16:59:19 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move all the SETUP_XXX
Message-ID: <20141201155919.35B9C1D36F5@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74776:ec616f23ec15
Date: 2014-11-26 17:18 +0100
http://bitbucket.org/pypy/pypy/changeset/ec616f23ec15/

Log:	move all the SETUP_XXX

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -507,6 +507,36 @@
         block = ExceptBlock(ctx.stackdepth, self.arg)
         ctx.blockstack.append(block)
 
+ at bc_reader.register_opcode
+class SETUP_LOOP(BCInstruction):
+    def eval(self, ctx):
+        from rpython.flowspace.flowcontext import LoopBlock
+        block = LoopBlock(ctx.stackdepth, self.arg)
+        ctx.blockstack.append(block)
+
+ at bc_reader.register_opcode
+class SETUP_FINALLY(BCInstruction):
+    def eval(self, ctx):
+        from rpython.flowspace.flowcontext import FinallyBlock
+        block = FinallyBlock(ctx.stackdepth, self.arg)
+        ctx.blockstack.append(block)
+
+ at bc_reader.register_opcode
+class SETUP_WITH(BCInstruction):
+    def eval(self, ctx):
+        from rpython.flowspace.flowcontext import WithBlock
+        # A simpler version than the 'real' 2.7 one:
+        # directly call manager.__enter__(), don't use special lookup functions
+        # which don't make sense on the RPython type system.
+        w_manager = ctx.peekvalue()
+        w_exit = op.getattr(w_manager, const("__exit__")).eval(ctx)
+        ctx.settopvalue(w_exit)
+        w_enter = op.getattr(w_manager, const('__enter__')).eval(ctx)
+        w_result = op.simple_call(w_enter).eval(ctx)
+        block = WithBlock(ctx.stackdepth, self.arg)
+        ctx.blockstack.append(block)
+        ctx.pushvalue(w_result)
+
 _unary_ops = [
     ('UNARY_POSITIVE', op.pos),
     ('UNARY_NEGATIVE', op.neg),
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -702,27 +702,6 @@
         w_iterator = op.iter(w_iterable).eval(self)
         self.pushvalue(w_iterator)
 
-    def SETUP_LOOP(self, target):
-        block = LoopBlock(self.stackdepth, target)
-        self.blockstack.append(block)
-
-    def SETUP_FINALLY(self, target):
-        block = FinallyBlock(self.stackdepth, target)
-        self.blockstack.append(block)
-
-    def SETUP_WITH(self, target):
-        # A simpler version than the 'real' 2.7 one:
-        # directly call manager.__enter__(), don't use special lookup functions
-        # which don't make sense on the RPython type system.
-        w_manager = self.peekvalue()
-        w_exit = op.getattr(w_manager, const("__exit__")).eval(self)
-        self.settopvalue(w_exit)
-        w_enter = op.getattr(w_manager, const('__enter__')).eval(self)
-        w_result = op.simple_call(w_enter).eval(self)
-        block = WithBlock(self.stackdepth, target)
-        self.blockstack.append(block)
-        self.pushvalue(w_result)
-
     def WITH_CLEANUP(self, oparg):
         # Note: RPython context managers receive None in lieu of tracebacks
         # and cannot suppress the exception.

From noreply at buildbot.pypy.org  Mon Dec  1 16:59:20 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon,  1 Dec 2014 16:59:20 +0100 (CET)
Subject: [pypy-commit] pypy framestate: create a base class for the
	SETUP_XXX opcodes
Message-ID: <20141201155920.62DEA1D36F5@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74777:13de37f01288
Date: 2014-11-26 17:36 +0100
http://bitbucket.org/pypy/pypy/changeset/13de37f01288/

Log:	create a base class for the SETUP_XXX opcodes

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -499,30 +499,36 @@
             else:
                 raise
 
+class SetupInstruction(BCInstruction):
+    def bc_flow(self, reader):
+        reader.curr_block.operations.append(self)
+        self.target = reader.get_block_at(self.arg)
+
+    def eval(self, ctx):
+        block = self.make_block(ctx)
+        ctx.blockstack.append(block)
+
 
 @bc_reader.register_opcode
-class SETUP_EXCEPT(BCInstruction):
-    def eval(self, ctx):
+class SETUP_EXCEPT(SetupInstruction):
+    def make_block(self, ctx):
         from rpython.flowspace.flowcontext import ExceptBlock
-        block = ExceptBlock(ctx.stackdepth, self.arg)
-        ctx.blockstack.append(block)
+        return ExceptBlock(ctx.stackdepth, self.arg)
 
 @bc_reader.register_opcode
-class SETUP_LOOP(BCInstruction):
-    def eval(self, ctx):
+class SETUP_LOOP(SetupInstruction):
+    def make_block(self, ctx):
         from rpython.flowspace.flowcontext import LoopBlock
-        block = LoopBlock(ctx.stackdepth, self.arg)
-        ctx.blockstack.append(block)
+        return LoopBlock(ctx.stackdepth, self.arg)
 
 @bc_reader.register_opcode
-class SETUP_FINALLY(BCInstruction):
-    def eval(self, ctx):
+class SETUP_FINALLY(SetupInstruction):
+    def make_block(self, ctx):
         from rpython.flowspace.flowcontext import FinallyBlock
-        block = FinallyBlock(ctx.stackdepth, self.arg)
-        ctx.blockstack.append(block)
+        return FinallyBlock(ctx.stackdepth, self.arg)
 
 @bc_reader.register_opcode
-class SETUP_WITH(BCInstruction):
+class SETUP_WITH(SetupInstruction):
     def eval(self, ctx):
         from rpython.flowspace.flowcontext import WithBlock
         # A simpler version than the 'real' 2.7 one:

From noreply at buildbot.pypy.org  Mon Dec  1 16:59:21 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon,  1 Dec 2014 16:59:21 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move BREAK_LOOP, CONTINUE_LOOP
Message-ID: <20141201155921.9F4641D36F5@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74778:8796e220baf7
Date: 2014-11-26 19:38 +0100
http://bitbucket.org/pypy/pypy/changeset/8796e220baf7/

Log:	move BREAK_LOOP, CONTINUE_LOOP

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -499,6 +499,18 @@
             else:
                 raise
 
+ at bc_reader.register_opcode
+class BREAK_LOOP(BCInstruction):
+    def eval(self, ctx):
+        from rpython.flowspace.flowcontext import Break
+        raise Break
+
+ at bc_reader.register_opcode
+class CONTINUE_LOOP(BCInstruction):
+    def eval(self, ctx):
+        from rpython.flowspace.flowcontext import Continue
+        raise Continue(self.arg)
+
 class SetupInstruction(BCInstruction):
     def bc_flow(self, reader):
         reader.curr_block.operations.append(self)
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -479,12 +479,6 @@
     def BAD_OPCODE(self, _):
         raise FlowingError("This operation is not RPython")
 
-    def BREAK_LOOP(self, oparg):
-        raise Break
-
-    def CONTINUE_LOOP(self, startofloop):
-        raise Continue(startofloop)
-
     def not_(self, w_obj):
         w_bool = op.bool(w_obj).eval(self)
         return const(not self.guessbool(w_bool))

From noreply at buildbot.pypy.org  Mon Dec  1 16:59:22 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Mon,  1 Dec 2014 16:59:22 +0100 (CET)
Subject: [pypy-commit] pypy framestate: return BC blocks from
	FrameBlock.handle()
Message-ID: <20141201155922.DA2831D36F5@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74779:3b6c35deccbc
Date: 2014-11-26 19:54 +0100
http://bitbucket.org/pypy/pypy/changeset/3b6c35deccbc/

Log:	return BC blocks from FrameBlock.handle()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -507,9 +507,13 @@
 
 @bc_reader.register_opcode
 class CONTINUE_LOOP(BCInstruction):
+    def bc_flow(self, reader):
+        reader.curr_block.operations.append(self)
+        self.target = reader.get_block_at(self.arg)
+
     def eval(self, ctx):
         from rpython.flowspace.flowcontext import Continue
-        raise Continue(self.arg)
+        raise Continue(self.target)
 
 class SetupInstruction(BCInstruction):
     def bc_flow(self, reader):
@@ -525,19 +529,19 @@
 class SETUP_EXCEPT(SetupInstruction):
     def make_block(self, ctx):
         from rpython.flowspace.flowcontext import ExceptBlock
-        return ExceptBlock(ctx.stackdepth, self.arg)
+        return ExceptBlock(ctx.stackdepth, self.target)
 
 @bc_reader.register_opcode
 class SETUP_LOOP(SetupInstruction):
     def make_block(self, ctx):
         from rpython.flowspace.flowcontext import LoopBlock
-        return LoopBlock(ctx.stackdepth, self.arg)
+        return LoopBlock(ctx.stackdepth, self.target)
 
 @bc_reader.register_opcode
 class SETUP_FINALLY(SetupInstruction):
     def make_block(self, ctx):
         from rpython.flowspace.flowcontext import FinallyBlock
-        return FinallyBlock(ctx.stackdepth, self.arg)
+        return FinallyBlock(ctx.stackdepth, self.target)
 
 @bc_reader.register_opcode
 class SETUP_WITH(SetupInstruction):
@@ -551,7 +555,7 @@
         ctx.settopvalue(w_exit)
         w_enter = op.getattr(w_manager, const('__enter__')).eval(ctx)
         w_result = op.simple_call(w_enter).eval(ctx)
-        block = WithBlock(ctx.stackdepth, self.arg)
+        block = WithBlock(ctx.stackdepth, self.target)
         ctx.blockstack.append(block)
         ctx.pushvalue(w_result)
 
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -16,6 +16,7 @@
     rpython_print_newline)
 from rpython.flowspace.operation import op
 from rpython.flowspace.bytecode import BytecodeCorruption, bc_reader
+from rpython.flowspace.bytecode import BytecodeBlock
 from rpython.flowspace.pygraph import PyGraph
 
 w_None = const(None)
@@ -356,12 +357,11 @@
                 try:
                     next_block = instr.eval(self)
                 except FlowSignal as signal:
-                    next_position = bc_graph.get_position(self.unroll(signal))
+                    next_block = self.unroll(signal)
+                if next_block is None:
+                    next_position = bc_graph.next_pos()
                 else:
-                    if next_block is None:
-                        next_position = bc_graph.next_pos()
-                    else:
-                        next_position = next_block, 0
+                    next_position = next_block, 0
                 bc_graph.curr_position = next_position
                 self.recorder.final_state = self.getstate(next_position)
 
@@ -1125,20 +1125,20 @@
     """Abstract base class for frame blocks from the blockstack,
     used by the SETUP_XXX and POP_BLOCK opcodes."""
 
-    def __init__(self, stackdepth, handlerposition):
-        self.handlerposition = handlerposition
+    def __init__(self, stackdepth, handler):
+        self.handler = handler
         self.stackdepth = stackdepth
 
     def __eq__(self, other):
         return (self.__class__ is other.__class__ and
-                self.handlerposition == other.handlerposition and
+                self.handler == other.handler and
                 self.stackdepth == other.stackdepth)
 
     def __ne__(self, other):
         return not (self == other)
 
     def __hash__(self):
-        return hash((self.handlerposition, self.stackdepth))
+        return hash((self.handler, self.stackdepth))
 
     def cleanupstack(self, ctx):
         ctx.dropvaluesuntil(self.stackdepth)
@@ -1161,7 +1161,7 @@
         else:
             # jump to the end of the loop
             self.cleanupstack(ctx)
-            return self.handlerposition
+            return self.handler
 
 class ExceptBlock(FrameBlock):
     """An try:except: block.  Stores the position of the exception handler."""
@@ -1181,7 +1181,7 @@
         ctx.pushvalue(w_exc.w_value)
         ctx.pushvalue(w_exc.w_type)
         ctx.last_exception = w_exc
-        return self.handlerposition   # jump to the handler
+        return self.handler   # jump to the handler
 
 class FinallyBlock(FrameBlock):
     """A try:finally: block.  Stores the position of the exception handler."""
@@ -1193,7 +1193,7 @@
         # the block unrolling and the entering the finally: handler.
         self.cleanupstack(ctx)
         ctx.pushvalue(unroller)
-        return self.handlerposition   # jump to the handler
+        return self.handler   # jump to the handler
 
 
 class WithBlock(FinallyBlock):

From noreply at buildbot.pypy.org  Mon Dec  1 22:14:28 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Dec 2014 22:14:28 +0100 (CET)
Subject: [pypy-commit] creflect default: another reorganization,
 split more the ffi and the lib
Message-ID: <20141201211428.BBE401C34C4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r137:111c65f6cc20
Date: 2014-12-01 21:40 +0100
http://bitbucket.org/cffi/creflect/changeset/111c65f6cc20/

Log:	another reorganization, split more the ffi and the lib

diff --git a/zeffir/zef_builder.c b/zeffir/builder.c
rename from zeffir/zef_builder.c
rename to zeffir/builder.c
--- a/zeffir/zef_builder.c
+++ b/zeffir/builder.c
@@ -5,11 +5,36 @@
 typedef struct {
     _crx_builder_t cb;
     ZefLibObject *lib;
+    PyObject *l_dict;
     ZefFFIObject *ffi;
-    PyObject *l_dict;
+    PyObject *types_dict;
 } zeffir_builder_t;
 
-static PyObject *zef_cache_primitive_signed;
+static CTypeDescrObject *get_cached_type(_crx_builder_t *cb, const char *name,
+                                         int accept_flags)
+{
+    PyObject *types_dict = ((zeffir_builder_t *)cb)->types_dict;
+    PyObject *x = PyDict_GetItemString(types_dict, name);
+    CTypeDescrObject *ct;
+
+    if (x == NULL || !CTypeDescr_Check(x))
+        return NULL;
+    ct = (CTypeDescrObject *)x;
+    if ((ct->ct_flags & accept_flags) == 0)
+        return NULL;
+    return ct;
+}
+
+static _crx_type_t *put_cached_type(_crx_builder_t *cb, const char *name,
+                                    CTypeDescrObject *ct)
+{
+    PyObject *types_dict = ((zeffir_builder_t *)cb)->types_dict;
+    int err = PyDict_SetItemString(types_dict, name, (PyObject *)ct);
+    Py_DECREF(ct);
+    if (err < 0)
+        return NULL;
+    return ct;    /* still a reference in the dict */
+}
 
 static _crx_type_t *zef_get_void_type(_crx_builder_t *cb)
 {
@@ -29,33 +54,26 @@
 static _crx_type_t *zef_get_signed_type(_crx_builder_t *cb, size_t sz,
                                         const char *name)
 {
-    PyObject *x = PyDict_GetItemString(zef_cache_primitive_signed, name);
-    CTypeDescrObject *td = (CTypeDescrObject *)x;
+    CTypeDescrObject *td = get_cached_type(cb, name, CT_PRIMITIVE_SIGNED);
+    if (td != NULL && td->ct_size == sz)
+        return td;
 
-    if (td == NULL) {
-        int err;
-        size_t name_length = strlen(name);
+    size_t name_length = strlen(name);
 
-        td = ctypedescr_new(name_length + 1);
-        if (td == NULL)
-            return NULL;
+    td = ctypedescr_new(name_length + 1);
+    if (td == NULL)
+        return NULL;
 
-        memcpy(td->ct_name, name, name_length + 1);
-        td->ct_name_position = name_length;
-        td->ct_size = sz;
-        //td->ct_length = ptypes->align;
-        //td->ct_extra = ffitype;
-        td->ct_flags = CT_PRIMITIVE_SIGNED;
-        if (td->ct_size <= sizeof(long))
-            td->ct_flags |= CT_PRIMITIVE_FITS_LONG;
+    memcpy(td->ct_name, name, name_length + 1);
+    td->ct_name_position = name_length;
+    td->ct_size = sz;
+    //td->ct_length = ptypes->align;
+    //td->ct_extra = ffitype;
+    td->ct_flags = CT_PRIMITIVE_SIGNED;
+    if (td->ct_size <= sizeof(long))
+        td->ct_flags |= CT_PRIMITIVE_FITS_LONG;
 
-        x = (PyObject *)td;
-        err = PyDict_SetItemString(zef_cache_primitive_signed, name, x);
-        Py_DECREF(x);
-        if (err < 0)
-            return NULL;
-    }
-    return td;
+    return put_cached_type(cb, name, td);
 }
 
 static _crx_type_t *zef_get_unsigned_type(_crx_builder_t *cb, size_t sz,
@@ -158,10 +176,11 @@
                             _crx_trampoline0_fn trampl, void *directcall)
 {
     PyObject *l_dict = ((zeffir_builder_t *)cb)->l_dict;
-    ZefFFIObject *ffi = ((zeffir_builder_t *)cb)->ffi;
+    PyObject *l_libname_obj = ((zeffir_builder_t *)cb)->lib->l_libname_obj;
 
     assert(trampl != NULL);
-    PyObject *x = make_builtin_func(ffi, name, ret, args, nargs, trampl);
+    PyObject *x = make_builtin_func(l_libname_obj, name, ret,
+                                    args, nargs, trampl);
     if (x == NULL)
         return;
 
@@ -189,27 +208,27 @@
     abort();
 }
 
-static int load_creflect_main(ZefLibObject *lib)
+static int load_creflect_main(ZefFFIObject *ffi, ZefLibObject *lib)
 {
     const char *creflect_main = "_creflect_main";   /* XXX fixed for now */
     int (*crxmain)(_crx_builder_t *);
-    crxmain = (int(*)(_crx_builder_t *))dlsym(lib->ffi->f_dl_lib,
-                                              creflect_main);
+
+    crxmain = (int(*)(_crx_builder_t *))dlsym(lib->l_dl_lib, creflect_main);
     if (crxmain == NULL) {
         PyErr_Format(PyExc_OSError, "%s: symbol '%s' not found",
-                     lib->ffi->f_libname, creflect_main);
+                     lib->l_libname, creflect_main);
         return -1;
     }
 
     int result = crxmain(NULL);
     if ((result & ~0xFF) != MAGIC_CREFLECT_TAG) {
         PyErr_Format(PyExc_OSError, "%s: %s() is not a creflect entry point",
-                     lib->ffi->f_libname, creflect_main);
+                     lib->l_libname, creflect_main);
         return -1;
     }
     if ((result & 0xFF) != CREFLECT_VERSION) {
         PyErr_Format(PyExc_OSError, "%s was made with creflect version %d, "
-                     "expected %d", lib->ffi->f_libname, result & 0xFF,
+                     "expected %d", lib->l_libname, result & 0xFF,
                      CREFLECT_VERSION);
         return -1;
     }
@@ -240,17 +259,12 @@
             zef_define_num_const,
             zef_error,
         },
-        lib,          /* lib */
-        lib->ffi,     /* ffi */
-        lib->l_dict,  /* l_dict */
+        lib,              /* lib */
+        lib->l_dict,      /* l_dict */
+        ffi,              /* ffi */
+        ffi->types_dict,  /* types_dict */
     };
     crxmain(&builder.cb);
 
     return PyErr_Occurred() ? -1 : 0;
 }
-
-static void init_caches(void)
-{
-    if ((zef_cache_primitive_signed = PyDict_New()) == NULL)
-        return;
-}
diff --git a/zeffir/zef_cdata.c b/zeffir/cdata.c
rename from zeffir/zef_cdata.c
rename to zeffir/cdata.c
diff --git a/zeffir/zef_cfunc.c b/zeffir/cfunc.c
rename from zeffir/zef_cfunc.c
rename to zeffir/cfunc.c
--- a/zeffir/zef_cfunc.c
+++ b/zeffir/cfunc.c
@@ -2,7 +2,6 @@
 typedef struct {
     PyObject_HEAD
 
-    ZefFFIObject       *zfs_ffi;
     int                 zfs_nargs;
     _crx_trampoline0_fn zfs_trampl;
     PyMethodDef         zfs_md;
@@ -12,13 +11,12 @@
 
 static void zef_builtin_support_dealloc(ZefFuncSupportObject *zfs)
 {
-    Py_DECREF(zfs->zfs_ffi);
     PyObject_Del(zfs);
 }
 
 static PyTypeObject ZefFuncSupport_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
-    "zeffir.BuiltinSupport",
+    "zeffir.FuncSupport",
     sizeof(ZefFuncSupportObject),
     0,
     (destructor)zef_builtin_support_dealloc,    /* tp_dealloc */
@@ -67,8 +65,8 @@
     return PyInt_FromLong(lloutput);
 }
 
-static PyObject *make_builtin_func(ZefFFIObject *ffi, const char *funcname,
-                                   _crx_type_t *ret,
+static PyObject *make_builtin_func(PyObject *libname_obj,
+                                   const char *funcname, _crx_type_t *ret,
                                    _crx_qual_type args[], int nargs,
                                    _crx_trampoline0_fn trampl)
 {
@@ -78,8 +76,6 @@
         return PyErr_NoMemory();
     PyObject_Init((PyObject *)zfs, &ZefFuncSupport_Type);
 
-    Py_INCREF(ffi);
-    zfs->zfs_ffi = ffi;
     zfs->zfs_nargs = nargs;
     zfs->zfs_trampl = trampl;
 
@@ -90,7 +86,7 @@
     /*zfs->zfs_md.ml_doc = ... */
 
     PyObject *res = PyCFunction_NewEx(&zfs->zfs_md, (PyObject *)zfs,
-                                      ffi->f_libname_obj);
+                                      libname_obj);
     Py_DECREF(zfs);
     return res;
 }
diff --git a/zeffir/zef_ctype.c b/zeffir/ctype.c
rename from zeffir/zef_ctype.c
rename to zeffir/ctype.c
diff --git a/zeffir/zef_ffi_obj.c b/zeffir/ffi_obj.c
rename from zeffir/zef_ffi_obj.c
rename to zeffir/ffi_obj.c
--- a/zeffir/zef_ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -1,24 +1,125 @@
+
+/* An FFI object has methods like ffi.new().  It is also a container for the
+   type declarations (typedefs and structs) that you can use, say in ffi.new().
+   They are in a dictionary available in the 'types' attribute, which you
+   can edit manually (e.g. 'ffi.types.update(ffi2.types)').
+
+   The method ffi.load_library() finds a creflect dynamic library, load
+   its type declarations into ffi.types, and returns a Lib object with
+   the functions, global variables and constants as attributes.  Once the
+   Lib object goes out of scope (or ffi.close_library() is called), the
+   library is unloaded; accessing functions or global variables afterwards
+   will segfault.
+
+   There is no direct internal reference between the FFI and the Lib object.
+*/
 
 struct ZefFFIObject_s {
     PyObject_HEAD
-    void *f_dl_lib;      /* keep the library returned by 'dlopen()' */
-    char *f_libname;     /* the original argument to 'dlopen()' */
-    PyObject *f_libname_obj;   /* same, as a PyObject */
+    PyObject *types_dict;
 };
 
 static void ffi_dealloc(ZefFFIObject *ffi)
 {
-    if (ffi->f_dl_lib != NULL)
-        dlclose(ffi->f_dl_lib);
-    Py_DECREF(ffi->f_libname_obj);
-    PyObject_Del(ffi);
+    PyObject_GC_UnTrack(ffi);
+    Py_DECREF(ffi->types_dict);
+    PyObject_GC_Del(ffi);
 }
 
-static PyObject *ffi_repr(ZefFFIObject *ffi)
+static int ffi_traverse(ZefFFIObject *ffi, visitproc visit, void *arg)
 {
-    return PyString_FromFormat("<zeffir.FFI object for '%s'>", ffi->f_libname);
+    Py_VISIT(ffi->types_dict);
+    return 0;
 }
 
+static PyObject *ffiobj_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    char *keywords[] = {NULL};
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "", keywords))
+        return NULL;
+
+    PyObject *dict = PyDict_New();
+    if (dict == NULL)
+        return NULL;
+
+    ZefFFIObject *ffi = PyObject_GC_New(ZefFFIObject, &ZefFFI_Type);
+    if (ffi == NULL) {
+        Py_DECREF(dict);
+        return NULL;
+    }
+    ffi->types_dict = dict;
+    PyObject_GC_Track(ffi);
+
+    return (PyObject *)ffi;
+}
+
+static PyObject *ffi_load_library(ZefFFIObject *self, PyObject *args,
+                                  PyObject *kwds)
+{
+    char *keywords[] = {"libname", "relative_to", NULL};
+    char *libname, *relative_to = NULL;
+    char *relative_dir;
+    PyObject *path;
+    ZefLibObject *lib;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|z", keywords,
+                                     &libname, &relative_to))
+        return NULL;
+
+    if (relative_to == NULL) {
+        relative_dir = strdup("");
+    }
+    else {
+        char *last_slash = strrchr(relative_to, '/');
+        if (last_slash == NULL)
+            relative_dir = strdup("./");
+        else
+            relative_dir = strndup(relative_to, last_slash + 1 - relative_to);
+    }
+    if (relative_dir == NULL)
+        return PyErr_NoMemory();
+
+    path = PyString_FromFormat("%slib%s.so", relative_dir, libname);
+    free(relative_dir);
+    if (path == NULL)
+        return NULL;
+
+    lib = lib_create(path);
+    Py_DECREF(path);
+
+    if (load_creflect_main(self, lib) < 0) {
+        Py_DECREF(lib);
+        return NULL;
+    }
+    return (PyObject *)lib;
+}
+
+static PyObject *ffi_close_library(PyObject *no_self, PyObject *args)
+{
+    ZefLibObject *lib;
+
+    if (!PyArg_ParseTuple(args, "O!", &lib, &ZefLib_Type))
+        return NULL;
+
+    if (lib_close(lib) < 0)
+        return NULL;
+
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+static PyMethodDef ffi_methods[] = {
+    {"load_library",   (PyCFunction)ffi_load_library,
+                                           METH_VARARGS | METH_KEYWORDS},
+    {"close_library",  ffi_close_library,  METH_VARARGS | METH_STATIC},
+    {NULL}
+};
+
+static PyMemberDef ffi_members[] = {
+    {"types", T_OBJECT, offsetof(ZefFFIObject, types_dict), READONLY},
+    {NULL}
+};
+
 static PyTypeObject ZefFFI_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "zeffir.FFI",
@@ -29,7 +130,7 @@
     0,                                          /* tp_getattr */
     0,                                          /* tp_setattr */
     0,                                          /* tp_compare */
-    (reprfunc)ffi_repr,                         /* tp_repr */
+    0,                                          /* tp_repr */
     0,                                          /* tp_as_number */
     0,                                          /* tp_as_sequence */
     0,                                          /* tp_as_mapping */
@@ -39,14 +140,24 @@
     PyObject_GenericGetAttr,                    /* tp_getattro */
     0,                                          /* tp_setattro */
     0,                                          /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,    /* tp_flags */
     0,                                          /* tp_doc */
-    0,                                          /* tp_traverse */
+    (traverseproc)ffi_traverse,                 /* tp_traverse */
     0,                                          /* tp_clear */
     0,                                          /* tp_richcompare */
     0,                                          /* tp_weaklistoffset */
     0,                                          /* tp_iter */
     0,                                          /* tp_iternext */
-    0,                                          /* tp_methods */
-    0,                                          /* tp_members */
+    ffi_methods,                                /* tp_methods */
+    ffi_members,                                /* tp_members */
+    0,                                          /* tp_getset */
+    0,                                          /* tp_base */
+    0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    0,                                          /* tp_dictoffset */
+    0,                                          /* tp_init */
+    0,                                          /* tp_alloc */
+    ffiobj_new,                                 /* tp_new */
+    PyObject_GC_Del,                            /* tp_free */
 };
diff --git a/zeffir/zef_lib_obj.c b/zeffir/lib_obj.c
rename from zeffir/zef_lib_obj.c
rename to zeffir/lib_obj.c
--- a/zeffir/zef_lib_obj.c
+++ b/zeffir/lib_obj.c
@@ -1,21 +1,47 @@
+
+/* A Lib object is special in the sense that it has a custom __getattr__
+   which only returns objects from the internal, inaccessible 'l_dict'.
+   (It raises AttributeError for anything else, like '__class__'.)
+*/
 
 struct ZefLibObject_s {
     PyObject_HEAD
-    PyObject *l_dict;
-    ZefFFIObject *ffi;
+    PyObject *l_dict;         /* functions, global variables and constants */
+    void *l_dl_lib;           /* keep the library returned by 'dlopen()' */
+    char *l_libname;          /* the original argument to 'dlopen()' */
+    PyObject *l_libname_obj;  /* same, as a PyObject */
 };
 
 static void lib_dealloc(ZefLibObject *lib)
 {
-    Py_DECREF(lib->l_dict);
-    Py_DECREF(lib->ffi);
+    (void)lib_close(lib);
     PyObject_Del(lib);
 }
 
 static PyObject *lib_repr(ZefLibObject *lib)
 {
-    return PyString_FromFormat("<zeffir.Lib object for '%s'>",
-                               lib->ffi->f_libname);
+    return PyString_FromFormat("<zeffir.Lib object for '%.200s'%s>",
+                               lib->l_libname,
+                               lib->l_dl_lib == NULL ? " (closed)" : "");
+}
+
+static PyObject *lib_getattr(ZefLibObject *lib, PyObject *name)
+{
+    if (lib->l_dict == NULL) {
+        PyErr_Format(PyExc_ValueError, "lib '%.200s' was closed",
+                     lib->l_libname);
+        return NULL;
+    }
+
+    PyObject *x = PyDict_GetItem(lib->l_dict, name);
+    if (x == NULL) {
+        PyErr_Format(PyExc_AttributeError,
+                     "lib '%.200s' has no function,"
+                     " global variable or constant '%.200s'",
+                     lib->l_libname,
+                     PyString_Check(name) ? PyString_AS_STRING(name) : "?");
+    }
+    return x;
 }
 
 static PyObject *lib_dir(PyObject *lib, PyObject *noarg)
@@ -28,11 +54,6 @@
     {NULL,        NULL}           /* sentinel */
 };
 
-static PyMemberDef lib_members[] = {
-    {"__dict__", T_OBJECT, offsetof(ZefLibObject, l_dict), READONLY},
-    {NULL}
-};
-
 static PyTypeObject ZefLib_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "zeffir.Lib",
@@ -50,7 +71,7 @@
     0,                                          /* tp_hash */
     0,                                          /* tp_call */
     0,                                          /* tp_str */
-    PyObject_GenericGetAttr,                    /* tp_getattro */
+    (getattrofunc)lib_getattr,                  /* tp_getattro */
     0,                                          /* tp_setattro */
     0,                                          /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT,                         /* tp_flags */
@@ -62,7 +83,7 @@
     0,                                          /* tp_iter */
     0,                                          /* tp_iternext */
     lib_methods,                                /* tp_methods */
-    lib_members,                                /* tp_members */
+    0,                                          /* tp_members */
     0,                                          /* tp_getset */
     0,                                          /* tp_base */
     0,                                          /* tp_dict */
@@ -70,3 +91,54 @@
     0,                                          /* tp_descr_set */
     offsetof(ZefLibObject, l_dict),              /* tp_dictoffset */
 };
+
+static void lib_dlerror(ZefLibObject *lib)
+{
+    char *error = dlerror();
+    if (error == NULL)
+        error = "(no error reported)";
+    PyErr_Format(PyExc_OSError, "%s: %s", lib->l_libname, error);
+}
+
+static ZefLibObject *lib_create(PyObject *path)
+{
+    ZefLibObject *lib;
+
+    lib = PyObject_New(ZefLibObject, &ZefLib_Type);
+    if (lib == NULL)
+        return NULL;
+
+    lib->l_dl_lib = NULL;
+    lib->l_libname = PyString_AS_STRING(path);
+    Py_INCREF(path);
+    lib->l_libname_obj = path;
+    lib->l_dict = PyDict_New();
+    if (lib->l_dict == NULL) {
+        Py_DECREF(lib);
+        return NULL;
+    }
+
+    lib->l_dl_lib = dlopen(lib->l_libname, RTLD_LAZY);
+    if (lib->l_dl_lib == NULL) {
+        lib_dlerror(lib);
+        Py_DECREF(lib);
+        return NULL;
+    }
+    return lib;
+}
+
+static int lib_close(ZefLibObject *lib)
+{
+    void *dl_lib;
+    Py_CLEAR(lib->l_dict);
+
+    dl_lib = lib->l_dl_lib;
+    if (dl_lib != NULL) {
+        lib->l_dl_lib = NULL;
+        if (dlclose(dl_lib) != 0) {
+            lib_dlerror(lib);
+            return -1;
+        }
+    }
+    return 0;
+}
diff --git a/zeffir/test/support.py b/zeffir/test/support.py
--- a/zeffir/test/support.py
+++ b/zeffir/test/support.py
@@ -11,9 +11,8 @@
     if err:
         raise OSError("%r\n==> error code %s" % (cmd, err))
 
-def compile_and_open(modname):
-    """Process with creflect 'modname.crx', compile the result, and load it."""
 
+def new_ffi():
     if not os.path.exists(zeffir_lib_path):
         run("cd '%s' && "
             "gcc -g -I'%s' -Wall -Werror -fPIC -shared zeffir.c -o '%s'" %
@@ -24,6 +23,13 @@
     import zeffir
     del sys.path[0]
 
+    return zeffir.FFI()
+
+
+def compile_and_open(modname):
+    """Process with creflect 'modname.crx', compile the result, and load it."""
+
+    ffi = new_ffi()
     mod_c_path = str(udir.join('%s.c' % modname))
     mod_lib_path = str(udir.join('lib%s.so' % modname))
     if not os.path.exists(mod_lib_path):
@@ -33,4 +39,5 @@
             (zeffir_dir, modname, mod_c_path, mod_c_path, mod_lib_path))
         assert os.path.exists(mod_lib_path)
 
-    return zeffir.open(modname, relative_to=zeffir_lib_path)
+    lib = ffi.load_library(modname, relative_to=zeffir_lib_path)
+    return ffi, lib
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -1,3 +1,4 @@
+import py
 import os
 import support
 
@@ -6,7 +7,7 @@
     ffi, lib = support.compile_and_open('basic')
     assert type(ffi).__module__ == 'zeffir'
     assert type(ffi).__name__ == 'FFI'
-    assert repr(ffi) == "<zeffir.FFI object for '%s/libbasic.so'>" % (
+    assert repr(lib) == "<zeffir.Lib object for '%s/libbasic.so'>" % (
         os.path.dirname(support.zeffir_lib_path),)
 
 def test_forty_two():
@@ -17,3 +18,7 @@
 def test_dir():
     ffi, lib = support.compile_and_open('basic')
     assert dir(lib) == ['forty_two']
+
+def test_no_special_attribute():
+    ffi, lib = support.compile_and_open('basic')
+    py.test.raises(AttributeError, getattr, lib, '__class__')
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -5,98 +5,21 @@
 #include "../creflect/creflect_cdecl.h"
 
 
-typedef struct _crx_type_s CTypeDescrObject;
-typedef struct ZefLibObject_s ZefLibObject;
-typedef struct ZefFFIObject_s ZefFFIObject;
-
-
 /************************************************************/
 /* Works by including all other .c files.                   */
 /* Allows all function and global symbols to remain static. */
 
-#include "zef_ctype.c"
-#include "zef_cdata.c"
-#include "zef_ffi_obj.c"
-#include "zef_lib_obj.c"
-#include "zef_cfunc.c"
-#include "zef_builder.c"
+#include "zeffir.h"
+#include "ctype.c"
+#include "cdata.c"
+#include "lib_obj.c"
+#include "ffi_obj.c"
+#include "cfunc.c"
+#include "builder.c"
 
 /************************************************************/
 
-
-static PyObject *z_open(PyObject *self, PyObject *args, PyObject *kwds)
-{
-    char *keywords[] = {"libname", "relative_to", NULL};
-    char *libname, *relative_to, *relative_dir;
-    PyObject *path;
-
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "ss", keywords,
-                                     &libname, &relative_to))
-        return NULL;
-
-    char *last_slash = strrchr(relative_to, '/');
-    if (last_slash == NULL)
-        relative_dir = strdup(".");
-    else
-        relative_dir = strndup(relative_to, last_slash - relative_to);
-    if (relative_dir == NULL)
-        return PyErr_NoMemory();
-
-    path = PyString_FromFormat("%s/lib%s.so", relative_dir, libname);
-    free(relative_dir);
-    if (path == NULL)
-        return NULL;
-
-    ZefFFIObject *ffi = NULL;
-    ZefLibObject *lib = NULL;
-    PyObject *dict = NULL;
-
-    dlerror();   /* clear errors */
-    void *dl_lib = dlopen(PyString_AS_STRING(path), RTLD_LAZY);
-    if (dl_lib == NULL) {
-        char *error = dlerror();
-        if (error == NULL)
-            error = "failed to open";
-        PyErr_Format(PyExc_OSError, "%s: %s", PyString_AS_STRING(path), error);
-        goto error;
-    }
-
-    ffi = PyObject_New(ZefFFIObject, &ZefFFI_Type);
-    if (ffi == NULL)
-        goto error;
-    ffi->f_dl_lib = dl_lib;     dl_lib = NULL;
-    ffi->f_libname = PyString_AS_STRING(path);
-    ffi->f_libname_obj = path;  path = NULL;
-
-    dict = PyDict_New();
-    if (dict == NULL)
-        goto error;
-
-    lib = PyObject_New(ZefLibObject, &ZefLib_Type);
-    if (lib == NULL)
-        goto error;
-    lib->ffi = ffi;      ffi = NULL;
-    lib->l_dict = dict;  dict = NULL;
-
-    if (load_creflect_main(lib) < 0)
-        goto error;
-
-    PyObject *result = Py_BuildValue("OO", lib->ffi, lib);
-    Py_DECREF(lib);
-    return result;
-
- error:
-    Py_XDECREF(dict);
-    Py_XDECREF(lib);
-    Py_XDECREF(ffi);
-    if (dl_lib != NULL)
-        dlclose(dl_lib);
-    Py_XDECREF(path);
-    return NULL;
-}
-
 static PyMethodDef ZeffirMethods[] = {
-    {"open",   (PyCFunction)z_open, METH_KEYWORDS},
     {NULL,     NULL}    /* Sentinel */
 };
 
@@ -106,10 +29,7 @@
     PyObject *m;
 
     m = Py_InitModule("zeffir", ZeffirMethods);
-    (void)m;
-
-    init_caches();
-    if (PyErr_Occurred())
+    if (m == NULL)
         return;
 
     if (PyType_Ready(&ZefFFI_Type) < 0)
@@ -120,4 +40,7 @@
         return;
     if (PyType_Ready(&ZefFuncSupport_Type) < 0)
         return;
+
+    if (PyModule_AddObject(m, "FFI", (PyObject *)&ZefFFI_Type) < 0)
+        return;
 }
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
new file mode 100644
--- /dev/null
+++ b/zeffir/zeffir.h
@@ -0,0 +1,9 @@
+
+typedef struct _crx_type_s CTypeDescrObject;
+typedef struct ZefLibObject_s ZefLibObject;
+typedef struct ZefFFIObject_s ZefFFIObject;
+
+static PyTypeObject ZefFFI_Type;
+
+static int lib_close(ZefLibObject *);
+static int load_creflect_main(ZefFFIObject *, ZefLibObject *);

From noreply at buildbot.pypy.org  Mon Dec  1 22:14:29 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Dec 2014 22:14:29 +0100 (CET)
Subject: [pypy-commit] creflect default: add minimal test
Message-ID: <20141201211429.DA4A21C34C4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r138:26ec90ac79fe
Date: 2014-12-01 21:41 +0100
http://bitbucket.org/cffi/creflect/changeset/26ec90ac79fe/

Log:	add minimal test

diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -22,3 +22,7 @@
 def test_no_special_attribute():
     ffi, lib = support.compile_and_open('basic')
     py.test.raises(AttributeError, getattr, lib, '__class__')
+
+def test_types():
+    ffi = support.new_ffi()
+    assert ffi.types == {}

From noreply at buildbot.pypy.org  Mon Dec  1 22:14:31 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  1 Dec 2014 22:14:31 +0100 (CET)
Subject: [pypy-commit] creflect default: in-progress: parsing types given in
	ffi.xyz()
Message-ID: <20141201211431.08CED1C34C4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r139:c04b2a53ae95
Date: 2014-12-01 22:14 +0100
http://bitbucket.org/cffi/creflect/changeset/c04b2a53ae95/

Log:	in-progress: parsing types given in ffi.xyz()

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -268,3 +268,46 @@
 
     return PyErr_Occurred() ? -1 : 0;
 }
+
+static CTypeDescrObject *parse_c_decl(ZefFFIObject *ffi, const char *str)
+{
+    zeffir_builder_t builder = {
+        {
+            zef_get_void_type,
+            zef_get_char_type,
+            zef_get_bool_type,
+            zef_get_signed_type,
+            zef_get_unsigned_type,
+            zef_get_float_type,
+            zef_get_function_type,
+            zef_get_ellipsis_function_type,
+            zef_get_pointer_type,
+            zef_get_array_type,
+            zef_get_incomplete_array_type,
+            zef_get_struct_type,
+            zef_get_union_type,
+            zef_get_enum_type,
+            zef_get_user_type,
+            zef_get_unknown_type,
+            0,  /* complete */
+            0,  /* complete_enum */
+            0,  /* define_type */
+            0,  /* define_var */
+            0,  /* define_func */
+            0,  /* define_num_const */
+            zef_error,
+        },
+        NULL,             /* lib */
+        NULL,             /* l_dict */
+        ffi,              /* ffi */
+        ffi->types_dict,  /* types_dict */
+    };
+
+    _crx_qual_type result;
+    const char *err = creflect_decl_parser(&builder.cb, str, &result);
+    if (err != NULL)
+        abort();
+
+    Py_INCREF(result.type);
+    return result.type;
+}
diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -81,6 +81,11 @@
     return ct;
 }
 
+static PyObject *ctypedescr_repr(CTypeDescrObject *ct)
+{
+    return PyString_FromFormat("<ctype '%s'>", ct->ct_name);
+}
+
 static void ctypedescr_dealloc(CTypeDescrObject *ct)
 {
     PyObject_GC_UnTrack(ct);
@@ -109,7 +114,7 @@
 
 static PyTypeObject CTypeDescr_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
-    "_cffi_backend.CTypeDescr",
+    "zeffir.CType",
     offsetof(CTypeDescrObject, ct_name),
     sizeof(char),
     (destructor)ctypedescr_dealloc,             /* tp_dealloc */
@@ -117,7 +122,7 @@
     0,                                          /* tp_getattr */
     0,                                          /* tp_setattr */
     0,                                          /* tp_compare */
-    0,//(reprfunc)ctypedescr_repr,                  /* tp_repr */
+    (reprfunc)ctypedescr_repr,                  /* tp_repr */
     0,                                          /* tp_as_number */
     0,                                          /* tp_as_sequence */
     0,                                          /* tp_as_mapping */
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -108,10 +108,30 @@
     return Py_None;
 }
 
+static PyObject *ffi_typeof(ZefFFIObject *self, PyObject *arg)
+{
+    if (!PyString_Check(arg)) {
+        PyErr_SetString(PyExc_TypeError, "XXX");
+        return NULL;
+    }
+
+    PyObject *x = PyDict_GetItem(self->types_dict, arg);
+    if (x != NULL) {
+        Py_INCREF(x);
+        return x;
+    }
+
+    x = (PyObject *)parse_c_decl(self, PyString_AS_STRING(arg));
+    if (x != NULL)
+        PyDict_SetItem(self->types_dict, arg, x);
+    return x;
+}
+
 static PyMethodDef ffi_methods[] = {
-    {"load_library",   (PyCFunction)ffi_load_library,
-                                           METH_VARARGS | METH_KEYWORDS},
-    {"close_library",  ffi_close_library,  METH_VARARGS | METH_STATIC},
+    {"close_library", ffi_close_library,         METH_VARARGS | METH_STATIC},
+    {"load_library",  (PyCFunction)ffi_load_library,
+                                                 METH_VARARGS | METH_KEYWORDS},
+    {"typeof",        (PyCFunction)ffi_typeof,   METH_O},
     {NULL}
 };
 
diff --git a/zeffir/lib_obj.c b/zeffir/lib_obj.c
--- a/zeffir/lib_obj.c
+++ b/zeffir/lib_obj.c
@@ -28,8 +28,7 @@
 static PyObject *lib_getattr(ZefLibObject *lib, PyObject *name)
 {
     if (lib->l_dict == NULL) {
-        PyErr_Format(PyExc_ValueError, "lib '%.200s' was closed",
-                     lib->l_libname);
+        PyErr_Format(ZefError, "lib '%.200s' was closed", lib->l_libname);
         return NULL;
     }
 
diff --git a/zeffir/test/test_ctype.py b/zeffir/test/test_ctype.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_ctype.py
@@ -0,0 +1,8 @@
+import support
+
+
+def test_typeof():
+    ffi = support.new_ffi()
+    assert repr(ffi.typeof("int")) == "<ctype 'int'>"
+    assert repr(ffi.typeof("long int")) == "<ctype 'long'>"
+    #assert repr(ffi.typeof("int*")) == "<ctype 'int *'>"
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -16,6 +16,7 @@
 #include "ffi_obj.c"
 #include "cfunc.c"
 #include "builder.c"
+#include "../creflect/creflect_cdecl.c"
 
 /************************************************************/
 
@@ -43,4 +44,13 @@
 
     if (PyModule_AddObject(m, "FFI", (PyObject *)&ZefFFI_Type) < 0)
         return;
+    if (PyModule_AddObject(m, "CType", (PyObject *)&CTypeDescr_Type) < 0)
+        return;
+
+    ZefError = PyErr_NewException("zeffir.error", NULL, NULL);
+    if (ZefError == NULL)
+        return;
+    Py_INCREF(ZefError);
+    if (PyModule_AddObject(m, "error", ZefError) < 0)
+        return;
 }
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -5,5 +5,8 @@
 
 static PyTypeObject ZefFFI_Type;
 
+static PyObject *ZefError;
+
 static int lib_close(ZefLibObject *);
 static int load_creflect_main(ZefFFIObject *, ZefLibObject *);
+static CTypeDescrObject *parse_c_decl(ZefFFIObject *ffi, const char *str);

From noreply at buildbot.pypy.org  Tue Dec  2 12:52:31 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  2 Dec 2014 12:52:31 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: move implementation-level methods to
 base file to simplify circular imports
Message-ID: <20141202115231.356F31C3581@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r74780:34c0bc5ac7e8
Date: 2014-12-02 13:50 +0200
http://bitbucket.org/pypy/pypy/changeset/34c0bc5ac7e8/

Log:	move implementation-level methods to base file to simplify circular
	imports

diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -25,6 +25,7 @@
 
 class W_NDimArray(W_NumpyObject):
     __metaclass__ = extendabletype
+    _immutable_fields_ = ['implementation']
 
     def __init__(self, implementation):
         from pypy.module.micronumpy.concrete import BaseConcreteArray
@@ -98,6 +99,19 @@
         w_arr.set_scalar_value(w_scalar)
         return w_arr
 
+    def get_shape(self):
+        return self.implementation.get_shape()
+
+    def get_dtype(self):
+        return self.implementation.dtype
+
+    def get_order(self):
+        return self.implementation.order
+
+    def ndims(self):
+        return len(self.get_shape())
+    ndims._always_inline_ = True
+
 
 def convert_to_array(space, w_obj):
     from pypy.module.micronumpy.ctors import array
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -50,9 +50,6 @@
         shape = self.get_shape()
         return space.newtuple([space.wrap(i) for i in shape])
 
-    def get_shape(self):
-        return self.implementation.get_shape()
-
     def descr_set_shape(self, space, w_new_shape):
         shape = get_shape_from_iterable(space, self.get_size(), w_new_shape)
         self.implementation = self.implementation.set_shape(space, self, shape)
@@ -61,12 +58,6 @@
         strides = self.implementation.get_strides()
         return space.newtuple([space.wrap(i) for i in strides])
 
-    def get_dtype(self):
-        return self.implementation.dtype
-
-    def get_order(self):
-        return self.implementation.order
-
     def descr_get_dtype(self, space):
         return self.implementation.dtype
 
@@ -83,10 +74,6 @@
         raise OperationError(space.w_AttributeError, space.wrap(
             "Cannot delete array dtype"))
 
-    def ndims(self):
-        return len(self.get_shape())
-    ndims._always_inline_ = True
-
     def descr_get_ndim(self, space):
         return space.wrap(self.ndims())
 

From noreply at buildbot.pypy.org  Tue Dec  2 12:52:32 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  2 Dec 2014 12:52:32 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: pass resolver tests,
 still need to use results in creating output arrays
Message-ID: <20141202115232.82F3D1C3581@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r74781:af2cb61cc4b1
Date: 2014-12-02 13:50 +0200
http://bitbucket.org/pypy/pypy/changeset/af2cb61cc4b1/

Log:	pass resolver tests, still need to use results in creating output
	arrays

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -108,18 +108,25 @@
 
     def test_type_resolver(self, space):
         c128_dtype = get_dtype_cache(space).w_complex128dtype
+        c64_dtype = get_dtype_cache(space).w_complex64dtype
         f64_dtype = get_dtype_cache(space).w_float64dtype
+        f32_dtype = get_dtype_cache(space).w_float32dtype
         u32_dtype = get_dtype_cache(space).w_uint32dtype
         b_dtype = get_dtype_cache(space).w_booldtype
 
-        ufunc = W_UfuncGeneric(space, [None, None], 'eigenvals', None, 1, 1,
-                     [f64_dtype, c128_dtype, c128_dtype, c128_dtype],
+        ufunc = W_UfuncGeneric(space, [None, None, None], 'eigenvals', None, 1, 1,
+                     [f32_dtype, c64_dtype, 
+                      f64_dtype, c128_dtype, 
+                      c128_dtype, c128_dtype],
                      '')
-        f64 = W_NDimArray(VoidBoxStorage(0, f64_dtype))
-        c128 = W_NDimArray(VoidBoxStorage(0, c128_dtype))
-        index, dtypes = ufunc.type_resolver(space, [f64], [c128], 'd->D')
+        f32_array = W_NDimArray(VoidBoxStorage(0, f32_dtype))
+        index, dtypes = ufunc.type_resolver(space, [f32_array], [None], 'd->D')
+        #needs to cast input type, create output type
+        assert index == 1
+        assert dtypes == [f64_dtype, c128_dtype]
+        index, dtypes = ufunc.type_resolver(space, [f32_array], [None], '')
         assert index == 0
-        assert dtypes == [f64_dtype, c128_dtype]
+        assert dtypes == [f32_dtype, c64_dtype]
 
 
 class AppTestUfuncs(BaseNumpyAppTest):
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -6,7 +6,9 @@
 from rpython.rlib import jit
 from rpython.rlib.rarithmetic import LONG_BIT, maxint
 from rpython.tool.sourcetools import func_with_new_name
-from pypy.module.micronumpy import boxes, descriptor, loop, constants as NPY
+from pypy.module.micronumpy import boxes, loop, constants as NPY
+from pypy.module.micronumpy.descriptor import (get_dtype_cache,
+            variable_dtype, decode_w_dtype)
 from pypy.module.micronumpy.base import convert_to_array, W_NDimArray
 from pypy.module.micronumpy.ctors import numpify
 from pypy.module.micronumpy.nditer import W_NDIter, coalesce_iter
@@ -201,10 +203,10 @@
             if axis < 0:
                 axis += shapelen
         assert axis >= 0
-        dtype = descriptor.decode_w_dtype(space, dtype)
+        dtype = decode_w_dtype(space, dtype)
         if dtype is None:
             if self.comparison_func:
-                dtype = descriptor.get_dtype_cache(space).w_booldtype
+                dtype = get_dtype_cache(space).w_booldtype
             else:
                 dtype = find_unaryop_result_dtype(
                     space, obj.get_dtype(),
@@ -360,14 +362,14 @@
             #    raise oefmt(space.w_TypeError,
             #        "Cannot cast ufunc %s output from dtype('%s') to dtype('%s') with casting rule 'same_kind'", self.name, w_obj.get_dtype().name, res_dtype.name)
         elif self.bool_result:
-            res_dtype = descriptor.get_dtype_cache(space).w_booldtype
+            res_dtype = get_dtype_cache(space).w_booldtype
         else:
             res_dtype = calc_dtype
             if self.complex_to_float and calc_dtype.is_complex():
                 if calc_dtype.num == NPY.CFLOAT:
-                    res_dtype = descriptor.get_dtype_cache(space).w_float32dtype
+                    res_dtype = get_dtype_cache(space).w_float32dtype
                 else:
-                    res_dtype = descriptor.get_dtype_cache(space).w_float64dtype
+                    res_dtype = get_dtype_cache(space).w_float64dtype
         if w_obj.is_scalar():
             w_val = self.func(calc_dtype,
                               w_obj.get_scalar_value().convert_to(space, calc_dtype))
@@ -474,7 +476,7 @@
             out = w_out
             calc_dtype = out.get_dtype()
         if self.comparison_func:
-            res_dtype = descriptor.get_dtype_cache(space).w_booldtype
+            res_dtype = get_dtype_cache(space).w_booldtype
         else:
             res_dtype = calc_dtype
         if w_lhs.is_scalar() and w_rhs.is_scalar():
@@ -559,6 +561,7 @@
                  self.name, self.nin, len(args_w))
         for i in range(self.nin):
             inargs[i] = convert_to_array(space, args_w[i])
+            assert isinstance(inargs[i], W_NDimArray)
         outargs = [None] * self.nout
         for i in range(len(args_w)-self.nin):
             out = args_w[i+self.nin]
@@ -784,7 +787,7 @@
             if sig:
                 raise oefmt(space.w_RuntimeError,
                         "cannot specify both 'sig' and 'dtype'")
-            dtype = descriptor.decode_w_dtype(space, dtype_w)
+            dtype = decode_w_dtype(space, dtype_w)
             sig = space.newtuple([dtype])
         order = kwargs_w.pop('dtype', None)
         if not space.is_w(order, space.w_None) and not order is None:
@@ -810,32 +813,54 @@
         # linear_search_type_resolver in numpy ufunc_type_resolutions.c
         # type_tup can be '', a tuple of dtypes, or a string
         # of the form d,t -> D where the letters are dtype specs
-        inargs0 = inargs[0]
-        assert isinstance(inargs0, W_NDimArray)
         nop = len(inargs) + len(outargs)
         dtypes = []
-        if isinstance(type_tup, str):
+        if isinstance(type_tup, str) and len(type_tup) > 0:
             if len(type_tup) == 1:
                 dtypes = [get_dtype_cache(space).dtypes_by_name[type_tup]] * self.nargs
             elif len(type_tup) == self.nargs + 2:
-                for i in range(len(inargs)):
-                    dtypes.append(get_dtype_cache(space).dtype_by_name[type_tup[i]])
+                for i in range(self.nin):
+                    dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[i]])
                 #skip the '->' in the signature
-                for i in range(len(self.nout)):
-                    dtypes.append(get_dtype_cache(space).dtype_by_name[type_tup[i+2]])
+                for i in range(self.nout):
+                    j = i + self.nin + 2
+                    dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[j]])
             else:
-                raise oefmt(space.w_TypeError, "a type-string for %s," \
+                raise oefmt(space.w_TypeError, "a type-string for %s " \
                     "requires 1 typecode or %d typecode(s) before and %d" \
-                    " after the -> sign", self.name, self.nin, self.nout)
+                    " after the -> sign, not '%s'", self.name, self.nin, 
+                    self.nout, type_tup)
+        else:
+            # XXX does not pass translation
+            # dtypes = [i.get_dtype() for i in inargs]
+            for i in inargs:
+                if isinstance(i, W_NDimArray):
+                    dtypes.append(i.get_dtype())
+                else:
+                    dtypes.append(None)
+            for i in outargs:
+                if isinstance(i, W_NDimArray):
+                    dtypes.append(i.get_dtype())
+                else:
+                    dtypes.append(None)
+        #Find the first matchup of dtypes with self.dtypes
         for i in range(0, len(self.dtypes), self.nargs):
-            if inargs0.get_dtype() == self.dtypes[i]:
+            allok = True
+            for j in range(self.nargs):
+                if dtypes[j] is not None and dtypes[j] != self.dtypes[i+j]:
+                    allok = False
+            if allok:    
                 break
         else:
             if len(self.funcs) < 2:
                 return 0, dtypes
             raise oefmt(space.w_TypeError,
-                         'input dtype %s did not match any known dtypes',
-                                              str(inargs0.get_dtype()))
+                         'input dtype did not match any known dtypes',
+                       )
+        # Fill in empty dtypes
+        for j in range(self.nargs):
+            if dtypes[j] is None:
+                dtypes[j] = self.dtypes[i+j]
         return i / self.nargs, dtypes
 
     def alloc_outargs(self, space, index, inargs, outargs, dtypes):
@@ -880,7 +905,7 @@
         dt1, dt2 = dt2, dt1
     # Some operations promote op(bool, bool) to return int8, rather than bool
     if promote_bools and (dt1.kind == dt2.kind == NPY.GENBOOLLTR):
-        return descriptor.get_dtype_cache(space).w_int8dtype
+        return get_dtype_cache(space).w_int8dtype
 
     # Everything numeric promotes to complex
     if dt2.is_complex() or dt1.is_complex():
@@ -888,16 +913,16 @@
             dt1, dt2 = dt2, dt1
         if dt2.num == NPY.CFLOAT:
             if dt1.num == NPY.DOUBLE:
-                return descriptor.get_dtype_cache(space).w_complex128dtype
+                return get_dtype_cache(space).w_complex128dtype
             elif dt1.num == NPY.LONGDOUBLE:
-                return descriptor.get_dtype_cache(space).w_complexlongdtype
-            return descriptor.get_dtype_cache(space).w_complex64dtype
+                return get_dtype_cache(space).w_complexlongdtype
+            return get_dtype_cache(space).w_complex64dtype
         elif dt2.num == NPY.CDOUBLE:
             if dt1.num == NPY.LONGDOUBLE:
-                return descriptor.get_dtype_cache(space).w_complexlongdtype
-            return descriptor.get_dtype_cache(space).w_complex128dtype
+                return get_dtype_cache(space).w_complexlongdtype
+            return get_dtype_cache(space).w_complex128dtype
         elif dt2.num == NPY.CLONGDOUBLE:
-            return descriptor.get_dtype_cache(space).w_complexlongdtype
+            return get_dtype_cache(space).w_complexlongdtype
         else:
             raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
 
@@ -912,11 +937,11 @@
     # Everything promotes to float, and bool promotes to everything.
     if dt2.kind == NPY.FLOATINGLTR or dt1.kind == NPY.GENBOOLLTR:
         if dt2.num == NPY.HALF and dt1.itemtype.get_element_size() == 2:
-            return descriptor.get_dtype_cache(space).w_float32dtype
+            return get_dtype_cache(space).w_float32dtype
         if dt2.num == NPY.HALF and dt1.itemtype.get_element_size() >= 4:
-            return descriptor.get_dtype_cache(space).w_float64dtype
+            return get_dtype_cache(space).w_float64dtype
         if dt2.num == NPY.FLOAT and dt1.itemtype.get_element_size() >= 4:
-            return descriptor.get_dtype_cache(space).w_float64dtype
+            return get_dtype_cache(space).w_float64dtype
         return dt2
 
     # for now this means mixing signed and unsigned
@@ -942,7 +967,7 @@
     else:
         # increase to the next signed type
         dtypenum = dt2.num + 1
-    newdtype = descriptor.get_dtype_cache(space).dtypes_by_num[dtypenum]
+    newdtype = get_dtype_cache(space).dtypes_by_num[dtypenum]
 
     if (newdtype.itemtype.get_element_size() > dt2.itemtype.get_element_size() or
             newdtype.kind == NPY.FLOATINGLTR):
@@ -951,7 +976,7 @@
         # we only promoted to long on 32-bit or to longlong on 64-bit
         # this is really for dealing with the Long and Ulong dtypes
         dtypenum += 2
-        return descriptor.get_dtype_cache(space).dtypes_by_num[dtypenum]
+        return get_dtype_cache(space).dtypes_by_num[dtypenum]
 
 
 @jit.unroll_safe
@@ -960,21 +985,21 @@
     if promote_to_largest:
         if dt.kind == NPY.GENBOOLLTR or dt.kind == NPY.SIGNEDLTR:
             if dt.elsize * 8 < LONG_BIT:
-                return descriptor.get_dtype_cache(space).w_longdtype
+                return get_dtype_cache(space).w_longdtype
         elif dt.kind == NPY.UNSIGNEDLTR:
             if dt.elsize * 8 < LONG_BIT:
-                return descriptor.get_dtype_cache(space).w_ulongdtype
+                return get_dtype_cache(space).w_ulongdtype
         else:
             assert dt.kind == NPY.FLOATINGLTR or dt.kind == NPY.COMPLEXLTR
         return dt
     if promote_bools and (dt.kind == NPY.GENBOOLLTR):
-        return descriptor.get_dtype_cache(space).w_int8dtype
+        return get_dtype_cache(space).w_int8dtype
     if promote_to_float:
         if dt.kind == NPY.FLOATINGLTR or dt.kind == NPY.COMPLEXLTR:
             return dt
         if dt.num >= NPY.INT:
-            return descriptor.get_dtype_cache(space).w_float64dtype
-        for bytes, dtype in descriptor.get_dtype_cache(space).float_dtypes_by_num_bytes:
+            return get_dtype_cache(space).w_float64dtype
+        for bytes, dtype in get_dtype_cache(space).float_dtypes_by_num_bytes:
             if (dtype.kind == NPY.FLOATINGLTR and
                     dtype.itemtype.get_element_size() >
                     dt.itemtype.get_element_size()):
@@ -983,12 +1008,12 @@
 
 
 def find_dtype_for_scalar(space, w_obj, current_guess=None):
-    bool_dtype = descriptor.get_dtype_cache(space).w_booldtype
-    long_dtype = descriptor.get_dtype_cache(space).w_longdtype
-    int64_dtype = descriptor.get_dtype_cache(space).w_int64dtype
-    uint64_dtype = descriptor.get_dtype_cache(space).w_uint64dtype
-    complex_dtype = descriptor.get_dtype_cache(space).w_complex128dtype
-    float_dtype = descriptor.get_dtype_cache(space).w_float64dtype
+    bool_dtype = get_dtype_cache(space).w_booldtype
+    long_dtype = get_dtype_cache(space).w_longdtype
+    int64_dtype = get_dtype_cache(space).w_int64dtype
+    uint64_dtype = get_dtype_cache(space).w_uint64dtype
+    complex_dtype = get_dtype_cache(space).w_complex128dtype
+    float_dtype = get_dtype_cache(space).w_float64dtype
     if isinstance(w_obj, boxes.W_GenericBox):
         dtype = w_obj.get_dtype(space)
         return find_binop_result_dtype(space, dtype, current_guess)
@@ -1012,11 +1037,11 @@
         return complex_dtype
     elif space.isinstance_w(w_obj, space.w_str):
         if current_guess is None:
-            return descriptor.variable_dtype(space,
+            return variable_dtype(space,
                                                'S%d' % space.len_w(w_obj))
         elif current_guess.num == NPY.STRING:
             if current_guess.elsize < space.len_w(w_obj):
-                return descriptor.variable_dtype(space,
+                return variable_dtype(space,
                                                    'S%d' % space.len_w(w_obj))
         return current_guess
     raise oefmt(space.w_NotImplementedError,
@@ -1033,7 +1058,7 @@
             raise oefmt(space.w_NotImplementedError,
                         "%s not implemented for %s",
                         ufunc_name, dtype.get_name())
-    dtype_cache = descriptor.get_dtype_cache(space)
+    dtype_cache = get_dtype_cache(space)
     if nin == 1:
         def impl(res_dtype, value):
             res = get_op(res_dtype)(value)
@@ -1162,7 +1187,7 @@
         identity = extra_kwargs.get("identity")
         if identity is not None:
             identity = \
-                descriptor.get_dtype_cache(space).w_longdtype.box(identity)
+                get_dtype_cache(space).w_longdtype.box(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, nin,
@@ -1264,7 +1289,7 @@
             else:
                 dtypes = [None]*len(_dtypes)
                 for i in range(len(dtypes)):
-                    dtypes[i] = descriptor.decode_w_dtype(space, _dtypes[i])
+                    dtypes[i] = decode_w_dtype(space, _dtypes[i])
     else:
         raise oefmt(space.w_ValueError,
             'dtypes must be None or a list of dtypes')
@@ -1273,7 +1298,7 @@
         identity =  None
     elif space.isinstance_w(w_identity, space.w_int):
         identity = \
-            descriptor.get_dtype_cache(space).w_longdtype.box(space.int_w(w_identity))
+            get_dtype_cache(space).w_longdtype.box(space.int_w(w_identity))
     else:
         raise oefmt(space.w_ValueError,
             'identity must be None or an int')

From noreply at buildbot.pypy.org  Tue Dec  2 14:31:03 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Dec 2014 14:31:03 +0100 (CET)
Subject: [pypy-commit] pypy default: implement strides argument when
 buffer=something for ndarray constructor, 
Message-ID: <20141202133103.EB8471D2A97@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74782:86fcd1d9f5df
Date: 2014-12-02 15:29 +0200
http://bitbucket.org/pypy/pypy/changeset/86fcd1d9f5df/

Log:	implement strides argument when buffer=something for ndarray
	constructor, also for from_shape_and_storage

diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -44,11 +44,16 @@
         return W_NDimArray(impl)
 
     @staticmethod
-    def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False,
-                               w_subtype=None, w_base=None, writable=True):
+    def from_shape_and_storage(space, shape, storage, dtype, order='C',
+                               owning=False, w_subtype=None, w_base=None,
+                               writable=True, strides=None):
         from pypy.module.micronumpy import concrete
-        from pypy.module.micronumpy.strides import calc_strides
-        strides, backstrides = calc_strides(shape, dtype, order)
+        from pypy.module.micronumpy.strides import (calc_strides,
+                                                    calc_backstrides)
+        if strides is None:
+            strides, backstrides = calc_strides(shape, dtype, order)
+        else:
+            backstrides = calc_backstrides(strides, shape)
         if w_base is not None:
             if owning:
                 raise OperationError(space.w_ValueError,
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -11,7 +11,7 @@
 from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
     RecordChunk, calc_strides, calc_new_strides, shape_agreement,
-    calculate_broadcast_strides)
+    calculate_broadcast_strides, calc_backstrides)
 
 
 class BaseConcreteArray(object):
@@ -79,10 +79,7 @@
                                                self.get_strides(), self.order)
         if new_strides is not None:
             # We can create a view, strides somehow match up.
-            ndims = len(new_shape)
-            new_backstrides = [0] * ndims
-            for nd in range(ndims):
-                new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd]
+            new_backstrides = calc_backstrides(new_strides, new_shape)
             assert isinstance(orig_array, W_NDimArray) or orig_array is None
             return SliceArray(self.start, new_strides, new_backstrides,
                               new_shape, self, orig_array)
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -1213,8 +1213,10 @@
 
     if not space.is_none(w_buffer):
         if (not space.is_none(w_strides)):
-            raise OperationError(space.w_NotImplementedError,
-                                 space.wrap("unsupported param"))
+            strides = [space.int_w(w_i) for w_i in
+                       space.unpackiterable(w_strides)]
+        else:
+            strides = None
 
         try:
             buf = space.writebuf_w(w_buffer)
@@ -1237,7 +1239,8 @@
         return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
                                                   w_subtype=w_subtype,
                                                   w_base=w_buffer,
-                                                  writable=not buf.readonly)
+                                                  writable=not buf.readonly,
+                                                  strides=strides)
 
     order = order_converter(space, w_order, NPY.CORDER)
     if order == NPY.CORDER:
@@ -1256,7 +1259,7 @@
 
 
 @unwrap_spec(addr=int)
-def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype, w_subtype=None):
+def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype, w_subtype=None, w_strides=None):
     """
     Create an array from an existing buffer, given its address as int.
     PyPy-only implementation detail.
@@ -1265,14 +1268,21 @@
     dtype = space.interp_w(descriptor.W_Dtype, space.call_function(
         space.gettypefor(descriptor.W_Dtype), w_dtype))
     shape = shape_converter(space, w_shape, dtype)
+    if not space.is_none(w_strides):
+        strides = [space.int_w(w_i) for w_i in
+                   space.unpackiterable(w_strides)]
+    else:
+        strides = None
     if w_subtype:
         if not space.isinstance_w(w_subtype, space.w_type):
             raise OperationError(space.w_ValueError, space.wrap(
                 "subtype must be a subtype of ndarray, not a class instance"))
         return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
-                                                  'C', False, w_subtype)
+                                                  'C', False, w_subtype,
+                                                  strides=strides)
     else:
-        return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype)
+        return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
+                                                  strides=strides)
 
 app_take = applevel(r"""
     def take(a, indices, axis, out, mode):
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -362,6 +362,13 @@
         backstrides.reverse()
     return strides, backstrides
 
+ at jit.unroll_safe
+def calc_backstrides(strides, shape):
+    ndims = len(shape)
+    new_backstrides = [0] * ndims
+    for nd in range(ndims):
+        new_backstrides[nd] = (shape[nd] - 1) * strides[nd]
+    return new_backstrides
 
 # Recalculating strides. Find the steps that the iteration does for each
 # dimension, given the stride and shape. Then try to create a new stride that
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3926,3 +3926,27 @@
         assert x.__pypy_data__ is obj
         del x.__pypy_data__
         assert x.__pypy_data__ is None
+
+    def test_ndarray_buffer_strides(self):
+        from numpypy import ndarray, array
+        base = array([1, 2, 3, 4], dtype=int)
+        a = ndarray((4,), buffer=base, dtype=int)
+        assert a[1] == 2
+        a = ndarray((4,), buffer=base, dtype=int, strides=[base.strides[0]])
+        assert a[1] == 2
+        a = ndarray((4,), buffer=base, dtype=int, strides=[2 * base.strides[0]])
+        assert a[1] == 3
+
+    def test_from_shape_and_storage_strides(self):
+        from numpypy import ndarray, array
+        base = array([1, 2, 3, 4], dtype=int)
+        addr, _ = base.__array_interface__['data']
+        a = ndarray._from_shape_and_storage((4,), addr, int)
+        assert a[1] == 2
+        a = ndarray._from_shape_and_storage((4,), addr, int,
+                                           strides=[base.strides[0]])
+        assert a[1] == 2
+        a = ndarray._from_shape_and_storage((4,), addr, int,
+                                           strides=[2 * base.strides[0]])
+        assert a[1] == 3
+        

From noreply at buildbot.pypy.org  Tue Dec  2 17:54:55 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Tue,  2 Dec 2014 17:54:55 +0100 (CET)
Subject: [pypy-commit] pypy default: add failing test for int_signext case
Message-ID: <20141202165455.C62F91D2A97@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74783:a48527cf9df5
Date: 2014-12-02 11:54 -0500
http://bitbucket.org/pypy/pypy/changeset/a48527cf9df5/

Log:	add failing test for int_signext case

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3068,8 +3068,12 @@
     def test_int_signext(self):
         def f(n):
             return rffi.cast(rffi.SIGNEDCHAR, n)
+        def f1(n):
+            return rffi.cast(rffi.SIGNEDCHAR, n + 1)
         res = self.interp_operations(f, [128])
         assert res == -128
+        res = self.interp_operations(f1, [127])
+        assert res == -128
         res = self.interp_operations(f, [-35 + 256 * 29])
         assert res == -35
         res = self.interp_operations(f, [127 - 256 * 29])

From noreply at buildbot.pypy.org  Tue Dec  2 18:15:09 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  2 Dec 2014 18:15:09 +0100 (CET)
Subject: [pypy-commit] pypy default: move test so it runs (fails) with -A
Message-ID: <20141202171509.AAC521C142A@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r74784:7b945fce0e06
Date: 2014-12-02 15:59 +0200
http://bitbucket.org/pypy/pypy/changeset/7b945fce0e06/

Log:	move test so it runs (fails) with -A

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3416,6 +3416,17 @@
         assert str(array(1.5)) == '1.5'
         assert str(array(1.5).real) == '1.5'
 
+    def test_ndarray_buffer_strides(self):
+        from numpy import ndarray, array
+        base = array([1, 2, 3, 4], dtype=int)
+        a = ndarray((4,), buffer=base, dtype=int)
+        assert a[1] == 2
+        a = ndarray((4,), buffer=base, dtype=int, strides=[base.strides[0]])
+        assert a[1] == 2
+        a = ndarray((4,), buffer=base, dtype=int, strides=[2 * base.strides[0]])
+        assert a[1] == 3
+
+
 
 class AppTestRepr(BaseNumpyAppTest):
     def setup_class(cls):
@@ -3883,6 +3894,7 @@
         assert np.greater(a, a) is NotImplemented
         assert np.less_equal(a, a) is NotImplemented
 
+
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
@@ -3927,18 +3939,8 @@
         del x.__pypy_data__
         assert x.__pypy_data__ is None
 
-    def test_ndarray_buffer_strides(self):
-        from numpypy import ndarray, array
-        base = array([1, 2, 3, 4], dtype=int)
-        a = ndarray((4,), buffer=base, dtype=int)
-        assert a[1] == 2
-        a = ndarray((4,), buffer=base, dtype=int, strides=[base.strides[0]])
-        assert a[1] == 2
-        a = ndarray((4,), buffer=base, dtype=int, strides=[2 * base.strides[0]])
-        assert a[1] == 3
-
     def test_from_shape_and_storage_strides(self):
-        from numpypy import ndarray, array
+        from numpy import ndarray, array
         base = array([1, 2, 3, 4], dtype=int)
         addr, _ = base.__array_interface__['data']
         a = ndarray._from_shape_and_storage((4,), addr, int)
@@ -3949,4 +3951,3 @@
         a = ndarray._from_shape_and_storage((4,), addr, int,
                                            strides=[2 * base.strides[0]])
         assert a[1] == 3
-        

From noreply at buildbot.pypy.org  Tue Dec  2 18:15:11 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  2 Dec 2014 18:15:11 +0100 (CET)
Subject: [pypy-commit] pypy default: check strides,
 shape agains buf_len when creating array with a buffer
Message-ID: <20141202171511.195F81C142A@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r74785:46e91c62f6b8
Date: 2014-12-02 19:13 +0200
http://bitbucket.org/pypy/pypy/changeset/46e91c62f6b8/

Log:	check strides, shape agains buf_len when creating array with a
	buffer

diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from rpython.tool.pairtype import extendabletype
-
+from pypy.module.micronumpy import support
 
 def wrap_impl(space, w_cls, w_instance, impl):
     if w_cls is None or space.is_w(w_cls, space.gettypefor(W_NDimArray)):
@@ -44,15 +44,31 @@
         return W_NDimArray(impl)
 
     @staticmethod
-    def from_shape_and_storage(space, shape, storage, dtype, order='C',
-                               owning=False, w_subtype=None, w_base=None,
-                               writable=True, strides=None):
+    def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1,
+                               order='C', owning=False, w_subtype=None,
+                               w_base=None, writable=True, strides=None):
         from pypy.module.micronumpy import concrete
         from pypy.module.micronumpy.strides import (calc_strides,
                                                     calc_backstrides)
+        isize = dtype.elsize
+        if storage_bytes > 0 :
+            totalsize = support.product(shape) * isize
+            if totalsize > storage_bytes:
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "buffer is too small for requested array"))
+        else:
+            storage_bytes = support.product(shape) * isize
         if strides is None:
             strides, backstrides = calc_strides(shape, dtype, order)
         else:
+            if len(strides) != len(shape):
+                raise oefmt(space.w_ValueError,
+                    'strides, if given, must be the same length as shape')
+            for i in range(len(strides)):
+                if strides[i] < 0 or strides[i]*shape[i] > storage_bytes:
+                    raise oefmt(space.w_ValueError,
+                        'strides is incompatible with shape of requested '
+                        'array and size of buffer')
             backstrides = calc_backstrides(strides, shape)
         if w_base is not None:
             if owning:
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -302,5 +302,5 @@
         return a
     else:
         writable = not buf.readonly
-    return W_NDimArray.from_shape_and_storage(space, [n], storage, dtype=dtype,
-                                              w_base=w_buffer, writable=writable)
+    return W_NDimArray.from_shape_and_storage(space, [n], storage, storage_bytes=s, 
+                                dtype=dtype, w_base=w_buffer, writable=writable)
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -529,9 +529,10 @@
                 "__array__(dtype) not implemented"))
         if type(self) is W_NDimArray:
             return self
+        sz = support.product(self.get_shape()) * self.get_dtype().elsize
         return W_NDimArray.from_shape_and_storage(
             space, self.get_shape(), self.implementation.storage,
-            self.get_dtype(), w_base=self)
+            self.get_dtype(), storage_bytes=sz, w_base=self)
 
     def descr_array_iface(self, space):
         addr = self.implementation.get_storage_as_int(space)
@@ -1188,8 +1189,8 @@
                         "improper dtype '%R'", dtype)
         self.implementation = W_NDimArray.from_shape_and_storage(
             space, [space.int_w(i) for i in space.listview(shape)],
-            rffi.str2charp(space.str_w(storage), track_allocation=False),
-            dtype, owning=True).implementation
+            rffi.str2charp(space.str_w(storage), track_allocation=False), 
+            dtype, storage_bytes=space.len_w(storage), owning=True).implementation
 
     def descr___array_finalize__(self, space, w_obj):
         pass
@@ -1230,15 +1231,12 @@
         if not shape:
             raise OperationError(space.w_TypeError, space.wrap(
                 "numpy scalars from buffers not supported yet"))
-        totalsize = support.product(shape) * dtype.elsize
-        if totalsize + offset > buf.getlength():
-            raise OperationError(space.w_TypeError, space.wrap(
-                "buffer is too small for requested array"))
         storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr)
         storage = rffi.ptradd(storage, offset)
-        return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
+        return W_NDimArray.from_shape_and_storage(space, shape, storage,
+                                                  dtype, w_base=w_buffer,
+                                                  storage_bytes=buf.getlength()-offset,
                                                   w_subtype=w_subtype,
-                                                  w_base=w_buffer,
                                                   writable=not buf.readonly,
                                                   strides=strides)
 
@@ -1258,9 +1256,12 @@
     return w_ret
 
 
- at unwrap_spec(addr=int)
-def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype, w_subtype=None, w_strides=None):
+ at unwrap_spec(addr=int, buf_len=int)
+def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype,
+                buf_len=-1, w_subtype=None, w_strides=None):
     """
+    _from_shape_and_storage(shape, addr, dtype, buf_len, 
+                            subtype=None, strides=None)
     Create an array from an existing buffer, given its address as int.
     PyPy-only implementation detail.
     """
@@ -1278,10 +1279,11 @@
             raise OperationError(space.w_ValueError, space.wrap(
                 "subtype must be a subtype of ndarray, not a class instance"))
         return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
-                                                  'C', False, w_subtype,
+                                                  buf_len, 'C', False, w_subtype,
                                                   strides=strides)
     else:
         return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
+                                                  storage_bytes=buf_len,
                                                   strides=strides)
 
 app_take = applevel(r"""
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -186,7 +186,8 @@
         #
         dtypes = get_dtype_cache(self.space)
         w_array = W_NDimArray.from_shape_and_storage(self.space, [2, 2],
-                                                storage, dtypes.w_int8dtype)
+                                                storage, dtypes.w_int8dtype,
+                                                storage_bytes=4)
         def get(i, j):
             return w_array.getitem(self.space, [i, j]).value
         assert get(0, 0) == 0
@@ -3423,8 +3424,12 @@
         assert a[1] == 2
         a = ndarray((4,), buffer=base, dtype=int, strides=[base.strides[0]])
         assert a[1] == 2
-        a = ndarray((4,), buffer=base, dtype=int, strides=[2 * base.strides[0]])
+        a = ndarray((2,), buffer=base, dtype=int, strides=[2 * base.strides[0]])
         assert a[1] == 3
+        exc = raises(ValueError, ndarray, (4,), buffer=base, dtype=int, strides=[2 * base.strides[0]])
+        assert exc.value[0] == 'strides is incompatible with shape of requested array and size of buffer'
+        exc = raises(ValueError, ndarray, (2, 1), buffer=base, dtype=int, strides=[base.strides[0]])
+        assert exc.value[0] == 'strides, if given, must be the same length as shape'
 
 
 
@@ -3918,13 +3923,14 @@
         from numpypy import array, ndarray
         x = array([1, 2, 3, 4])
         addr, _ = x.__array_interface__['data']
-        y = ndarray._from_shape_and_storage([2, 2], addr, x.dtype)
+        sz = x.size * x.dtype.itemsize
+        y = ndarray._from_shape_and_storage([2, 2], addr, x.dtype, sz)
         assert y[0, 1] == 2
         y[0, 1] = 42
         assert x[1] == 42
         class C(ndarray):
             pass
-        z = ndarray._from_shape_and_storage([4, 1], addr, x.dtype, C)
+        z = ndarray._from_shape_and_storage([4, 1], addr, x.dtype, sz, C)
         assert isinstance(z, C)
         assert z.shape == (4, 1)
         assert z[1, 0] == 42
@@ -3943,11 +3949,12 @@
         from numpy import ndarray, array
         base = array([1, 2, 3, 4], dtype=int)
         addr, _ = base.__array_interface__['data']
-        a = ndarray._from_shape_and_storage((4,), addr, int)
+        sz = base.size * base.dtype.itemsize
+        a = ndarray._from_shape_and_storage((4,), addr, int, sz)
         assert a[1] == 2
-        a = ndarray._from_shape_and_storage((4,), addr, int,
+        a = ndarray._from_shape_and_storage((4,), addr, int, sz,
                                            strides=[base.strides[0]])
         assert a[1] == 2
-        a = ndarray._from_shape_and_storage((4,), addr, int,
+        a = ndarray._from_shape_and_storage((2,), addr, int, sz,
                                            strides=[2 * base.strides[0]])
         assert a[1] == 3

From noreply at buildbot.pypy.org  Tue Dec  2 18:32:24 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  2 Dec 2014 18:32:24 +0100 (CET)
Subject: [pypy-commit] pypy default: overzelous docstring removed
Message-ID: <20141202173224.4AF6F1D28F0@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r74786:8471ffc38d46
Date: 2014-12-02 19:32 +0200
http://bitbucket.org/pypy/pypy/changeset/8471ffc38d46/

Log:	overzelous docstring removed

diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -1260,8 +1260,6 @@
 def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype,
                 buf_len=-1, w_subtype=None, w_strides=None):
     """
-    _from_shape_and_storage(shape, addr, dtype, buf_len, 
-                            subtype=None, strides=None)
     Create an array from an existing buffer, given its address as int.
     PyPy-only implementation detail.
     """

From noreply at buildbot.pypy.org  Tue Dec  2 19:22:42 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Dec 2014 19:22:42 +0100 (CET)
Subject: [pypy-commit] pypy optresult: finish porting test_optimizebasic
Message-ID: <20141202182242.B87CB1D29E9@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74787:e2c252e3b7eb
Date: 2014-12-02 20:22 +0200
http://bitbucket.org/pypy/pypy/changeset/e2c252e3b7eb/

Log:	finish porting test_optimizebasic

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -151,8 +151,8 @@
                 # nonneg % power-of-two ==> nonneg & (power-of-two - 1)
                 arg1 = op.getarg(0)
                 arg2 = ConstInt(val-1)
-                xxx
-                op = op.copy_and_change(rop.INT_AND, args=[arg1, arg2])
+                op = self.replace_op_with(op, rop.INT_AND,
+                                          args=[arg1, arg2])
         self.emit_operation(op)
         if v2.is_constant():
             val = v2.box.getint()
@@ -268,8 +268,7 @@
         v2 = self.getvalue(op.getarg(1))
         resbound = v1.intbound.mul_bound(v2.intbound)
         if resbound.bounded():
-            xxx
-            op = op.copy_and_change(rop.INT_MUL)
+            op = self.replace_op_with(op, rop.INT_MUL)
         self.emit_operation(op)
         r = self.getvalue(op)
         r.intbound.intersect(resbound)
@@ -482,9 +481,9 @@
                 v1 = self.getvalue(op.getarg(0))
                 v2 = self.getvalue(op.getarg(1))
                 if v1.intbound.intersect(v2.intbound):
-                    self.propagate_bounds_backward(op.getarg(0))
+                    self.propagate_bounds_backward(op.getarg(0), v1)
                 if v2.intbound.intersect(v1.intbound):
-                    self.propagate_bounds_backward(op.getarg(1))
+                    self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_IS_TRUE(self, op):
         r = self.getvalue(op)
@@ -535,10 +534,10 @@
         r = self.getvalue(op)
         b = r.intbound.div_bound(v2.intbound)
         if v1.intbound.intersect(b):
-            self.propagate_bounds_backward(op.getarg(0))
+            self.propagate_bounds_backward(op.getarg(0), v1)
         b = r.intbound.div_bound(v1.intbound)
         if v2.intbound.intersect(b):
-            self.propagate_bounds_backward(op.getarg(1))
+            self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_LSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -293,8 +293,8 @@
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
 
-    def replace_op_with(self, op, newopnum, args=None):
-        return self.optimizer.replace_op_with(op, newopnum, args)
+    def replace_op_with(self, op, newopnum, args=None, descr=None):
+        return self.optimizer.replace_op_with(op, newopnum, args, descr)
 
     def make_constant(self, box, constbox):
         return self.optimizer.make_constant(box, constbox)
@@ -466,6 +466,8 @@
     def get_constant_box(self, box):
         if isinstance(box, Const):
             return box
+        while box.source_op is not None:
+            box = box.source_op
         try:
             value = self.values[box]
             self.ensure_imported(value)
@@ -494,15 +496,17 @@
 
     def make_constant(self, box, constbox):
         try:
+            while box.source_op is not None:
+                box = box.source_op
             value = self.values[box]
             value.level = LEVEL_CONSTANT
             value.make_constant(constbox)
         except KeyError:
             self.values[box] = ConstantValue(constbox)
 
-    def replace_op_with(self, op, newopnum, args=None):
+    def replace_op_with(self, op, newopnum, args=None, descr=None):
         v = self.getvalue(op)
-        newop = op.copy_and_change(newopnum, args=args)
+        newop = op.copy_and_change(newopnum, args=args, descr=descr)
         v.box = newop
         return newop
 
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -114,7 +114,7 @@
         if v2.is_constant() and v2.box.getint() == 0:
             self.make_equal_to(op, v1)
         elif v1.is_constant() and v1.box.getint() == 0:
-            op = op.copy_and_change(rop.INT_NEG, args=[v2.box])
+            op = self.replace_op_with(op, rop.INT_NEG, args=[v2.box])
             self.emit_operation(op)
         elif v1 is v2:
             self.make_constant_int(op, 0)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4131,7 +4131,7 @@
     def test_str_concat_1(self):
         ops = """
         [p1, p2]
-        p3 = call(0, p1, p2, descr=strconcatdescr)
+        p3 = call_r(0, p1, p2, descr=strconcatdescr)
         jump(p2, p3)
         """
         expected = """
@@ -4152,7 +4152,7 @@
         p1 = newstr(2)
         strsetitem(p1, 0, i0)
         strsetitem(p1, 1, i1)
-        p3 = call(0, p1, p2, descr=strconcatdescr)
+        p3 = call_r(0, p1, p2, descr=strconcatdescr)
         jump(i1, i0, p3)
         """
         expected = """
@@ -4173,7 +4173,7 @@
         p1 = newstr(2)
         strsetitem(p1, 0, i0)
         strsetitem(p1, 1, i1)
-        p3 = call(0, p2, p1, descr=strconcatdescr)
+        p3 = call_r(0, p2, p1, descr=strconcatdescr)
         jump(i1, i0, p3)
         """
         expected = """
@@ -4193,8 +4193,8 @@
     def test_str_concat_str_str_str(self):
         ops = """
         [p1, p2, p3]
-        p4 = call(0, p1, p2, descr=strconcatdescr)
-        p5 = call(0, p4, p3, descr=strconcatdescr)
+        p4 = call_r(0, p1, p2, descr=strconcatdescr)
+        p5 = call_r(0, p4, p3, descr=strconcatdescr)
         jump(p2, p3, p5)
         """
         expected = """
@@ -4215,7 +4215,7 @@
     def test_str_concat_str_cstr1(self):
         ops = """
         [p2]
-        p3 = call(0, p2, s"x", descr=strconcatdescr)
+        p3 = call_r(0, p2, s"x", descr=strconcatdescr)
         jump(p3)
         """
         expected = """
@@ -4232,9 +4232,9 @@
     def test_str_concat_consts(self):
         ops = """
         []
-        p1 = same_as(s"ab")
-        p2 = same_as(s"cde")
-        p3 = call(0, p1, p2, descr=strconcatdescr)
+        p1 = same_as_r(s"ab")
+        p2 = same_as_r(s"cde")
+        p3 = call_r(0, p1, p2, descr=strconcatdescr)
         escape(p3)
         jump()
         """
@@ -4251,8 +4251,8 @@
         p0 = newstr(1)
         strsetitem(p0, 0, i0)
         p1 = newstr(0)
-        p2 = call(0, p0, p1, descr=strconcatdescr)
-        i1 = call(0, p2, p0, descr=strequaldescr)
+        p2 = call_r(0, p0, p1, descr=strconcatdescr)
+        i1 = call_i(0, p2, p0, descr=strequaldescr)
         finish(i1)
         """
         expected = """
@@ -4267,8 +4267,8 @@
         p0 = newstr(0)
         p1 = newstr(1)
         strsetitem(p1, 0, i0)
-        p2 = call(0, p0, p1, descr=strconcatdescr)
-        i1 = call(0, p2, p1, descr=strequaldescr)
+        p2 = call_r(0, p0, p1, descr=strconcatdescr)
+        i1 = call_i(0, p2, p1, descr=strequaldescr)
         finish(i1)
         """
         expected = """
@@ -4280,7 +4280,7 @@
     def test_str_slice_1(self):
         ops = """
         [p1, i1, i2]
-        p2 = call(0, p1, i1, i2, descr=strslicedescr)
+        p2 = call_r(0, p1, i1, i2, descr=strslicedescr)
         jump(p2, i1, i2)
         """
         expected = """
@@ -4295,7 +4295,7 @@
     def test_str_slice_2(self):
         ops = """
         [p1, i2]
-        p2 = call(0, p1, 0, i2, descr=strslicedescr)
+        p2 = call_r(0, p1, 0, i2, descr=strslicedescr)
         jump(p2, i2)
         """
         expected = """
@@ -4309,8 +4309,8 @@
     def test_str_slice_3(self):
         ops = """
         [p1, i1, i2, i3, i4]
-        p2 = call(0, p1, i1, i2, descr=strslicedescr)
-        p3 = call(0, p2, i3, i4, descr=strslicedescr)
+        p2 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        p3 = call_r(0, p2, i3, i4, descr=strslicedescr)
         jump(p3, i1, i2, i3, i4)
         """
         expected = """
@@ -4327,7 +4327,7 @@
     def test_str_slice_getitem1(self):
         ops = """
         [p1, i1, i2, i3]
-        p2 = call(0, p1, i1, i2, descr=strslicedescr)
+        p2 = call_r(0, p1, i1, i2, descr=strslicedescr)
         i4 = strgetitem(p2, i3)
         escape(i4)
         jump(p1, i1, i2, i3)
@@ -4348,7 +4348,7 @@
         p1 = newstr(2)
         strsetitem(p1, 0, i3)
         strsetitem(p1, 1, i4)
-        p2 = call(0, p1, 1, 2, descr=strslicedescr)
+        p2 = call_r(0, p1, 1, 2, descr=strslicedescr)
         i5 = strgetitem(p2, 0)
         escape(i5)
         jump(i3, i4)
@@ -4363,8 +4363,8 @@
     def test_str_slice_concat(self):
         ops = """
         [p1, i1, i2, p2]
-        p3 = call(0, p1, i1, i2, descr=strslicedescr)
-        p4 = call(0, p3, p2, descr=strconcatdescr)
+        p3 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        p4 = call_r(0, p3, p2, descr=strconcatdescr)
         jump(p4, i1, i2, p2)
         """
         expected = """
@@ -4384,7 +4384,7 @@
         []
         p0 = newstr(11)
         copystrcontent(s"hello world", p0, 0, 0, 11)
-        p1 = call(0, p0, 0, 5, descr=strslicedescr)
+        p1 = call_r(0, p0, 0, 5, descr=strslicedescr)
         finish(p1)
         """
         expected = """
@@ -4415,7 +4415,7 @@
     def test_str_equal_noop1(self):
         ops = """
         [p1, p2]
-        i0 = call(0, p1, p2, descr=strequaldescr)
+        i0 = call_i(0, p1, p2, descr=strequaldescr)
         escape(i0)
         jump(p1, p2)
         """
@@ -4424,8 +4424,8 @@
     def test_str_equal_noop2(self):
         ops = """
         [p1, p2, p3]
-        p4 = call(0, p1, p2, descr=strconcatdescr)
-        i0 = call(0, p3, p4, descr=strequaldescr)
+        p4 = call_r(0, p1, p2, descr=strconcatdescr)
+        i0 = call_i(0, p3, p4, descr=strequaldescr)
         escape(i0)
         jump(p1, p2, p3)
         """
@@ -4437,7 +4437,7 @@
         p4 = newstr(i3)
         copystrcontent(p1, p4, 0, 0, i1)
         copystrcontent(p2, p4, 0, i1, i2)
-        i0 = call(0, p3, p4, descr=strequaldescr)
+        i0 = call_i(0, p3, p4, descr=strequaldescr)
         escape(i0)
         jump(p1, p2, p3)
         """
@@ -4446,15 +4446,15 @@
     def test_str_equal_slice1(self):
         ops = """
         [p1, i1, i2, p3]
-        p4 = call(0, p1, i1, i2, descr=strslicedescr)
-        i0 = call(0, p4, p3, descr=strequaldescr)
+        p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        i0 = call_i(0, p4, p3, descr=strequaldescr)
         escape(i0)
         jump(p1, i1, i2, p3)
         """
         expected = """
         [p1, i1, i2, p3]
         i3 = int_sub(i2, i1)
-        i0 = call(0, p1, i1, i3, p3, descr=streq_slice_checknull_descr)
+        i0 = call_i(0, p1, i1, i3, p3, descr=streq_slice_checknull_descr)
         escape(i0)
         jump(p1, i1, i2, p3)
         """
@@ -4463,15 +4463,15 @@
     def test_str_equal_slice2(self):
         ops = """
         [p1, i1, i2, p3]
-        p4 = call(0, p1, i1, i2, descr=strslicedescr)
-        i0 = call(0, p3, p4, descr=strequaldescr)
+        p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        i0 = call_i(0, p3, p4, descr=strequaldescr)
         escape(i0)
         jump(p1, i1, i2, p3)
         """
         expected = """
         [p1, i1, i2, p3]
         i4 = int_sub(i2, i1)
-        i0 = call(0, p1, i1, i4, p3, descr=streq_slice_checknull_descr)
+        i0 = call_i(0, p1, i1, i4, p3, descr=streq_slice_checknull_descr)
         escape(i0)
         jump(p1, i1, i2, p3)
         """
@@ -4481,8 +4481,8 @@
         ops = """
         [p1, i1, i2, p3]
         guard_nonnull(p3) []
-        p4 = call(0, p1, i1, i2, descr=strslicedescr)
-        i0 = call(0, p3, p4, descr=strequaldescr)
+        p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        i0 = call_i(0, p3, p4, descr=strequaldescr)
         escape(i0)
         jump(p1, i1, i2, p3)
         """
@@ -4490,7 +4490,7 @@
         [p1, i1, i2, p3]
         guard_nonnull(p3) []
         i4 = int_sub(i2, i1)
-        i0 = call(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
+        i0 = call_i(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
         escape(i0)
         jump(p1, i1, i2, p3)
         """
@@ -4499,15 +4499,15 @@
     def test_str_equal_slice4(self):
         ops = """
         [p1, i1, i2]
-        p3 = call(0, p1, i1, i2, descr=strslicedescr)
-        i0 = call(0, p3, s"x", descr=strequaldescr)
+        p3 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        i0 = call_i(0, p3, s"x", descr=strequaldescr)
         escape(i0)
         jump(p1, i1, i2)
         """
         expected = """
         [p1, i1, i2]
         i3 = int_sub(i2, i1)
-        i0 = call(0, p1, i1, i3, 120, descr=streq_slice_char_descr)
+        i0 = call_i(0, p1, i1, i3, 120, descr=streq_slice_char_descr)
         escape(i0)
         jump(p1, i1, i2)
         """
@@ -4516,17 +4516,17 @@
     def test_str_equal_slice5(self):
         ops = """
         [p1, i1, i2, i3]
-        p4 = call(0, p1, i1, i2, descr=strslicedescr)
+        p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
         p5 = newstr(1)
         strsetitem(p5, 0, i3)
-        i0 = call(0, p5, p4, descr=strequaldescr)
+        i0 = call_i(0, p5, p4, descr=strequaldescr)
         escape(i0)
         jump(p1, i1, i2, i3)
         """
         expected = """
         [p1, i1, i2, i3]
         i4 = int_sub(i2, i1)
-        i0 = call(0, p1, i1, i4, i3, descr=streq_slice_char_descr)
+        i0 = call_i(0, p1, i1, i4, i3, descr=streq_slice_char_descr)
         escape(i0)
         jump(p1, i1, i2, i3)
         """
@@ -4535,7 +4535,7 @@
     def test_str_equal_none1(self):
         ops = """
         [p1]
-        i0 = call(0, p1, NULL, descr=strequaldescr)
+        i0 = call_i(0, p1, NULL, descr=strequaldescr)
         escape(i0)
         jump(p1)
         """
@@ -4550,7 +4550,7 @@
     def test_str_equal_none2(self):
         ops = """
         [p1]
-        i0 = call(0, NULL, p1, descr=strequaldescr)
+        i0 = call_i(0, NULL, p1, descr=strequaldescr)
         escape(i0)
         jump(p1)
         """
@@ -4566,14 +4566,14 @@
         ops = """
         [p1]
         guard_nonnull(p1) []
-        i0 = call(0, p1, s"hello world", descr=strequaldescr)
+        i0 = call_i(0, p1, s"hello world", descr=strequaldescr)
         escape(i0)
         jump(p1)
         """
         expected = """
         [p1]
         guard_nonnull(p1) []
-        i0 = call(0, p1, s"hello world", descr=streq_nonnull_descr)
+        i0 = call_i(0, p1, s"hello world", descr=streq_nonnull_descr)
         escape(i0)
         jump(p1)
         """
@@ -4583,7 +4583,7 @@
         ops = """
         [p1]
         guard_nonnull(p1) []
-        i0 = call(0, p1, s"", descr=strequaldescr)
+        i0 = call_i(0, p1, s"", descr=strequaldescr)
         escape(i0)
         jump(p1)
         """
@@ -4601,14 +4601,14 @@
         ops = """
         [p1]
         guard_nonnull(p1) []
-        i0 = call(0, p1, s"x", descr=strequaldescr)
+        i0 = call_i(0, p1, s"x", descr=strequaldescr)
         escape(i0)
         jump(p1)
         """
         expected = """
         [p1]
         guard_nonnull(p1) []
-        i0 = call(0, p1, 120, descr=streq_nonnull_char_descr)
+        i0 = call_i(0, p1, 120, descr=streq_nonnull_char_descr)
         escape(i0)
         jump(p1)
         """
@@ -4617,8 +4617,8 @@
     def test_str_equal_nonnull4(self):
         ops = """
         [p1, p2]
-        p4 = call(0, p1, p2, descr=strconcatdescr)
-        i0 = call(0, s"hello world", p4, descr=strequaldescr)
+        p4 = call_r(0, p1, p2, descr=strconcatdescr)
+        i0 = call_i(0, s"hello world", p4, descr=strequaldescr)
         escape(i0)
         jump(p1, p2)
         """
@@ -4630,7 +4630,7 @@
         p4 = newstr(i3)
         copystrcontent(p1, p4, 0, 0, i1)
         copystrcontent(p2, p4, 0, i1, i2)
-        i0 = call(0, s"hello world", p4, descr=streq_nonnull_descr)
+        i0 = call_i(0, s"hello world", p4, descr=streq_nonnull_descr)
         escape(i0)
         jump(p1, p2)
         """
@@ -4640,7 +4640,7 @@
         ops = """
         [i1]
         p1 = newstr(0)
-        i0 = call(0, p1, s"", descr=strequaldescr)
+        i0 = call_i(0, p1, s"", descr=strequaldescr)
         escape(i0)
         jump(i1)
         """
@@ -4656,7 +4656,7 @@
         [i1]
         p1 = newstr(1)
         strsetitem(p1, 0, i1)
-        i0 = call(0, p1, s"x", descr=strequaldescr)
+        i0 = call_i(0, p1, s"x", descr=strequaldescr)
         escape(i0)
         jump(i1)
         """
@@ -4674,7 +4674,7 @@
         p1 = newstr(2)
         strsetitem(p1, 0, i1)
         strsetitem(p1, 1, i2)
-        i0 = call(0, p1, s"xy", descr=strequaldescr)
+        i0 = call_i(0, p1, s"xy", descr=strequaldescr)
         escape(i0)
         jump(i1, i2)
         """
@@ -4683,7 +4683,7 @@
         p1 = newstr(2)
         strsetitem(p1, 0, i1)
         strsetitem(p1, 1, i2)
-        i0 = call(0, p1, s"xy", descr=streq_lengthok_descr)
+        i0 = call_i(0, p1, s"xy", descr=streq_lengthok_descr)
         escape(i0)
         jump(i1, i2)
         """
@@ -4692,13 +4692,13 @@
     def test_str_equal_chars3(self):
         ops = """
         [p1]
-        i0 = call(0, s"x", p1, descr=strequaldescr)
+        i0 = call_i(0, s"x", p1, descr=strequaldescr)
         escape(i0)
         jump(p1)
         """
         expected = """
         [p1]
-        i0 = call(0, p1, 120, descr=streq_checknull_char_descr)
+        i0 = call_i(0, p1, 120, descr=streq_checknull_char_descr)
         escape(i0)
         jump(p1)
         """
@@ -4709,7 +4709,7 @@
         [i1]
         p1 = newstr(1)
         strsetitem(p1, 0, i1)
-        i0 = call(0, s"xy", p1, descr=strequaldescr)
+        i0 = call_i(0, s"xy", p1, descr=strequaldescr)
         escape(i0)
         jump(i1)
         """
@@ -4723,7 +4723,7 @@
     def test_str2unicode_constant(self):
         ops = """
         []
-        p0 = call(0, "xy", descr=s2u_descr)      # string -> unicode
+        p0 = call_r(0, "xy", descr=s2u_descr)      # string -> unicode
         escape(p0)
         jump()
         """
@@ -4737,7 +4737,7 @@
     def test_str2unicode_nonconstant(self):
         ops = """
         [p0]
-        p1 = call(0, p0, descr=s2u_descr)      # string -> unicode
+        p1 = call_r(0, p0, descr=s2u_descr)      # string -> unicode
         escape(p1)
         jump(p1)
         """
@@ -4787,7 +4787,7 @@
         ops = """
         [p0, i0]
         i1 = int_add(i0, 1)
-        p1 = call(0, p0, i0, i1, descr=strslicedescr)
+        p1 = call_r(0, p0, i0, i1, descr=strslicedescr)
         escape(p1)
         jump(p0, i1)
         """
@@ -4956,10 +4956,10 @@
         [p0, i0]
         i1 = int_gt(i0, 2)
         guard_true(i1) []
-        setarrayitem_gc(p0, 0, 3)
-        setarrayitem_gc(p0, 2, 4)
-        setarrayitem_gc(p0, i0, 15)
-        i2 = getarrayitem_gc(p0, 2)
+        setarrayitem_gc(p0, 0, 3, descr=arraydescr)
+        setarrayitem_gc(p0, 2, 4, descr=arraydescr)
+        setarrayitem_gc(p0, i0, 15, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p0, 2, descr=arraydescr)
         jump(p0, i2)
         """
         # Remove the getarrayitem_gc, because we know that p[i0] does not alias
@@ -4968,9 +4968,9 @@
         [p0, i0]
         i1 = int_gt(i0, 2)
         guard_true(i1) []
-        setarrayitem_gc(p0, i0, 15)
-        setarrayitem_gc(p0, 0, 3)
-        setarrayitem_gc(p0, 2, 4)
+        setarrayitem_gc(p0, i0, 15, descr=arraydescr)
+        setarrayitem_gc(p0, 0, 3, descr=arraydescr)
+        setarrayitem_gc(p0, 2, 4, descr=arraydescr)
         jump(p0, 4)
         """
         self.optimize_loop(ops, expected)
@@ -5036,7 +5036,7 @@
         [i1]
         p0 = newstr(6)
         copystrcontent(s"hello!", p0, 0, 0, 6)
-        p1 = call(0, p0, s"abc123", descr=strconcatdescr)
+        p1 = call_r(0, p0, s"abc123", descr=strconcatdescr)
         i0 = strgetitem(p1, i1)
         finish(i0)
         """
@@ -5052,7 +5052,7 @@
         []
         p0 = newstr(6)
         copystrcontent(s"hello!", p0, 0, 0, 6)
-        p1 = call(0, p0, s"abc123", descr=strconcatdescr)
+        p1 = call_r(0, p0, s"abc123", descr=strconcatdescr)
         i0 = strgetitem(p1, 0)
         finish(i0)
         """
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -568,7 +568,9 @@
         if self._last_guard_not_forced_2 is not None:
             guard_op = self._last_guard_not_forced_2
             self.emit_operation(op)
-            guard_op = self.optimizer.store_final_boxes_in_guard(guard_op, [])
+            v = self.getvalue(op)
+            guard_op = self.optimizer.store_final_boxes_in_guard(guard_op, [],
+                                                                 v)
             i = len(self.optimizer._newoperations) - 1
             assert i >= 0
             self.optimizer._newoperations.insert(i, guard_op)
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -5,7 +5,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0, CONST_1
 from rpython.jit.metainterp.optimizeopt.optimizer import llhelper, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, DONT_CHANGE
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper import annlowlevel
@@ -61,11 +61,12 @@
             return None
         self.ensure_nonnull()
         box = self.force_box(string_optimizer)
-        if lengthop is None:
-            xxx
-        else:
+        if lengthop is not None:
             lengthop = string_optimizer.optimizer.replace_op_with(lengthop,
                 mode.STRLEN, [box])
+        else:
+            lengthop = ResOperation(mode.STRLEN, [box])
+            lengthop.is_source_op = True
         string_optimizer.emit_operation(lengthop)
         return lengthop
 
@@ -90,9 +91,8 @@
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, keybox,
-                                                 source_op)
+    def __init__(self, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, source_op)
         self.mode = mode
 
     def _really_force(self, optforce):
@@ -109,13 +109,14 @@
                 self.make_constant(c_s)
                 return
         assert self.source_op is not None
-        self.box = box = self.source_op.result
         lengthbox = self.getstrlen(optforce, self.mode, None)
-        op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
+        op = ResOperation(self.mode.NEWSTR, [lengthbox])
+        op.source_op = self.source_op
+        self.box = op
         if not we_are_translated():
             op.name = 'FORCE'
         optforce.emit_operation(op)
-        self.initialize_forced_string(optforce, box, CONST_0, self.mode)
+        self.initialize_forced_string(optforce, self.box, CONST_0, self.mode)
 
     def initialize_forced_string(self, string_optimizer, targetbox,
                                  offsetbox, mode):
@@ -188,7 +189,7 @@
     def initialize_forced_string(self, string_optimizer, targetbox,
                                  offsetbox, mode):
         for i in range(len(self._chars)):
-            assert isinstance(targetbox, BoxPtr)   # ConstPtr never makes sense
+            assert not isinstance(targetbox, Const) # ConstPtr never makes sense
             charvalue = self.getitem(i)
             if charvalue is not None:
                 charbox = charvalue.force_box(string_optimizer)
@@ -281,7 +282,6 @@
         self.vlength = vlength
 
     def getstrlen(self, optforce, mode, lengthbox):
-        xxx
         return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
@@ -330,7 +330,7 @@
         for i in range(lengthbox.value):
             charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
             srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
-            assert isinstance(targetbox, BoxPtr)   # ConstPtr never makes sense
+            assert not isinstance(targetbox, Const)# ConstPtr never makes sense
             string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
                                                                            offsetbox,
                                                                            charbox],
@@ -341,10 +341,10 @@
             nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
         else:
             nextoffsetbox = None
-        assert isinstance(targetbox, BoxPtr)   # ConstPtr never makes sense
+        assert not isinstance(targetbox, Const)   # ConstPtr never makes sense
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
-                                                lengthbox], None)
+                                                lengthbox])
         string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
@@ -359,9 +359,10 @@
         return box1
     if string_optimizer is None:
         return None
-    resbox = BoxInt()
-    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
-    return resbox
+    op = ResOperation(rop.INT_ADD, [box1, box2])
+    op.is_source_op = True
+    string_optimizer.emit_operation(op)
+    return op
 
 def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
@@ -369,9 +370,10 @@
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
-    resbox = BoxInt()
-    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
-    return resbox
+    op = ResOperation(rop.INT_SUB, [box1, box2])
+    op.is_source_op = True
+    string_optimizer.emit_operation(op)
+    return op
 
 def _strgetitem(string_optimizer, strbox, indexbox, mode, resbox=None):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
@@ -382,28 +384,30 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     if resbox is None:
-        resbox = BoxInt()
-    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                                 resbox))
+        resbox = ResOperation(mode.STRGETITEM, [strbox, indexbox])
+    else:
+        resbox = string_optimizer.replace_op_with(resbox, mode.STRGETITEM,
+                                                  [strbox, indexbox])
+    string_optimizer.emit_operation(resbox)
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
 
-    def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(box, source_op, mode)
-        self.make_equal_to(box, vvalue)
+    def make_vstring_plain(self, source_op, mode):
+        vvalue = VStringPlainValue(source_op, mode)
+        self.make_equal_to(source_op, vvalue)
         return vvalue
 
-    def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(box, source_op, mode)
-        self.make_equal_to(box, vvalue)
+    def make_vstring_concat(self, source_op, mode):
+        vvalue = VStringConcatValue(source_op, mode)
+        self.make_equal_to(source_op, vvalue)
         return vvalue
 
-    def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(box, source_op, mode)
-        self.make_equal_to(box, vvalue)
+    def make_vstring_slice(self, source_op, mode):
+        vvalue = VStringSliceValue(source_op, mode)
+        self.make_equal_to(source_op, vvalue)
         return vvalue
 
     def optimize_NEWSTR(self, op):
@@ -417,13 +421,13 @@
             # if the original 'op' did not have a ConstInt as argument,
             # build a new one with the ConstInt argument
             if not isinstance(op.getarg(0), ConstInt):
-                op = ResOperation(mode.NEWSTR, [length_box], op.result)
-            vvalue = self.make_vstring_plain(op.result, op, mode)
+                op = self.replace_op_with(op, mode.NEWSTR, [length_box])
+            vvalue = self.make_vstring_plain(op, mode)
             vvalue.setup(length_box.getint())
         else:
-            self.getvalue(op.result).ensure_nonnull()
+            self.getvalue(op).ensure_nonnull()
             self.emit_operation(op)
-            self.pure(mode.STRLEN, [op.result], op.getarg(0))
+            self.pure(mode.STRLEN, [op], op.getarg(0))
 
     def optimize_STRSETITEM(self, op):
         value = self.getvalue(op.getarg(0))
@@ -446,11 +450,11 @@
     def _optimize_STRGETITEM(self, op, mode):
         value = self.getvalue(op.getarg(0))
         vindex = self.getvalue(op.getarg(1))
-        vresult = self.strgetitem(value, vindex, mode, op.result)
-        if op.result in self.optimizer.values:
-            assert self.getvalue(op.result) is vresult
+        vresult = self.strgetitem(value, vindex, mode, op)
+        if op in self.optimizer.values:
+            assert self.getvalue(op) is vresult
         else:
-            self.make_equal_to(op.result, vresult)
+            self.make_equal_to(op, vresult)
 
     def strgetitem(self, value, vindex, mode, resbox=None):
         value.ensure_nonnull()
@@ -595,7 +599,7 @@
             u = unicode(s)
         except UnicodeDecodeError:
             return False
-        self.make_constant(op.result, get_const_ptr_for_unicode(u))
+        self.make_constant(op, get_const_ptr_for_unicode(u))
         self.last_emitted_operation = REMOVED
         return True
 
@@ -604,7 +608,7 @@
         vright = self.getvalue(op.getarg(2))
         vleft.ensure_nonnull()
         vright.ensure_nonnull()
-        value = self.make_vstring_concat(op.result, op, mode)
+        value = self.make_vstring_concat(op, mode)
         value.setup(vleft, vright)
         return True
 
@@ -633,7 +637,7 @@
                                 vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
-        value = self.make_vstring_slice(op.result, op, mode)
+        value = self.make_vstring_slice(op, mode)
         value.setup(vstr, vstart, self.getvalue(lengthbox))
         return True
 
@@ -648,16 +652,16 @@
             isinstance(l2box, ConstInt) and
             l1box.value != l2box.value):
             # statically known to have a different length
-            self.make_constant(op.result, CONST_0)
+            self.make_constant(op, CONST_0)
             return True
         #
-        if self.handle_str_equal_level1(v1, v2, op.result, mode):
+        if self.handle_str_equal_level1(v1, v2, op, mode):
             return True
-        if self.handle_str_equal_level1(v2, v1, op.result, mode):
+        if self.handle_str_equal_level1(v2, v1, op, mode):
             return True
-        if self.handle_str_equal_level2(v1, v2, op.result, mode):
+        if self.handle_str_equal_level2(v1, v2, op, mode):
             return True
-        if self.handle_str_equal_level2(v2, v1, op.result, mode):
+        if self.handle_str_equal_level2(v2, v1, op, mode):
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
@@ -666,17 +670,20 @@
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
             self.generate_modified_call(do, [v1.force_box(self),
-                                             v2.force_box(self)], op.result, mode)
+                                             v2.force_box(self)], op, mode)
             return True
         return False
 
-    def handle_str_equal_level1(self, v1, v2, resultbox, mode):
+    def handle_str_equal_level1(self, v1, v2, resultop, mode):
         l2box = v2.getstrlen(None, mode, None)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
                 lengthbox = v1.getstrlen(self, mode, None)
                 seo = self.optimizer.send_extra_operation
-                seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
+                op = self.replace_op_with(resultop, rop.INT_EQ,
+                                          [lengthbox, CONST_0],
+                                          descr=DONT_CHANGE)
+                seo(op)
                 return True
             if l2box.value == 1:
                 l1box = v1.getstrlen(None, mode, None)
@@ -685,9 +692,10 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
-                                                  vchar2.force_box(self)],
-                                     resultbox))
+                    op = self.optimizer.replace_op_with(resultop, rop.INT_EQ,
+                          [vchar1.force_box(self), vchar2.force_box(self)],
+                        descr=DONT_CHANGE)
+                    seo(op)
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
@@ -696,19 +704,20 @@
                                                      v1.vstart.force_box(self),
                                                      v1.vlength.force_box(self),
                                                      vchar.force_box(self)],
-                                                resultbox, mode)
+                                                     resultop, mode)
                     return True
         #
         if v2.is_null():
             if v1.is_nonnull():
-                self.make_constant(resultbox, CONST_0)
+                self.make_constant(resultop, CONST_0)
                 return True
             if v1.is_null():
-                self.make_constant(resultbox, CONST_1)
+                self.make_constant(resultop, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
-                                           llhelper.CONST_NULL],
-                              resultbox)
+            op = self.optimizer.replace_op_with(resultop, rop.PTR_EQ,
+                                                [v1.force_box(self),
+                                                 llhelper.CONST_NULL],
+                descr=DONT_CHANGE)
             self.emit_operation(op)
             return True
         #
@@ -776,8 +785,9 @@
         oopspecindex += mode.OS_offset
         cic = self.optimizer.metainterp_sd.callinfocollection
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
-        op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
-                          descr=calldescr)
+        op = self.optimizer.replace_op_with(result, rop.CALL_I,
+                                            [ConstInt(func)] + args,
+                                            descr=calldescr)
         self.emit_operation(op)
 
     def propagate_forward(self, op):
diff --git a/rpython/jit/metainterp/quasiimmut.py b/rpython/jit/metainterp/quasiimmut.py
--- a/rpython/jit/metainterp/quasiimmut.py
+++ b/rpython/jit/metainterp/quasiimmut.py
@@ -120,12 +120,12 @@
             return ConstInt(self.cpu.bh_getfield_gc_i(struct, fielddescr))
 
     def is_still_valid_for(self, structconst):
-        assert self.structbox is not None
-        if not self.structbox.constbox().same_constant(structconst):
+        assert self.struct
+        if self.struct != structconst.getref_base():
             return False
         cpu = self.cpu
-        gcref = self.structbox.getref_base()
-        qmut = get_current_qmut_instance(cpu, gcref, self.mutatefielddescr)
+        qmut = get_current_qmut_instance(cpu, self.struct,
+                                         self.mutatefielddescr)
         if qmut is not self.qmut:
             return False
         else:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -13,6 +13,8 @@
     def repr_short(self, memo):
         return self.repr(memo)
 
+DONT_CHANGE = AbstractValue()
+
 def ResOperation(opnum, args, descr=None):
     cls = opclasses[opnum]
     op = cls()
@@ -96,6 +98,8 @@
             args = self.getarglist()
         if descr is None:
             descr = self.getdescr()
+        if descr is DONT_CHANGE:
+            descr = None
         newop = ResOperation(opnum, args, descr)
         if self.type != 'v':
             newop.copy_value_from(self)

From noreply at buildbot.pypy.org  Tue Dec  2 19:23:53 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  2 Dec 2014 19:23:53 +0100 (CET)
Subject: [pypy-commit] pypy optresult: two untested cases
Message-ID: <20141202182353.A68A51D29E9@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74788:a64ec027f6c0
Date: 2014-12-02 20:23 +0200
http://bitbucket.org/pypy/pypy/changeset/a64ec027f6c0/

Log:	two untested cases

diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -762,9 +762,10 @@
             vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
             vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
             seo = self.optimizer.send_extra_operation
-            seo(ResOperation(rop.INT_SUB, [vchar1.force_box(self),
-                                           vchar2.force_box(self)],
-                             op.result))
+            op = self.replace_op_with(op, rop.INT_SUB,
+                                      [vchar1.force_box(self),
+                                       vchar2.force_box(self)])
+            seo(op)
             return True
         return False
 
@@ -777,7 +778,7 @@
             length = v2.box.getint()
             v1.shrink(length)
             self.last_emitted_operation = REMOVED
-            self.make_equal_to(op.result, v1)
+            self.make_equal_to(op, v1)
             return True
         return False
 

From noreply at buildbot.pypy.org  Tue Dec  2 20:02:32 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  2 Dec 2014 20:02:32 +0100 (CET)
Subject: [pypy-commit] pypy framestate: progress
Message-ID: <20141202190232.B8B0F1C3288@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74789:2796501df19e
Date: 2014-12-01 17:27 +0100
http://bitbucket.org/pypy/pypy/changeset/2796501df19e/

Log:	progress

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -521,32 +521,35 @@
         self.target = reader.get_block_at(self.arg)
 
     def eval(self, ctx):
-        block = self.make_block(ctx)
+        block = self.make_block(ctx.stackdepth)
         ctx.blockstack.append(block)
 
 
 @bc_reader.register_opcode
 class SETUP_EXCEPT(SetupInstruction):
-    def make_block(self, ctx):
+    def make_block(self, stackdepth):
         from rpython.flowspace.flowcontext import ExceptBlock
-        return ExceptBlock(ctx.stackdepth, self.target)
+        return ExceptBlock(stackdepth, self.target)
 
 @bc_reader.register_opcode
 class SETUP_LOOP(SetupInstruction):
-    def make_block(self, ctx):
+    def make_block(self, stackdepth):
         from rpython.flowspace.flowcontext import LoopBlock
-        return LoopBlock(ctx.stackdepth, self.target)
+        return LoopBlock(stackdepth, self.target)
 
 @bc_reader.register_opcode
 class SETUP_FINALLY(SetupInstruction):
-    def make_block(self, ctx):
+    def make_block(self, stackdepth):
         from rpython.flowspace.flowcontext import FinallyBlock
-        return FinallyBlock(ctx.stackdepth, self.target)
+        return FinallyBlock(stackdepth, self.target)
 
 @bc_reader.register_opcode
 class SETUP_WITH(SetupInstruction):
+    def make_block(self, stackdepth):
+        from rpython.flowspace.flowcontext import WithBlock
+        return WithBlock(stackdepth, self.target)
+
     def eval(self, ctx):
-        from rpython.flowspace.flowcontext import WithBlock
         # A simpler version than the 'real' 2.7 one:
         # directly call manager.__enter__(), don't use special lookup functions
         # which don't make sense on the RPython type system.
@@ -555,7 +558,7 @@
         ctx.settopvalue(w_exit)
         w_enter = op.getattr(w_manager, const('__enter__')).eval(ctx)
         w_result = op.simple_call(w_enter).eval(ctx)
-        block = WithBlock(ctx.stackdepth, self.target)
+        block = self.make_block(ctx.stackdepth)
         ctx.blockstack.append(block)
         ctx.pushvalue(w_result)
 

From noreply at buildbot.pypy.org  Tue Dec  2 20:02:34 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  2 Dec 2014 20:02:34 +0100 (CET)
Subject: [pypy-commit] pypy framestate: remove BCInstruction.decode() and
 decode LOAD_CONST arg later
Message-ID: <20141202190234.014371C3288@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74790:bc3042836ed0
Date: 2014-12-02 16:03 +0100
http://bitbucket.org/pypy/pypy/changeset/bc3042836ed0/

Log:	remove BCInstruction.decode() and decode LOAD_CONST arg later

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -128,7 +128,7 @@
         elif opnum in opcode.hasname:
             oparg = code.names[oparg]
         try:
-            op = self.num2cls[opnum].decode(oparg, offset, code)
+            op = self.num2cls[opnum](oparg, offset)
         except KeyError:
             op = GenericOpcode(self.opnames[opnum], opnum, oparg, offset)
         return next_offset, op
@@ -329,10 +329,6 @@
         self.arg = arg
         self.offset = offset
 
-    @classmethod
-    def decode(cls, arg, offset, code):
-        return cls(arg, offset)
-
     def eval(self, ctx):
         pass
 
@@ -375,12 +371,9 @@
 
 @bc_reader.register_opcode
 class LOAD_CONST(BCInstruction):
-    @staticmethod
-    def decode(arg, offset, code):
-        return LOAD_CONST(code.consts[arg], offset)
-
     def eval(self, ctx):
-        ctx.pushvalue(const(self.arg))
+        v_arg = const(ctx.pycode.consts[self.arg])
+        ctx.pushvalue(v_arg)
 
 @bc_reader.register_opcode
 class DUP_TOP(BCInstruction):

From noreply at buildbot.pypy.org  Tue Dec  2 20:02:35 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  2 Dec 2014 20:02:35 +0100 (CET)
Subject: [pypy-commit] pypy framestate: extract a bc_reader.new_instr()
	method
Message-ID: <20141202190235.8BA7B1C3288@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74791:1d97ee1c1edd
Date: 2014-12-02 16:33 +0100
http://bitbucket.org/pypy/pypy/changeset/1d97ee1c1edd/

Log:	extract a bc_reader.new_instr() method

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -127,11 +127,17 @@
             oparg += next_offset
         elif opnum in opcode.hasname:
             oparg = code.names[oparg]
+        instr = self.new_instr(opnum, oparg, offset)
+        return next_offset, instr
+
+    def new_instr(self, opnum, arg, offset=-1):
+        if not isinstance(opnum, int):
+            opnum = opcode.opmap[opnum]
         try:
-            op = self.num2cls[opnum](oparg, offset)
+            return self.num2cls[opnum](arg, offset)
         except KeyError:
-            op = GenericOpcode(self.opnames[opnum], opnum, oparg, offset)
-        return next_offset, op
+            return GenericOpcode(self.opnames[opnum], opnum, arg, offset)
+
 
     def _iter_instr(self, code):
         self.offset = 0
@@ -344,6 +350,10 @@
     def __repr__(self):
         return "%s(%s)" % (self.name, self.arg)
 
+    def __eq__(self, other):
+        # NB: offsets are ignored, for testing convenience
+        return other.num == self.num and other.arg == self.arg
+
 class GenericOpcode(BCInstruction):
     def __init__(self, name, opcode, arg, offset=-1):
         self.name = name
diff --git a/rpython/flowspace/test/test_bytecode.py b/rpython/flowspace/test/test_bytecode.py
--- a/rpython/flowspace/test/test_bytecode.py
+++ b/rpython/flowspace/test/test_bytecode.py
@@ -9,5 +9,5 @@
         else:
             return 0
     bc_graph = bc_reader.build_flow(bc_reader.build_code(f.__code__))
-    print bc_graph.dump()
     assert [lst[0].offset for lst in bc_graph.dump()] == [0, 6, 10]
+    assert bc_graph.dump()[0][0] == bc_reader.new_instr('LOAD_FAST', 0)

From noreply at buildbot.pypy.org  Tue Dec  2 20:02:36 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue,  2 Dec 2014 20:02:36 +0100 (CET)
Subject: [pypy-commit] pypy framestate: WIP: storing blockstack status on
	bc_graph
Message-ID: <20141202190236.C24D91C3288@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74792:af6873879d10
Date: 2014-12-02 19:02 +0000
http://bitbucket.org/pypy/pypy/changeset/af6873879d10/

Log:	WIP: storing blockstack status on bc_graph

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -263,9 +263,11 @@
             instr = self.read(offset)
             yield instr
 
+    def all_blocks(self):
+        return set(x[0] for x in self.pos_index.values())
+
     def dump(self):
-        all_blocks = set(x[0] for x in self.pos_index.values())
-        blocks = sorted(all_blocks, key=lambda b: b.startpos)
+        blocks = sorted(self.all_blocks(), key=lambda b: b.startpos)
         return [b.operations for b in blocks]
 
 
@@ -274,6 +276,7 @@
     def __init__(self):
         self.parents = set()
         self._exits = []
+        self.blockstack = []
 
     def __getitem__(self, i):
         return self.operations[i]
@@ -522,6 +525,7 @@
     def bc_flow(self, reader):
         reader.curr_block.operations.append(self)
         self.target = reader.get_block_at(self.arg)
+        reader.curr_block.blockstack.append(self.make_block(-1))
 
     def eval(self, ctx):
         block = self.make_block(ctx.stackdepth)
diff --git a/rpython/flowspace/test/test_bytecode.py b/rpython/flowspace/test/test_bytecode.py
--- a/rpython/flowspace/test/test_bytecode.py
+++ b/rpython/flowspace/test/test_bytecode.py
@@ -2,12 +2,27 @@
 
 from rpython.flowspace.bytecode import bc_reader
 
+def make_graph(func):
+    return bc_reader.build_flow(bc_reader.build_code(func.__code__))
+
 def test_graph_dump():
     def f(x):
         if x:
             return 1
         else:
             return 0
-    bc_graph = bc_reader.build_flow(bc_reader.build_code(f.__code__))
+    bc_graph = make_graph(f)
     assert [lst[0].offset for lst in bc_graph.dump()] == [0, 6, 10]
     assert bc_graph.dump()[0][0] == bc_reader.new_instr('LOAD_FAST', 0)
+
+def test_blockstack():
+    def f():
+        for x in lst:
+            xxx
+    graph = make_graph(f)
+    for block in graph.all_blocks():
+        if bc_reader.new_instr('LOAD_GLOBAL', 'xxx') in block.operations:
+            break
+    else:
+        assert False
+    assert len(block.blockstack) == 1

From noreply at buildbot.pypy.org  Wed Dec  3 00:06:59 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Dec 2014 00:06:59 +0100 (CET)
Subject: [pypy-commit] stmgc c8-private-pages: in-progress
Message-ID: <20141202230700.399551D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: c8-private-pages
Changeset: r1510:e305d59e0a3d
Date: 2014-12-02 21:07 +0100
http://bitbucket.org/pypy/stmgc/changeset/e305d59e0a3d/

Log:	in-progress

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -3,20 +3,6 @@
 #endif
 
 
-#ifdef NDEBUG
-#define EVENTUALLY(condition)    {/* nothing */}
-#else
-#define EVENTUALLY(condition)                                   \
-    {                                                           \
-        if (!(condition)) {                                     \
-            acquire_privatization_lock(STM_SEGMENT->segment_num);\
-            if (!(condition))                                   \
-                stm_fatalerror("fails: " #condition);           \
-            release_privatization_lock(STM_SEGMENT->segment_num);\
-        }                                                       \
-    }
-#endif
-
 /* General helper: copies objects into our own segment, from some
    source described by a range of 'struct stm_undo_s'.  Maybe later
    we could specialize this function to avoid the checks in the
@@ -280,7 +266,7 @@
     dprintf(("_stm_validate(%p)\n", free_if_abort));
     /* go from last known entry in commit log to the
        most current one and apply all changes done
-       by other transactions. Abort if we read one of
+       by other transactions. Abort if we have read one of
        the committed objs. */
     struct stm_commit_log_entry_s *first_cl = STM_PSEGMENT->last_commit_log_entry;
     struct stm_commit_log_entry_s *next_cl, *last_cl, *cl;
@@ -346,6 +332,10 @@
 
                 segment_really_copied_from |= (1UL << cl->segment_num);
                 import_objects(cl->segment_num, -1, undo, end);
+
+                copy_bk_objs_in_page_from(
+                    segnum, -1,     /* any page */
+                    !needs_abort);  /* if we abort, we still want to copy everything */
             }
 
             /* last fully validated entry */
@@ -355,18 +345,15 @@
         }
         assert(cl == last_cl);
 
-        OPT_ASSERT(segment_really_copied_from < (1 << NB_SEGMENTS));
-        int segnum;
-        for (segnum = 0; segnum < NB_SEGMENTS; segnum++) {
-            if (segment_really_copied_from & (1UL << segnum)) {
-                /* here we can actually have our own modified version, so
-                   make sure to only copy things that are not modified in our
-                   segment... (if we do not abort) */
-                copy_bk_objs_in_page_from(
-                    segnum, -1,     /* any page */
-                    !needs_abort);  /* if we abort, we still want to copy everything */
-            }
-        }
+        /* OPT_ASSERT(segment_really_copied_from <= ((1UL << NB_SEGMENTS) - 1)); */
+        /* int segnum; */
+        /* for (segnum = 0; segnum < NB_SEGMENTS; segnum++) { */
+        /*     if (segment_really_copied_from & (1UL << segnum)) { */
+        /*         /\* here we can actually have our own modified version, so */
+        /*            make sure to only copy things that are not modified in our */
+        /*            segment... (if we do not abort) *\/ */
+        /*     } */
+        /* } */
     }
 
     /* done with modifications */
@@ -654,7 +641,7 @@
     enter_safe_point_if_requested();
     dprintf(("> start_transaction\n"));
 
-    s_mutex_unlock();
+    s_mutex_unlock();   // XXX it's probably possible to not acquire this here
 
     uint8_t old_rv = STM_SEGMENT->transaction_read_version;
     STM_SEGMENT->transaction_read_version = old_rv + 1;
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -73,7 +73,7 @@
     struct tree_s *young_outside_nursery;
     struct tree_s *nursery_objects_shadows;
 
-    uint8_t privatization_lock;
+    uint8_t privatization_lock;  // XXX KILL
 
     uint8_t safe_point;
     uint8_t transaction_state;
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -5,9 +5,8 @@
 
 static void setup_gcpage(void)
 {
-    /* XXXXXXX should use stm_file_pages, no? */
-    uninitialized_page_start = stm_file_pages;
-    uninitialized_page_stop  = stm_file_pages + NB_SHARED_PAGES * 4096UL;
+    //uninitialized_page_start = stm_file_pages;
+    //uninitialized_page_stop  = stm_file_pages + NB_SHARED_PAGES * 4096UL;
 }
 
 static void teardown_gcpage(void)
diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h
--- a/c8/stm/gcpage.h
+++ b/c8/stm/gcpage.h
@@ -2,8 +2,8 @@
 /* Granularity when grabbing more unused pages: take 50 at a time */
 #define GCPAGE_NUM_PAGES   50
 
-static char *uninitialized_page_start;   /* within segment 0 */
-static char *uninitialized_page_stop;
+//static char *uninitialized_page_start;   /* within segment 0 */
+//static char *uninitialized_page_stop;
 
 static void setup_gcpage(void);
 static void teardown_gcpage(void);
diff --git a/c8/stm/pages.c b/c8/stm/pages.c
--- a/c8/stm/pages.c
+++ b/c8/stm/pages.c
@@ -16,71 +16,26 @@
 
 /************************************************************/
 
-static void pages_initialize_shared_for(long segnum, uintptr_t pagenum, uintptr_t count)
+
+static void page_mark_accessible(long segnum, uintptr_t pagenum)
 {
-    /* call remap_file_pages() to make all pages in the range(pagenum,
-       pagenum+count) PAGE_SHARED in segnum, and PAGE_NO_ACCESS in other segments
-       initialize to |S|N|N|N| */
+    assert(get_page_status_in(segnum, pagenum) == PAGE_NO_ACCESS);
+    dprintf(("page_mark_accessible(%lu) in seg:%ld\n", pagenum, segnum));
 
-    dprintf(("pages_initialize_shared: 0x%ld - 0x%ld\n", pagenum, pagenum + count));
+    mprotect(get_virtual_page(segnum, pagenum), 4096, PROT_READ | PROT_WRITE);
 
-    assert(all_privatization_locks_acquired());
-
-    assert(pagenum < NB_PAGES);
-    if (count == 0)
-        return;
-
-    /* already shared after setup.c (also for the other count-1 pages) */
-    assert(get_page_status_in(segnum, pagenum) == PAGE_SHARED);
-
-    /* make other segments NO_ACCESS: */
-    uintptr_t i;
-    for (i = 0; i < NB_SEGMENTS; i++) {
-        if (i != segnum) {
-            char *segment_base = get_segment_base(i);
-            mprotect(segment_base + pagenum * 4096UL,
-                     count * 4096UL, PROT_NONE);
-
-            /* char *result = mmap( */
-            /*     segment_base + pagenum * 4096UL, */
-            /*     count * 4096UL, */
-            /*     PROT_NONE, */
-            /*     MAP_FIXED|MAP_NORESERVE|MAP_PRIVATE|MAP_ANONYMOUS, */
-            /*     -1, 0); */
-            /* if (result == MAP_FAILED) */
-            /*     stm_fatalerror("pages_initialize_shared failed (mmap): %m"); */
-
-
-            long amount = count;
-            while (amount-->0) {
-                set_page_status_in(i, pagenum + amount, PAGE_NO_ACCESS);
-            }
-        }
-    }
+    /* set this flag *after* we un-protected it, because XXX later */
+    set_page_status_in(segnum, pagenum, PAGE_ACCESSIBLE);
 }
 
+static void page_mark_inaccessible(long segnum, uintptr_t pagenum)
+{
+    assert(get_page_status_in(segnum, pagenum) == PAGE_ACCESSIBLE);
+    dprintf(("page_mark_inaccessible(%lu) in seg:%ld\n", pagenum, segnum));
 
-static void page_privatize_in(int segnum, uintptr_t pagenum)
-{
-#ifndef NDEBUG
-    long l;
-    for (l = 0; l < NB_SEGMENTS; l++) {
-        assert(get_priv_segment(l)->privatization_lock);
-    }
-#endif
-    assert(get_page_status_in(segnum, pagenum) == PAGE_NO_ACCESS);
-    dprintf(("page_privatize(%lu) in seg:%d\n", pagenum, segnum));
+    set_page_status_in(segnum, pagenum, PAGE_ACCESSIBLE);
 
-    char *addr = (char*)(get_virt_page_of(segnum, pagenum) * 4096UL);
-    char *result = mmap(
-        addr, 4096UL, PROT_READ | PROT_WRITE,
-        MAP_FIXED | MAP_PRIVATE | MAP_NORESERVE,
-        stm_object_pages_fd, get_file_page_of(pagenum) * 4096UL);
-    if (result == MAP_FAILED)
-        stm_fatalerror("page_privatize_in failed (mmap): %m");
-
-    set_page_status_in(segnum, pagenum, PAGE_PRIVATE);
-
-    volatile char *dummy = REAL_ADDRESS(get_segment_base(segnum), pagenum*4096UL);
-    *dummy = *dummy;            /* force copy-on-write from shared page */
+    char *addr = get_virtual_page(segnum, pagenum);
+    madvise(get_virtual_page(segnum, pagenum), 4096, MADV_DONTNEED);
+    mprotect(addr, 4096, PROT_NONE);
 }
diff --git a/c8/stm/pages.h b/c8/stm/pages.h
--- a/c8/stm/pages.h
+++ b/c8/stm/pages.h
@@ -3,90 +3,61 @@
       logical page
   We have virtual pages: one virtual address can point in some
       virtual page. We have NB_SEGMENTS virtual pages per logical page.
-  We have file pages: they correspond mostly to physical memory pages
-      used for mmap/remap_file_pages
 
-  A logical page is SHARED iff all NB_SEGMENTS virtual pages point to
-  one file page, and thus to the same logical page.
-
-  A logical page becomes PRIVATE if one virtual page still maps to the
-  original file page, and all others turn read protected.
-   -> only one can modify it.
-
-  A logical page can also be "PRIVATE IN A SEGMENT", referring to
-  the virtual page of the segment having its own file page backing.
-  It also implies the logical page is not read protected.
+  Each virtual page is either accessible, or PAGE_NO_ACCESS (and then
+  has no underlying memory).
 */
 
 #define PAGE_FLAG_START   END_NURSERY_PAGE
 #define PAGE_FLAG_END     NB_PAGES
-/* == NB_SHARED_PAGES */
 
 struct page_shared_s {
-#if NB_SEGMENTS <= 4
+#if NB_SEGMENTS <= 8
     uint8_t by_segment;
-#elif NB_SEGMENTS <= 8
+#elif NB_SEGMENTS <= 16
     uint16_t by_segment;
-#elif NB_SEGMENTS <= 16
+#elif NB_SEGMENTS <= 32
     uint32_t by_segment;
-#elif NB_SEGMENTS <= 32
+#elif NB_SEGMENTS <= 64
     uint64_t by_segment;
 #else
-#   error "NB_SEGMENTS > 32 not supported right now"
+#   error "NB_SEGMENTS > 64 not supported right now"
 #endif
 };
 
 enum {
-    PAGE_SHARED = 0,
-    PAGE_PRIVATE = 1,
-    PAGE_NO_ACCESS = 2,
+    PAGE_NO_ACCESS = 0,
+    PAGE_ACCESSIBLE = 1
 };
 
-static struct page_shared_s pages_status[NB_SHARED_PAGES];
+static struct page_shared_s pages_status[PAGE_FLAG_END - PAGE_FLAG_START];
 
-static void pages_initialize_shared_for(long segnum, uintptr_t pagenum, uintptr_t count);
-static void page_privatize_in(int segnum, uintptr_t pagenum);
+static void page_mark_accessible(long segnum, uintptr_t pagenum);
+static void page_mark_inaccessible(long segnum, uintptr_t pagenum);
 
 
-
-
-static inline uintptr_t get_virt_page_of(long segnum, uintptr_t pagenum)
+static inline char *get_virtual_page(long segnum, uintptr_t pagenum)
 {
     /* logical page -> virtual page */
-    return (uintptr_t)get_segment_base(segnum) / 4096UL + pagenum;
+    return get_segment_base(segnum) + pagenum * 4096;
 }
 
-static inline uintptr_t get_file_page_of(uintptr_t pagenum)
+static inline bool get_page_status_in(long segnum, uintptr_t pagenum)
 {
-    /* logical page -> file page */
-    return pagenum - PAGE_FLAG_START;
+    OPT_ASSERT(segnum < 8 * sizeof(struct page_shared_s));
+    volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
+        &pages_status[pagenum - PAGE_FLAG_START];
+
+    return (ps->by_segment >> segnum) & 1;
 }
 
-static inline uintptr_t get_page_of_file_page(uintptr_t file_page)
+static inline void set_page_status_in(long segnum, uintptr_t pagenum,
+                                      bool status)
 {
-    return file_page + END_NURSERY_PAGE;
-}
-
-static inline uint8_t get_page_status_in(long segnum, uintptr_t pagenum)
-{
-    int seg_shift = segnum * 2;
-    OPT_ASSERT(seg_shift < 8 * sizeof(struct page_shared_s));
+    OPT_ASSERT(segnum < 8 * sizeof(struct page_shared_s));
     volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
-        &pages_status[get_file_page_of(pagenum)];
-
-    return (ps->by_segment >> seg_shift) & 3;
-}
-
-static inline void set_page_status_in(long segnum, uintptr_t pagenum, uint8_t status)
-{
-    OPT_ASSERT(status < 3);
-
-    int seg_shift = segnum * 2;
-    OPT_ASSERT(seg_shift < 8 * sizeof(struct page_shared_s));
-    volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
-        &pages_status[get_file_page_of(pagenum)];
+        &pages_status[pagenum - PAGE_FLAG_START];
 
     assert(status != get_page_status_in(segnum, pagenum));
-    ps->by_segment &= ~(3UL << seg_shift); /* clear */
-    ps->by_segment |= status << seg_shift; /* set */
+    __sync_fetch_and_xor(&ps->by_segment, 1UL << segnum); /* invert the flag */
 }
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -7,56 +7,14 @@
 
 static void setup_mmap(char *reason)
 {
-    char name[] = "/__stmgc_c8__";
-
-    /* Create the big shared memory object, and immediately unlink it.
-       There is a small window where if this process is killed the
-       object is left around.  It doesn't seem possible to do anything
-       about it...
-    */
-    stm_object_pages_fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, 0600);
-    shm_unlink(name);
-
-    if (stm_object_pages_fd == -1)
-        stm_fatalerror("%s failed (stm_open): %m", reason);
-
-    if (ftruncate(stm_object_pages_fd, NB_SHARED_PAGES * 4096UL) != 0)
-        stm_fatalerror("%s failed (ftruncate): %m", reason);
-
-    stm_file_pages = mmap(NULL, NB_SHARED_PAGES * 4096UL,
-                          PROT_READ | PROT_WRITE,
-                          MAP_SHARED | MAP_NORESERVE,
-                          stm_object_pages_fd, 0);
-
-    if (stm_file_pages == MAP_FAILED)
-        stm_fatalerror("%s failed (mmap): %m", reason);
-
-
     /* reserve the whole virtual memory space of the program for
-       all segments: */
-    stm_object_pages = mmap(NULL, TOTAL_MEMORY,
-                            PROT_READ | PROT_WRITE,
+       all segments: (for now in one big block, but later could be
+       allocated per-segment) */
+    stm_object_pages = mmap(NULL, TOTAL_MEMORY, PROT_NONE,
                             MAP_PRIVATE | MAP_NORESERVE | MAP_ANONYMOUS,
                             -1, 0);
     if (stm_object_pages == MAP_FAILED)
         stm_fatalerror("%s failed (mmap): %m", reason);
-
-    /* remap the shared part of the segments to the file pages */
-    long l;
-    for (l = 0; l < NB_SEGMENTS; l++) {
-        char *result = mmap(
-            stm_object_pages + (l * NB_PAGES + END_NURSERY_PAGE) * 4096UL, /* addr */
-            NB_SHARED_PAGES * 4096UL, /* len */
-            PROT_READ | PROT_WRITE,
-            MAP_FIXED | MAP_SHARED | MAP_NORESERVE,
-            stm_object_pages_fd, 0); /* file & offset */
-        if (result == MAP_FAILED)
-            stm_fatalerror("%s failed (mmap): %m", reason);
-    }
-}
-static void close_fd_mmap(int map_fd)
-{
-    close(map_fd);
 }
 
 static void setup_protection_settings(void)
@@ -65,17 +23,13 @@
     for (i = 0; i < NB_SEGMENTS; i++) {
         char *segment_base = get_segment_base(i);
 
-        /* In each segment, the first page is where TLPREFIX'ed
-           NULL accesses land.  We mprotect it so that accesses fail. */
-        mprotect(segment_base, 4096, PROT_NONE);
+        /* In each segment, the second page is where STM_SEGMENT lands. */
+        mprotect(segment_base + 4096, 4096, PROT_READ | PROT_WRITE);
 
-        /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */
-        if (FIRST_READMARKER_PAGE > 2)
-            mprotect(segment_base + 2 * 4096,
-                     (FIRST_READMARKER_PAGE - 2) * 4096UL,
-                     PROT_NONE);
-
-        /* STM_SEGMENT-TL is in page 1 */
+        /* Make the read marker pages accessible, as well as the nursery. */
+        mprotect(segment_base + FIRST_READMARKER_PAGE * 4096,
+                 (NB_READMARKER_PAGES + NB_NURSERY_PAGES) * 4096,
+                 PROT_READ | PROT_WRITE);
     }
 }
 
@@ -184,7 +138,6 @@
     stm_object_pages = NULL;
     commit_log_root.next = NULL; /* xxx:free them */
     commit_log_root.segment_num = -1;
-    close_fd_mmap(stm_object_pages_fd);
 
     teardown_sync();
     teardown_gcpage();

From noreply at buildbot.pypy.org  Wed Dec  3 08:31:13 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed,  3 Dec 2014 08:31:13 +0100 (CET)
Subject: [pypy-commit] stmgc c8-small-uniform: add test showing a bug in
	stm_validate()
Message-ID: <20141203073113.87C581C07E8@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: c8-small-uniform
Changeset: r1511:c849b169bc4e
Date: 2014-12-02 15:56 +0100
http://bitbucket.org/pypy/stmgc/changeset/c849b169bc4e/

Log:	add test showing a bug in stm_validate()

diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -5,7 +5,6 @@
 
 static void setup_gcpage(void)
 {
-    /* XXXXXXX should use stm_file_pages, no? */
     uninitialized_page_start = stm_file_pages;
     uninitialized_page_stop  = stm_file_pages + NB_SHARED_PAGES * 4096UL;
 }
diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py
--- a/c8/test/test_basic.py
+++ b/c8/test/test_basic.py
@@ -811,6 +811,33 @@
 
         assert stm_get_char(lp_char_5, 384 - 1) == 'o'
 
+    def test_bug3(self):
+        lp_char_5 = stm_allocate_old(384)
+
+        for i in range(NB_SEGMENTS):
+            self.start_transaction()
+            stm_set_char(lp_char_5, '\0', HDR, False)
+            self.commit_transaction()
+            
+        #
+        self.switch(2)
+        self.start_transaction()
+        stm_set_char(lp_char_5, 'i', HDR, False)
+        self.commit_transaction()
+
+        self.start_transaction()
+        stm_set_char(lp_char_5, 'x', HDR, False)
+
+        self.switch(1)
+        self.start_transaction()
+        stm_set_char(lp_char_5, 'a', HDR, False)
+        self.commit_transaction()
+        
+        self.switch(0)
+        self.start_transaction()
+        assert stm_get_char(lp_char_5, HDR) == 'a'
+
+        
     def test_repeated_wb(self):
         lp_char_5 = stm_allocate_old(384)
 

From noreply at buildbot.pypy.org  Wed Dec  3 08:31:14 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed,  3 Dec 2014 08:31:14 +0100 (CET)
Subject: [pypy-commit] stmgc c8-small-uniform: fix for previous commit
Message-ID: <20141203073114.9AB061C07E8@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: c8-small-uniform
Changeset: r1512:757f479c2b41
Date: 2014-12-02 16:12 +0100
http://bitbucket.org/pypy/stmgc/changeset/757f479c2b41/

Log:	fix for previous commit

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -3,20 +3,6 @@
 #endif
 
 
-#ifdef NDEBUG
-#define EVENTUALLY(condition)    {/* nothing */}
-#else
-#define EVENTUALLY(condition)                                   \
-    {                                                           \
-        if (!(condition)) {                                     \
-            acquire_privatization_lock(STM_SEGMENT->segment_num);\
-            if (!(condition))                                   \
-                stm_fatalerror("fails: " #condition);           \
-            release_privatization_lock(STM_SEGMENT->segment_num);\
-        }                                                       \
-    }
-#endif
-
 /* General helper: copies objects into our own segment, from some
    source described by a range of 'struct stm_undo_s'.  Maybe later
    we could specialize this function to avoid the checks in the
@@ -346,6 +332,13 @@
 
                 segment_really_copied_from |= (1UL << cl->segment_num);
                 import_objects(cl->segment_num, -1, undo, end);
+
+                /* here we can actually have our own modified version, so
+                   make sure to only copy things that are not modified in our
+                   segment... (if we do not abort) */
+                copy_bk_objs_in_page_from
+                    (cl->segment_num, -1,     /* any page */
+                     !needs_abort);  /* if we abort, we still want to copy everything */
             }
 
             /* last fully validated entry */
@@ -355,18 +348,19 @@
         }
         assert(cl == last_cl);
 
-        OPT_ASSERT(segment_really_copied_from < (1 << NB_SEGMENTS));
-        int segnum;
-        for (segnum = 0; segnum < NB_SEGMENTS; segnum++) {
-            if (segment_really_copied_from & (1UL << segnum)) {
-                /* here we can actually have our own modified version, so
-                   make sure to only copy things that are not modified in our
-                   segment... (if we do not abort) */
-                copy_bk_objs_in_page_from(
-                    segnum, -1,     /* any page */
-                    !needs_abort);  /* if we abort, we still want to copy everything */
-            }
-        }
+        /* XXX: this optimization fails in test_basic.py, bug3 */
+        /* OPT_ASSERT(segment_really_copied_from < (1 << NB_SEGMENTS)); */
+        /* int segnum; */
+        /* for (segnum = 0; segnum < NB_SEGMENTS; segnum++) { */
+        /*     if (segment_really_copied_from & (1UL << segnum)) { */
+        /*         /\* here we can actually have our own modified version, so */
+        /*            make sure to only copy things that are not modified in our */
+        /*            segment... (if we do not abort) *\/ */
+        /*         copy_bk_objs_in_page_from( */
+        /*             segnum, -1,     /\* any page *\/ */
+        /*             !needs_abort);  /\* if we abort, we still want to copy everything *\/ */
+        /*     } */
+        /* } */
     }
 
     /* done with modifications */
@@ -503,6 +497,7 @@
     assert(!(bk_obj->stm_flags & GCFLAG_WB_EXECUTED));
 
     dprintf(("write_slowpath(%p): sz=%lu, bk=%p\n", obj, obj_size, bk_obj));
+
  retry:
     /* privatize pages: */
     /* XXX don't always acquire all locks... */
@@ -553,6 +548,7 @@
     /* all pages are either private or we were the first to write to a shared
        page and therefore got it as our private one */
 
+
     /* phew, now add the obj to the write-set and register the
        backup copy. */
     /* XXX: we should not be here at all fiddling with page status
diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py
--- a/c8/test/test_basic.py
+++ b/c8/test/test_basic.py
@@ -828,15 +828,20 @@
         self.start_transaction()
         stm_set_char(lp_char_5, 'x', HDR, False)
 
+        #
         self.switch(1)
         self.start_transaction()
         stm_set_char(lp_char_5, 'a', HDR, False)
         self.commit_transaction()
-        
+
+        #
         self.switch(0)
         self.start_transaction()
         assert stm_get_char(lp_char_5, HDR) == 'a'
 
+        #
+        py.test.raises(Conflict, self.switch, 2)
+
         
     def test_repeated_wb(self):
         lp_char_5 = stm_allocate_old(384)

From noreply at buildbot.pypy.org  Wed Dec  3 08:31:15 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed,  3 Dec 2014 08:31:15 +0100 (CET)
Subject: [pypy-commit] stmgc c8-small-uniform: fix another (currently
	untestable) bug
Message-ID: <20141203073115.C9D261C07E8@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at gmail.com>
Branch: c8-small-uniform
Changeset: r1513:222d1845dff3
Date: 2014-12-02 17:00 +0100
http://bitbucket.org/pypy/stmgc/changeset/222d1845dff3/

Log:	fix another (currently untestable) bug

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -278,94 +278,108 @@
         return;
     }
 
-    /* Find the set of segments we need to copy from and lock them: */
-    uint64_t segments_to_lock = 1UL << my_segnum;
-    cl = first_cl;
-    while ((next_cl = cl->next) != NULL) {
-        if (next_cl == INEV_RUNNING) {
+    bool needs_abort = false;
+    
+    while(1) {
+        /* retry IF: */
+        /* if at the time of "HERE" (s.b.) there happen to be
+           more commits (and bk copies) then it could be that
+           copy_bk_objs_in_page_from (s.b.) reads a bk copy that
+           is itself more recent than last_cl. This is fixed
+           by re-validating. */
+        first_cl = STM_PSEGMENT->last_commit_log_entry;
+        if (first_cl->next == NULL)
+            break;
+
+        /* Find the set of segments we need to copy from and lock them: */
+        uint64_t segments_to_lock = 1UL << my_segnum;
+        cl = first_cl;
+        while ((next_cl = cl->next) != NULL) {
+            if (next_cl == INEV_RUNNING) {
 #if STM_TESTS
-            if (free_if_abort != (void *)-1)
-                free(free_if_abort);
-            stm_abort_transaction();
+                if (free_if_abort != (void *)-1)
+                    free(free_if_abort);
+                stm_abort_transaction();
 #endif
-            /* only validate entries up to INEV */
-            break;
+                /* only validate entries up to INEV */
+                break;
+            }
+            assert(next_cl->rev_num > cl->rev_num);
+            cl = next_cl;
+
+            if (cl->written_count) {
+                segments_to_lock |= (1UL << cl->segment_num);
+            }
         }
-        assert(next_cl->rev_num > cl->rev_num);
-        cl = next_cl;
+        last_cl = cl;
+        /* HERE */
+        acquire_modification_lock_set(segments_to_lock);
 
-        if (cl->written_count) {
-            segments_to_lock |= (1UL << cl->segment_num);
-        }
-    }
-    last_cl = cl;
-    acquire_modification_lock_set(segments_to_lock);
 
+        /* import objects from first_cl to last_cl: */
+        if (first_cl != last_cl) {
+            uint64_t segment_really_copied_from = 0UL;
 
-    /* import objects from first_cl to last_cl: */
-    bool needs_abort = false;
-    if (first_cl != last_cl) {
-        uint64_t segment_really_copied_from = 0UL;
-
-        cl = first_cl;
-        while ((cl = cl->next) != NULL) {
-            if (!needs_abort) {
-                struct stm_undo_s *undo = cl->written;
-                struct stm_undo_s *end = cl->written + cl->written_count;
-                for (; undo < end; undo++) {
-                    if (_stm_was_read(undo->object)) {
-                        /* first reset all modified objects from the backup
-                           copies as soon as the first conflict is detected;
-                           then we will proceed below to update our segment from
-                           the old (but unmodified) version to the newer version.
-                        */
-                        reset_modified_from_backup_copies(my_segnum);
-                        needs_abort = true;
-                        break;
+            cl = first_cl;
+            while ((cl = cl->next) != NULL) {
+                if (!needs_abort) {
+                    struct stm_undo_s *undo = cl->written;
+                    struct stm_undo_s *end = cl->written + cl->written_count;
+                    for (; undo < end; undo++) {
+                        if (_stm_was_read(undo->object)) {
+                            /* first reset all modified objects from the backup
+                               copies as soon as the first conflict is detected;
+                               then we will proceed below to update our segment from
+                               the old (but unmodified) version to the newer version.
+                            */
+                            reset_modified_from_backup_copies(my_segnum);
+                            needs_abort = true;
+                            break;
+                        }
                     }
                 }
+
+                if (cl->written_count) {
+                    struct stm_undo_s *undo = cl->written;
+                    struct stm_undo_s *end = cl->written + cl->written_count;
+
+                    segment_really_copied_from |= (1UL << cl->segment_num);
+                    import_objects(cl->segment_num, -1, undo, end);
+
+                    /* here we can actually have our own modified version, so
+                       make sure to only copy things that are not modified in our
+                       segment... (if we do not abort) */
+                    copy_bk_objs_in_page_from
+                        (cl->segment_num, -1,     /* any page */
+                         !needs_abort);  /* if we abort, we still want to copy everything */
+                }
+
+                /* last fully validated entry */
+                STM_PSEGMENT->last_commit_log_entry = cl;
+                if (cl == last_cl)
+                    break;
             }
+            assert(cl == last_cl);
 
-            if (cl->written_count) {
-                struct stm_undo_s *undo = cl->written;
-                struct stm_undo_s *end = cl->written + cl->written_count;
+            /* XXX: this optimization fails in test_basic.py, bug3 */
+            /* OPT_ASSERT(segment_really_copied_from < (1 << NB_SEGMENTS)); */
+            /* int segnum; */
+            /* for (segnum = 0; segnum < NB_SEGMENTS; segnum++) { */
+            /*     if (segment_really_copied_from & (1UL << segnum)) { */
+            /*         /\* here we can actually have our own modified version, so */
+            /*            make sure to only copy things that are not modified in our */
+            /*            segment... (if we do not abort) *\/ */
+            /*         copy_bk_objs_in_page_from( */
+            /*             segnum, -1,     /\* any page *\/ */
+            /*             !needs_abort);  /\* if we abort, we still want to copy everything *\/ */
+            /*     } */
+            /* } */
+        }
 
-                segment_really_copied_from |= (1UL << cl->segment_num);
-                import_objects(cl->segment_num, -1, undo, end);
-
-                /* here we can actually have our own modified version, so
-                   make sure to only copy things that are not modified in our
-                   segment... (if we do not abort) */
-                copy_bk_objs_in_page_from
-                    (cl->segment_num, -1,     /* any page */
-                     !needs_abort);  /* if we abort, we still want to copy everything */
-            }
-
-            /* last fully validated entry */
-            STM_PSEGMENT->last_commit_log_entry = cl;
-            if (cl == last_cl)
-                break;
-        }
-        assert(cl == last_cl);
-
-        /* XXX: this optimization fails in test_basic.py, bug3 */
-        /* OPT_ASSERT(segment_really_copied_from < (1 << NB_SEGMENTS)); */
-        /* int segnum; */
-        /* for (segnum = 0; segnum < NB_SEGMENTS; segnum++) { */
-        /*     if (segment_really_copied_from & (1UL << segnum)) { */
-        /*         /\* here we can actually have our own modified version, so */
-        /*            make sure to only copy things that are not modified in our */
-        /*            segment... (if we do not abort) *\/ */
-        /*         copy_bk_objs_in_page_from( */
-        /*             segnum, -1,     /\* any page *\/ */
-        /*             !needs_abort);  /\* if we abort, we still want to copy everything *\/ */
-        /*     } */
-        /* } */
+        /* done with modifications */
+        release_modification_lock_set(segments_to_lock);
     }
 
-    /* done with modifications */
-    release_modification_lock_set(segments_to_lock);
-
     if (needs_abort) {
         if (free_if_abort != (void *)-1)
             free(free_if_abort);

From noreply at buildbot.pypy.org  Wed Dec  3 11:06:40 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Dec 2014 11:06:40 +0100 (CET)
Subject: [pypy-commit] pypy default: oups. test and fix
Message-ID: <20141203100640.0C54F1D2925@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74793:d088178015dd
Date: 2014-12-03 11:06 +0100
http://bitbucket.org/pypy/pypy/changeset/d088178015dd/

Log:	oups. test and fix

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -3892,12 +3892,16 @@
 
     def test_int_signext(self):
         numbytes_cases = [1, 2] if IS_32_BIT else [1, 2, 4]
-        for numbytes in numbytes_cases:
+        for spill in ["", "force_spill(i1)"]:
+          for numbytes in numbytes_cases:
+            print (spill, numbytes)
             ops = """
             [i0]
-            i1 = int_signext(i0, %d)
-            finish(i1, descr=descr)
-            """ % numbytes
+            i1 = int_sub(i0, 0)     # force in register
+            %s
+            i2 = int_signext(i1, %d)
+            finish(i2, descr=descr)
+            """ % (spill, numbytes)
             descr = BasicFinalDescr()
             loop = parse(ops, self.cpu, namespace=locals())
             looptoken = JitCellToken()
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1148,6 +1148,8 @@
         argloc, numbytesloc = arglocs
         assert isinstance(numbytesloc, ImmedLoc)
         if numbytesloc.value == 1:
+            if isinstance(argloc, RegLoc):
+                argloc = argloc.lowest8bits()
             self.mc.MOVSX8(resloc, argloc)
         elif numbytesloc.value == 2:
             self.mc.MOVSX16(resloc, argloc)

From noreply at buildbot.pypy.org  Wed Dec  3 11:33:05 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Dec 2014 11:33:05 +0100 (CET)
Subject: [pypy-commit] pypy default: Add INT_SIGNEXT to test_random,
 and found out another corner case on
Message-ID: <20141203103305.9BD0E1C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74794:4634aad50952
Date: 2014-12-03 11:29 +0100
http://bitbucket.org/pypy/pypy/changeset/4634aad50952/

Log:	Add INT_SIGNEXT to test_random, and found out another corner case on
	32-bit only.

diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py
--- a/rpython/jit/backend/test/test_random.py
+++ b/rpython/jit/backend/test/test_random.py
@@ -321,6 +321,14 @@
         else:
             self.put(builder, [ConstFloat(r.random_float_storage())])
 
+class SignExtOperation(AbstractOperation):
+    def produce_into(self, builder, r):
+        sizes = [1, 2]
+        if sys.maxint > (1 << 32):
+            sizes.append(4)
+        self.put(builder, [r.choice(builder.intvars),
+                           ConstInt(r.choice(sizes))])
+
 class BinaryOperation(AbstractOperation):
     def __init__(self, opnum, and_mask=-1, or_mask=0, boolres=False):
         AbstractOperation.__init__(self, opnum, boolres=boolres)
@@ -509,6 +517,7 @@
 OPERATIONS.append(UnaryOperation(rop.INT_IS_TRUE, boolres=True))
 OPERATIONS.append(UnaryOperation(rop.INT_IS_ZERO, boolres=True))
 OPERATIONS.append(ConstUnaryOperation(rop.SAME_AS, boolres='sometimes'))
+OPERATIONS.append(SignExtOperation(rop.INT_SIGNEXT))
 
 for _op in [rop.INT_ADD_OVF,
             rop.INT_SUB_OVF,
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1147,9 +1147,20 @@
     def genop_int_signext(self, op, arglocs, resloc):
         argloc, numbytesloc = arglocs
         assert isinstance(numbytesloc, ImmedLoc)
+        assert isinstance(resloc, RegLoc)
         if numbytesloc.value == 1:
             if isinstance(argloc, RegLoc):
-                argloc = argloc.lowest8bits()
+                if WORD == 4 and argloc.value >= 4:
+                    # meh, can't read the lowest byte of esi or edi on 32-bit
+                    if resloc.value < 4:
+                        self.mc.MOV(resloc, argloc)
+                        argloc = resloc
+                    else:
+                        tmploc = RawEspLoc(0, INT)
+                        self.mc.MOV(tmploc, argloc)
+                        argloc = tmploc
+                if isinstance(argloc, RegLoc):
+                    argloc = argloc.lowest8bits()
             self.mc.MOVSX8(resloc, argloc)
         elif numbytesloc.value == 2:
             self.mc.MOVSX16(resloc, argloc)
diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py
--- a/rpython/jit/backend/x86/regloc.py
+++ b/rpython/jit/backend/x86/regloc.py
@@ -151,6 +151,8 @@
 
     def lowest8bits(self):
         assert not self.is_xmm
+        if WORD == 4:
+            assert 0 <= self.value < 4
         return RegLoc(rx86.low_byte(self.value), False)
 
     def higher8bits(self):
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -33,8 +33,9 @@
     xmmnames = ['xmm%d' % i for i in range(16)]
 
 def low_byte(reg):
-    # XXX: On 32-bit, this only works for 0 <= reg < 4
-    # Maybe we should check this?
+    # On 32-bit, this only works for 0 <= reg < 4.  The caller checks that.
+    # On 64-bit, it works for any register, but the assembler instruction
+    # must include a REX prefix (possibly with no modifier flags).
     return reg | BYTE_REG_FLAG
 
 def high_byte(reg):

From noreply at buildbot.pypy.org  Wed Dec  3 12:20:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Dec 2014 12:20:01 +0100 (CET)
Subject: [pypy-commit] pypy default: fix a bit more cleanly without writing
	to the stack
Message-ID: <20141203112001.5FB231D286C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74795:4e877792713b
Date: 2014-12-03 12:19 +0100
http://bitbucket.org/pypy/pypy/changeset/4e877792713b/

Log:	fix a bit more cleanly without writing to the stack

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1152,15 +1152,15 @@
             if isinstance(argloc, RegLoc):
                 if WORD == 4 and argloc.value >= 4:
                     # meh, can't read the lowest byte of esi or edi on 32-bit
-                    if resloc.value < 4:
+                    if resloc is not argloc:
                         self.mc.MOV(resloc, argloc)
                         argloc = resloc
-                    else:
-                        tmploc = RawEspLoc(0, INT)
-                        self.mc.MOV(tmploc, argloc)
-                        argloc = tmploc
-                if isinstance(argloc, RegLoc):
-                    argloc = argloc.lowest8bits()
+                    if resloc.value >= 4:
+                        # still annoyed, hack needed
+                        self.mc.SHL_ri(resloc.value, 24)
+                        self.mc.SAR_ri(resloc.value, 24)
+                        return
+                argloc = argloc.lowest8bits()
             self.mc.MOVSX8(resloc, argloc)
         elif numbytesloc.value == 2:
             self.mc.MOVSX16(resloc, argloc)

From noreply at buildbot.pypy.org  Wed Dec  3 15:19:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Dec 2014 15:19:52 +0100 (CET)
Subject: [pypy-commit] creflect default: pointer types
Message-ID: <20141203141952.93E711C363D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r140:dc4373af8fab
Date: 2014-12-03 15:20 +0100
http://bitbucket.org/cffi/creflect/changeset/dc4373af8fab/

Log:	pointer types

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -10,30 +10,63 @@
     PyObject *types_dict;
 } zeffir_builder_t;
 
-static CTypeDescrObject *get_cached_type(_crx_builder_t *cb, const char *name,
-                                         int accept_flags)
+static PyObject *combine_type_name(_crx_builder_t *cb, CTypeDescrObject *ct,
+                                   const char *extra_text)
+{
+    /* Build a PyString with the name given by combining 'ct' and 'extra_text'.
+       'ct' can be NULL.  If an error occurs (or has already occurred),
+       returns NULL. */
+    size_t base_name_len, extra_name_len;
+    PyObject *result;
+    char *p;
+
+    if (PyErr_Occurred())
+        return NULL;
+
+    base_name_len = (ct != NULL ? strlen(ct->ct_name) : 0);
+    extra_name_len = strlen(extra_text);
+    result = PyString_FromStringAndSize(NULL, base_name_len + extra_name_len);
+    if (result == NULL)
+        return NULL;
+
+    p = PyString_AS_STRING(result);
+    if (ct != NULL) {
+        memcpy(p, ct->ct_name, ct->ct_name_position);
+        p += ct->ct_name_position;
+    }
+    memcpy(p, extra_text, extra_name_len);
+    if (ct != NULL) {
+        p += extra_name_len;
+        memcpy(p, ct->ct_name + ct->ct_name_position,
+               base_name_len - ct->ct_name_position);
+    }
+    return result;
+}
+
+static CTypeDescrObject *get_cached_type(_crx_builder_t *cb, PyObject *name_obj)
+{
+    /* Look up the type called 'name_obj' and check that it's a type with the
+       right flags.  Returns either the type or NULL, without changing
+       the reference counts.  Never raises an exception.
+    */
+    PyObject *types_dict = ((zeffir_builder_t *)cb)->types_dict;
+    PyObject *x;
+
+    assert(name_obj != NULL);
+    x = PyDict_GetItem(types_dict, name_obj);
+    if (x == NULL || !CTypeDescr_Check(x))
+        return NULL;
+    else
+        return (CTypeDescrObject *)x;
+}
+
+static void put_cached_type(_crx_builder_t *cb, PyObject *name_obj,
+                            CTypeDescrObject *ct)
 {
     PyObject *types_dict = ((zeffir_builder_t *)cb)->types_dict;
-    PyObject *x = PyDict_GetItemString(types_dict, name);
-    CTypeDescrObject *ct;
-
-    if (x == NULL || !CTypeDescr_Check(x))
-        return NULL;
-    ct = (CTypeDescrObject *)x;
-    if ((ct->ct_flags & accept_flags) == 0)
-        return NULL;
-    return ct;
-}
-
-static _crx_type_t *put_cached_type(_crx_builder_t *cb, const char *name,
-                                    CTypeDescrObject *ct)
-{
-    PyObject *types_dict = ((zeffir_builder_t *)cb)->types_dict;
-    int err = PyDict_SetItemString(types_dict, name, (PyObject *)ct);
-    Py_DECREF(ct);
-    if (err < 0)
-        return NULL;
-    return ct;    /* still a reference in the dict */
+    PyDict_SetItem(types_dict, name_obj, (PyObject *)ct);
+    Py_DECREF(ct);   /* still a reference in the dict,
+                        unless PyDict_SetItem failed */
 }
 
 static _crx_type_t *zef_get_void_type(_crx_builder_t *cb)
@@ -54,26 +87,33 @@
 static _crx_type_t *zef_get_signed_type(_crx_builder_t *cb, size_t sz,
                                         const char *name)
 {
-    CTypeDescrObject *td = get_cached_type(cb, name, CT_PRIMITIVE_SIGNED);
-    if (td != NULL && td->ct_size == sz)
-        return td;
+    PyObject *name_obj;
+    CTypeDescrObject *ct;
 
-    size_t name_length = strlen(name);
-
-    td = ctypedescr_new(name_length + 1);
-    if (td == NULL)
+    name_obj = combine_type_name(cb, NULL, name);
+    if (name_obj == NULL)
         return NULL;
 
-    memcpy(td->ct_name, name, name_length + 1);
-    td->ct_name_position = name_length;
-    td->ct_size = sz;
-    //td->ct_length = ptypes->align;
-    //td->ct_extra = ffitype;
-    td->ct_flags = CT_PRIMITIVE_SIGNED;
-    if (td->ct_size <= sizeof(long))
-        td->ct_flags |= CT_PRIMITIVE_FITS_LONG;
+    ct = get_cached_type(cb, name_obj);
+    if (ct && (ct->ct_flags & CT_PRIMITIVE_SIGNED) && ct->ct_size == sz)
+        goto done;
 
-    return put_cached_type(cb, name, td);
+    ct = ctypedescr_new(name_obj, PyString_GET_SIZE(name_obj));
+    if (ct == NULL)
+        goto done;
+
+    ct->ct_size = sz;
+    //ct->ct_length = ptypes->align;
+    //ct->ct_extra = ffitype;
+    ct->ct_flags = CT_PRIMITIVE_SIGNED;
+    if (ct->ct_size <= sizeof(long))
+        ct->ct_flags |= CT_PRIMITIVE_FITS_LONG;
+
+    put_cached_type(cb, name_obj, ct);
+
+ done:
+    Py_DECREF(name_obj);
+    return ct;
 }
 
 static _crx_type_t *zef_get_unsigned_type(_crx_builder_t *cb, size_t sz,
@@ -106,7 +146,41 @@
 static _crx_type_t *zef_get_pointer_type(_crx_builder_t *cb,
                                          _crx_type_t *totype, int toquals)
 {
-    abort();
+    const char *extra;
+    PyObject *name_obj;
+    CTypeDescrObject *ct;
+
+    if (totype->ct_flags & CT_ARRAY)
+        extra = "(*)";
+    else
+        extra = " *";
+
+    name_obj = combine_type_name(cb, totype, extra);
+    if (name_obj == NULL)
+        return NULL;
+
+    ct = get_cached_type(cb, name_obj);
+    if (ct && (ct->ct_flags & CT_POINTER))
+        goto done;
+
+    ct = ctypedescr_new(name_obj, totype->ct_name_position + 2);
+    if (ct == NULL)
+        goto done;
+
+    ct->ct_size = sizeof(void *);
+    ct->ct_flags = CT_POINTER;
+    if (totype->ct_flags & CT_VOID)
+        ct->ct_flags |= CT_IS_VOID_PTR;
+    if ((totype->ct_flags & CT_VOID) ||
+        ((totype->ct_flags & CT_PRIMITIVE_CHAR) &&
+         totype->ct_size == sizeof(char)))
+        ct->ct_flags |= CT_CAST_ANYTHING;   /* 'void *' or 'char *' only */
+
+    put_cached_type(cb, name_obj, ct);
+
+ done:
+    Py_DECREF(name_obj);
+    return ct;
 }
 
 static _crx_type_t *zef_get_array_type(_crx_builder_t *cb, _crx_type_t *t,
diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -45,7 +45,7 @@
     Py_ssize_t ct_length;   /* length of arrays, or -1 if unknown */
     int ct_flags;           /* CT_xxx flags */
 
-    int ct_name_position;   /* index in ct_name of where to put a var name */
+    size_t ct_name_position;/* index in ct_name of where to put a var name */
     char ct_name[1];        /* string, e.g. "int *" for pointers to ints */
 };
 
@@ -64,11 +64,12 @@
 
 /************************************************************/
 
-static CTypeDescrObject *ctypedescr_new(size_t name_size)
+static CTypeDescrObject *ctypedescr_new(PyObject *name_obj,
+                                        size_t name_position)
 {
     CTypeDescrObject *ct = PyObject_GC_NewVar(CTypeDescrObject,
                                               &CTypeDescr_Type,
-                                              name_size);
+                                              PyString_GET_SIZE(name_obj) + 1);
     if (ct == NULL)
         return NULL;
 
@@ -77,6 +78,9 @@
     ct->ct_weakreflist = NULL;
     ct->ct_size = -1;
     ct->ct_length = -1;
+    ct->ct_name_position = name_position;
+    memcpy(ct->ct_name, PyString_AS_STRING(name_obj),
+           PyString_GET_SIZE(name_obj) + 1);
     PyObject_GC_Track(ct);
     return ct;
 }
diff --git a/zeffir/test/test_ctype.py b/zeffir/test/test_ctype.py
--- a/zeffir/test/test_ctype.py
+++ b/zeffir/test/test_ctype.py
@@ -1,8 +1,18 @@
 import support
 
 
-def test_typeof():
+def test_typeof_primitive():
     ffi = support.new_ffi()
-    assert repr(ffi.typeof("int")) == "<ctype 'int'>"
-    assert repr(ffi.typeof("long int")) == "<ctype 'long'>"
-    #assert repr(ffi.typeof("int*")) == "<ctype 'int *'>"
+    ct_int = ffi.typeof("int")
+    assert repr(ct_int) == "<ctype 'int'>"
+    #
+    ct_long = ffi.typeof("long int")
+    assert repr(ct_long) == "<ctype 'long'>"
+    #
+    assert ffi.types == {'int': ct_int,
+                         'long': ct_long,
+                         'long int': ct_long}
+
+def test_typeof_pointer():
+    ffi = support.new_ffi()
+    assert repr(ffi.typeof("int*")) == "<ctype 'int *'>"

From noreply at buildbot.pypy.org  Wed Dec  3 18:00:39 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  3 Dec 2014 18:00:39 +0100 (CET)
Subject: [pypy-commit] pypy optresult: clean the escape mess and start
 passing first unrolling tests
Message-ID: <20141203170039.82C5C1C363D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74796:fe81bdbcbbaa
Date: 2014-12-03 18:59 +0200
http://bitbucket.org/pypy/pypy/changeset/fe81bdbcbbaa/

Log:	clean the escape mess and start passing first unrolling tests

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -13,7 +13,6 @@
     TargetToken, AbstractFailDescr, ConstInt)
 from rpython.jit.metainterp import history, jitexc
 from rpython.jit.metainterp.optimize import InvalidLoop
-from rpython.jit.metainterp.inliner import Inliner
 from rpython.jit.metainterp.resume import NUMBERING, PENDINGFIELDSP, ResumeDataDirectReader
 from rpython.jit.codewriter import heaptracker, longlong
 
@@ -104,9 +103,12 @@
 # ____________________________________________________________
 
 class Memo(object):
-    def __init__(self):
+    def __init__(self, initial_args=None, initial_res=None):
         self.snapshots = {}
         self.box_mapping = {}
+        if initial_args is not None:
+            for i in range(len(initial_args)):
+                self.box_mapping[initial_args[i]] = initial_res[i]
 
     def get(self, box, default):
         return self.box_mapping.get(box, default)
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -285,6 +285,9 @@
     def getfloatstorage(self):
         return self.value
 
+    def getfloat(self):
+        return longlong.getrealfloat(self.value)
+
     getvalue = getfloatstorage
 
     def _get_hash_(self):
diff --git a/rpython/jit/metainterp/inliner.py b/rpython/jit/metainterp/inliner.py
deleted file mode 100644
--- a/rpython/jit/metainterp/inliner.py
+++ /dev/null
@@ -1,57 +0,0 @@
-from rpython.jit.metainterp.history import Const
-from rpython.jit.metainterp.resume import Snapshot
-
-
-class Inliner(object):
-    def __init__(self, inputargs, jump_args):
-        assert len(inputargs) == len(jump_args)
-        self.argmap = {}
-        for i in range(len(inputargs)):
-            if inputargs[i] in self.argmap:
-                assert self.argmap[inputargs[i]] == jump_args[i]
-            else:
-                self.argmap[inputargs[i]] = jump_args[i]
-        self.snapshot_map = {None: None}
-
-    def inline_op(self, newop, ignore_result=False, clone=True,
-                  ignore_failargs=False):
-        if clone:
-            newop = newop.clone()
-        args = newop.getarglist()
-        newop.initarglist([self.inline_arg(a) for a in args])
-
-        if newop.is_guard():
-            args = newop.getfailargs()
-            if args and not ignore_failargs:
-                newop.setfailargs([self.inline_arg(a) for a in args])
-            else:
-                newop.setfailargs([])
-
-        if newop.result and not ignore_result:
-            old_result = newop.result
-            newop.result = newop.result.clonebox()
-            self.argmap[old_result] = newop.result
-
-        self.inline_descr_inplace(newop.getdescr())
-
-        return newop
-
-    def inline_descr_inplace(self, descr):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
-        if isinstance(descr, ResumeGuardDescr):
-            descr.rd_snapshot = self.inline_snapshot(descr.rd_snapshot)
-
-    def inline_arg(self, arg):
-        if arg is None:
-            return None
-        if isinstance(arg, Const):
-            return arg
-        return self.argmap[arg]
-
-    def inline_snapshot(self, snapshot):
-        if snapshot in self.snapshot_map:
-            return self.snapshot_map[snapshot]
-        boxes = [self.inline_arg(a) for a in snapshot.boxes]
-        new_snapshot = Snapshot(self.inline_snapshot(snapshot.prev), boxes)
-        self.snapshot_map[snapshot] = new_snapshot
-        return new_snapshot
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -579,11 +579,11 @@
         self.emit_operation(op)
 
     def optimize_CAST_PTR_TO_INT(self, op):
-        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.pure(rop.CAST_INT_TO_PTR, [op], op.getarg(0))
         self.emit_operation(op)
 
     def optimize_CAST_INT_TO_PTR(self, op):
-        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.pure(rop.CAST_PTR_TO_INT, [op], op.getarg(0))
         self.emit_operation(op)
 
     def optimize_SAME_AS_i(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -219,14 +219,14 @@
         ops = """
         [i0]
         p0 = new_with_vtable(ConstClass(node_vtable))
-        escape(p0)
+        escape_n(p0)
         guard_class(p0, ConstClass(node_vtable)) []
         jump(i0)
         """
         expected = """
         [i0]
         p0 = new_with_vtable(ConstClass(node_vtable))
-        escape(p0)
+        escape_n(p0)
         jump(i0)
         """
         self.optimize_loop(ops, expected)
@@ -315,19 +315,19 @@
     def test_remove_consecutive_guard_value_constfold(self):
         ops = """
         []
-        i0 = escape()
+        i0 = escape_i()
         guard_value(i0, 0) []
         i1 = int_add(i0, 1)
         guard_value(i1, 1) []
         i2 = int_add(i1, 2)
-        escape(i2)
+        escape_n(i2)
         jump()
         """
         expected = """
         []
-        i0 = escape()
+        i0 = escape_i()
         guard_value(i0, 0) []
-        escape(3)
+        escape_n(3)
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -428,14 +428,14 @@
     def test_ooisnull_on_null_ptr_1(self):
         ops = """
         []
-        p0 = escape()
+        p0 = escape_r()
         guard_isnull(p0) []
         guard_isnull(p0) []
         jump()
         """
         expected = """
         []
-        p0 = escape()
+        p0 = escape_r()
         guard_isnull(p0) []
         jump()
         """
@@ -517,7 +517,7 @@
     def test_constptr_guard_value(self):
         ops = """
         []
-        p1 = escape()
+        p1 = escape_r()
         guard_value(p1, ConstPtr(myptr)) []
         jump()
         """
@@ -592,7 +592,7 @@
         ops = """
         [i1, p2, p3]
         i3 = getfield_gc_i(p3, descr=valuedescr)
-        escape(i3)
+        escape_n(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
         jump(i1, p1, p2)
@@ -604,7 +604,7 @@
         ops = """
         [i1, p2, p3]
         i3 = getfield_gc_i(p3, descr=valuedescr)
-        escape(i3)
+        escape_n(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
         p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
@@ -621,7 +621,7 @@
         [i1, p2, p3]
         p3sub = getfield_gc_r(p3, descr=nextdescr)
         i3 = getfield_gc_i(p3sub, descr=valuedescr)
-        escape(i3)
+        escape_n(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
@@ -974,7 +974,7 @@
         f2 = getinteriorfield_gc_f(p0, 0, descr=complexrealdescr)
         f3 = getinteriorfield_gc_f(p0, 0, descr=compleximagdescr)
         f4 = float_mul(f2, f3)
-        i0 = escape(f4, p0)
+        i0 = escape_i(f4, p0)
         finish(i0)
         """
         expected = """
@@ -983,7 +983,7 @@
         p0 = new_array_clear(1, descr=complexarraydescr)
         setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
         setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
-        i0 = escape(f2, p0)
+        i0 = escape_i(f2, p0)
         finish(i0)
         """
         self.optimize_loop(ops, expected)
@@ -1008,8 +1008,8 @@
         setfield_gc(p1, i, descr=valuedescr)
         i0 = getfield_gc_i(p1, descr=valuedescr)
         i1 = int_add(i0, 1)
-        escape(p1)
-        escape(p1)
+        escape_n(p1)
+        escape_n(p1)
         jump(i1)
         """
         expected = """
@@ -1017,8 +1017,8 @@
         i1 = int_add(i, 1)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        escape(p1)
-        escape(p1)
+        escape_n(p1)
+        escape_n(p1)
         jump(i1)
         """
         self.optimize_loop(ops, expected)
@@ -1027,7 +1027,7 @@
         ops = """
         [i, p0]
         i0 = getfield_gc_i(p0, descr=valuedescr)
-        escape(p0)
+        escape_n(p0)
         i1 = int_add(i0, i)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
@@ -1042,7 +1042,7 @@
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
         i1 = getfield_gc_i(p1, descr=valuedescr)
-        escape(p1)
+        escape_n(p1)
         i2 = getfield_gc_i(p1, descr=valuedescr)
         i3 = int_add(i1, i2)
         jump(i3)
@@ -1051,7 +1051,7 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        escape(p1)
+        escape_n(p1)
         i2 = getfield_gc_i(p1, descr=valuedescr)
         i3 = int_add(i, i2)
         jump(i3)
@@ -1065,7 +1065,7 @@
         setfield_gc(p1, i, descr=valuedescr)
         i1 = getfield_gc_i(p1, descr=valuedescr)
         setfield_gc(p1, 0, descr=valuedescr)
-        escape(p1)
+        escape_n(p1)
         i2 = getfield_gc_i(p1, descr=valuedescr)
         jump(i2)
         """
@@ -1073,7 +1073,7 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, 0, descr=valuedescr)
-        escape(p1)
+        escape_n(p1)
         i2 = getfield_gc_i(p1, descr=valuedescr)
         jump(i2)
         """
@@ -1185,7 +1185,7 @@
         p1 = new_array(5, descr=arraydescr)
         setarrayitem_gc(p1, 0, i1, descr=arraydescr)
         setarrayitem_gc(p1, 1, 0, descr=arraydescr)
-        escape(p1)
+        escape_n(p1)
         jump(i1)
         """
         expected = """
@@ -1193,7 +1193,7 @@
         p1 = new_array(5, descr=arraydescr)
         setarrayitem_gc(p1, 0, i1, descr=arraydescr)
         setarrayitem_gc(p1, 1, 0, descr=arraydescr)
-        escape(p1)
+        escape_n(p1)
         jump(i1)
         """
         self.optimize_loop(ops, expected)
@@ -1223,7 +1223,7 @@
         ops = """
         [i1, p2, p3]
         i3 = getarrayitem_gc_i(p3, 0, descr=arraydescr)
-        escape(i3)
+        escape_n(i3)
         p1 = new_array(1, descr=arraydescr)
         setarrayitem_gc(p1, 0, i1, descr=arraydescr)
         jump(i1, p1, p2)
@@ -1238,17 +1238,17 @@
         setfield_gc(p2, 3, descr=valuedescr)
         i1 = getfield_gc_i(p2, descr=valuedescr)    # i1 = const 3
         p1 = new_array(i1, descr=arraydescr)
-        escape(p1)
+        escape_n(p1)
         i2 = arraylen_gc(p1)
-        escape(i2)
+        escape_n(i2)
         jump()
         """
         expected = """
         []
         p1 = new_array(3, descr=arraydescr)
-        escape(p1)
+        escape_n(p1)
         i2 = arraylen_gc(p1)
-        escape(i2)
+        escape_n(i2)
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -1257,14 +1257,14 @@
         ops = """
         [i1, p2]
         i2 = getfield_gc(p2, descr=adescr)
-        escape(i2)
+        escape_n(i2)
         p3 = new(descr=ssize)
         setfield_gc(p3, i1, descr=adescr)
         jump(i1, p3)
         """
         expected = """
         [i1, i2]
-        escape(i2)
+        escape_n(i2)
         jump(i1, i1)
         """
         py.test.skip("XXX")
@@ -1274,7 +1274,7 @@
         ops = """
         [i1, p2, p3]
         i3 = getfield_gc_i(p3, descr=adescr)
-        escape(i3)
+        escape_n(i3)
         p1 = new(descr=ssize)
         setfield_gc(p1, i1, descr=adescr)
         jump(i1, p1, p2)
@@ -1289,20 +1289,20 @@
         i2 = getfield_gc_i(p2, descr=valuedescr)
         i3 = getfield_gc_i(p1, descr=valuedescr)
         i4 = getfield_gc_i(p2, descr=valuedescr)
-        escape(i1)
-        escape(i2)
-        escape(i3)
-        escape(i4)
+        escape_n(i1)
+        escape_n(i2)
+        escape_n(i3)
+        escape_n(i4)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
         i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = getfield_gc_i(p2, descr=valuedescr)
-        escape(i1)
-        escape(i2)
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
+        escape_n(i1)
+        escape_n(i2)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -1312,13 +1312,13 @@
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
         i2 = getfield_gc_i(p1, descr=valuedescr)
-        escape(i2)
+        escape_n(i2)
         jump(p1, i1)
         """
         expected = """
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
-        escape(i1)
+        escape_n(i1)
         jump(p1, i1)
         """
         self.optimize_loop(ops, expected)
@@ -1329,14 +1329,14 @@
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, p1, descr=nextdescr)
         i2 = getfield_gc_i(p1, descr=valuedescr)
-        escape(i2)
+        escape_n(i2)
         jump(p1, p2, i1)
         """
         expected = """
         [p1, p2, i1]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, p1, descr=nextdescr)
-        escape(i1)
+        escape_n(i1)
         jump(p1, p2, i1)
         """
         self.optimize_loop(ops, expected)
@@ -1347,7 +1347,7 @@
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, i2, descr=valuedescr)
         i3 = getfield_gc_i(p1, descr=valuedescr)
-        escape(i3)
+        escape_n(i3)
         jump(p1, p2, i1, i3)
         """
         self.optimize_loop(ops, ops)
@@ -1358,16 +1358,16 @@
         i1 = getfield_gc_i(p1, descr=valuedescr)
         debug_merge_point(15, 0)
         i2 = getfield_gc_i(p1, descr=valuedescr)
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
         jump(p1)
         """
         expected = """
         [p1]
         i1 = getfield_gc_i(p1, descr=valuedescr)
         debug_merge_point(15, 0)
-        escape(i1)
-        escape(i1)
+        escape_n(i1)
+        escape_n(i1)
         jump(p1)
         """
         self.optimize_loop(ops, expected)
@@ -1379,8 +1379,8 @@
         i2 = int_add_ovf(i1, 14)
         guard_no_overflow() []
         i3 = getfield_gc_i(p1, descr=valuedescr)
-        escape(i2)
-        escape(i3)
+        escape_n(i2)
+        escape_n(i3)
         jump(p1)
         """
         expected = """
@@ -1388,8 +1388,8 @@
         i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = int_add_ovf(i1, 14)
         guard_no_overflow() []
-        escape(i2)
-        escape(i1)
+        escape_n(i2)
+        escape_n(i1)
         jump(p1)
         """
         self.optimize_loop(ops, expected)
@@ -1400,16 +1400,16 @@
         i1 = getfield_gc_i(p1, descr=valuedescr)
         setarrayitem_gc(p2, 0, p1, descr=arraydescr2)
         i3 = getfield_gc_i(p1, descr=valuedescr)
-        escape(i1)
-        escape(i3)
+        escape_n(i1)
+        escape_n(i3)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
         i1 = getfield_gc_i(p1, descr=valuedescr)
         setarrayitem_gc(p2, 0, p1, descr=arraydescr2)
-        escape(i1)
-        escape(i1)
+        escape_n(i1)
+        escape_n(i1)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -1419,15 +1419,15 @@
         []
         i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
         i2 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
         jump()
         """
         expected = """
         []
         i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
-        escape(i1)
-        escape(i1)
+        escape_n(i1)
+        escape_n(i1)
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -1438,15 +1438,15 @@
         guard_value(p1, ConstPtr(myptr)) []
         i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
         jump(p1)
         """
         expected = """
         []
         i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
-        escape(i1)
-        escape(i1)
+        escape_n(i1)
+        escape_n(i1)
         jump()
         """
         py.test.skip("XXX")
@@ -1456,10 +1456,10 @@
         ops = """
         [p1]
         i1 = getfield_gc_i(p1, descr=valuedescr)
-        escape()
+        escape_n()
         i2 = getfield_gc_i(p1, descr=valuedescr)
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
         jump(p1)
         """
         self.optimize_loop(ops, ops)
@@ -1468,9 +1468,9 @@
         ops = """
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
-        escape()
+        escape_n()
         i2 = getfield_gc_i(p1, descr=valuedescr)
-        escape(i2)
+        escape_n(i2)
         jump(p1, i1)
         """
         self.optimize_loop(ops, ops)
@@ -1495,13 +1495,13 @@
         setfield_gc(p1, i1, descr=valuedescr)
         i2 = getfield_gc_i(p1, descr=valuedescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        escape(i2)
+        escape_n(i2)
         jump(p1, i1, i3)
         """
         expected = """
         [p1, i1, i3]
         setfield_gc(p1, i3, descr=valuedescr)
-        escape(i1)
+        escape_n(i1)
         jump(p1, i1, i3)
         """
         self.optimize_loop(ops, expected)
@@ -1512,7 +1512,7 @@
         setfield_gc(p1, i1, descr=valuedescr)
         i2 = getfield_gc_i(p2, descr=valuedescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        escape(i2)
+        escape_n(i2)
         jump(p1, p2, i1, i3)
         """
         # potential aliasing of p1 and p2 means that we cannot kill the
@@ -1558,14 +1558,14 @@
         p2 = getfield_gc_r(p0, descr=nextdescr)
         i2 = getfield_gc_i(p2, descr=valuedescr)
         setfield_gc(p0, NULL, descr=nextdescr)
-        escape(i2)
+        escape_n(i2)
         jump(p0, i1)
         """
         expected = """
         [p0, i1]
         setfield_raw(i1, i1, descr=valuedescr)
         setfield_gc(p0, NULL, descr=nextdescr)
-        escape(i1)
+        escape_n(i1)
         jump(p0, i1)
         """
         self.optimize_loop(ops, expected)
@@ -1574,7 +1574,7 @@
         ops = """
         [p1, i1, i2]
         setfield_gc(p1, i1, descr=valuedescr)
-        escape()
+        escape_n()
         setfield_gc(p1, i2, descr=valuedescr)
         jump(p1, i1, i2)
         """
@@ -1691,20 +1691,20 @@
         p3 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
         p4 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         p5 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
-        escape(p2)
-        escape(p3)
-        escape(p4)
-        escape(p5)
+        escape_n(p2)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(p5)
         jump(p1)
         """
         expected = """
         [p1]
         p2 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         p3 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
-        escape(p2)
-        escape(p3)
-        escape(p2)
-        escape(p3)
+        escape_n(p2)
+        escape_n(p3)
+        escape_n(p2)
+        escape_n(p3)
         jump(p1)
         """
         self.optimize_loop(ops, expected)
@@ -1714,13 +1714,13 @@
         [p1, p2]
         setarrayitem_gc(p1, 0, p2, descr=arraydescr2)
         p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
-        escape(p3)
+        escape_n(p3)
         jump(p1, p3)
         """
         expected = """
         [p1, p2]
         setarrayitem_gc(p1, 0, p2, descr=arraydescr2)
-        escape(p2)
+        escape_n(p2)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -1733,8 +1733,8 @@
         setarrayitem_gc(p1, i1, p3, descr=arraydescr2)
         p4 = getarrayitem_gc(p1, 0, descr=arraydescr2)
         p5 = getarrayitem_gc(p1, i1, descr=arraydescr2)
-        escape(p4)
-        escape(p5)
+        escape_n(p4)
+        escape_n(p5)
         jump(p1, p2, p3, i1)
         """
         expected = """
@@ -1742,8 +1742,8 @@
         setarrayitem_gc(p1, 0, p2, descr=arraydescr2)
         setarrayitem_gc(p1, i1, p3, descr=arraydescr2)
         p4 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        escape(p4)
-        escape(p3)
+        escape_n(p4)
+        escape_n(p3)
         jump(p1, p2, p3, i1)
         """
         self.optimize_loop(ops, expected)
@@ -1757,9 +1757,9 @@
         p5 = getarrayitem_gc_r(p1, i1, descr=arraydescr2)
         p6 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         p7 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
-        escape(p5)
-        escape(p6)
-        escape(p7)
+        escape_n(p5)
+        escape_n(p6)
+        escape_n(p7)
         jump(p1, p2, p3, p4, i1)
         """
         expected = """
@@ -1768,9 +1768,9 @@
         setarrayitem_gc(p1, 0, p3, descr=arraydescr2)
         setarrayitem_gc(p1, 1, p4, descr=arraydescr2)
         p5 = getarrayitem_gc_r(p1, i1, descr=arraydescr2)
-        escape(p5)
-        escape(p3)
-        escape(p4)
+        escape_n(p5)
+        escape_n(p3)
+        escape_n(p4)
         jump(p1, p2, p3, p4, i1)
         """
         self.optimize_loop(ops, expected)
@@ -1781,17 +1781,17 @@
         p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         i4 = getfield_gc_pure_i(ConstPtr(myptr), descr=valuedescr)
         p5 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
-        escape(p3)
-        escape(i4)
-        escape(p5)
+        escape_n(p3)
+        escape_n(i4)
+        escape_n(p5)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
         p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
-        escape(p3)
-        escape(5)
-        escape(p3)
+        escape_n(p3)
+        escape_n(5)
+        escape_n(p3)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -1803,16 +1803,16 @@
         setarrayitem_gc(p2, 1, p4, descr=arraydescr2)
         p5 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         p6 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
-        escape(p5)
-        escape(p6)
+        escape_n(p5)
+        escape_n(p6)
         jump(p1, p2, p3, p4, i1)
         """
         expected = """
         [p1, p2, p3, p4, i1]
         setarrayitem_gc(p1, 0, p3, descr=arraydescr2)
         setarrayitem_gc(p2, 1, p4, descr=arraydescr2)
-        escape(p3)
-        escape(p4)
+        escape_n(p3)
+        escape_n(p4)
         jump(p1, p2, p3, p4, i1)
         """
         self.optimize_loop(ops, expected)
@@ -1864,19 +1864,19 @@
         [i0, p1]
         p4 = getfield_gc_r(p1, descr=nextdescr)
         guard_nonnull(p4) []
-        escape(p4)
+        escape_n(p4)
         #
         p2 = new_with_vtable(ConstClass(node_vtable))
-        p3 = escape()
+        p3 = escape_r()
         setfield_gc(p2, p3, descr=nextdescr)
         jump(i0, p2)
         """
         expected = """
         [i0, p4]
         guard_nonnull(p4) []
-        escape(p4)
+        escape_n(p4)
         #
-        p3 = escape()
+        p3 = escape_r()
         jump(i0, p3)
         """
         py.test.skip("XXX")
@@ -1888,19 +1888,19 @@
         [i0, p1]
         p4 = getarrayitem_gc(p1, 0, descr=arraydescr2)
         guard_nonnull(p4) []
-        escape(p4)
+        escape_n(p4)
         #
         p2 = new_array(1, descr=arraydescr2)
-        p3 = escape()
+        p3 = escape_r()
         setarrayitem_gc(p2, 0, p3, descr=arraydescr2)
         jump(i0, p2)
         """
         expected = """
         [i0, p4]
         guard_nonnull(p4) []
-        escape(p4)
+        escape_n(p4)
         #
-        p3 = escape()
+        p3 = escape_r()
         jump(i0, p3)
         """
         py.test.skip("XXX")
@@ -1922,7 +1922,7 @@
         p1a = new_with_vtable(ConstClass(node_vtable2))
         p2a = new_with_vtable(ConstClass(node_vtable))
         p3a = new_with_vtable(ConstClass(node_vtable))
-        escape(p3a)
+        escape_n(p3a)
         setfield_gc(p1a, p2a, descr=nextdescr)
         setfield_gc(p1a, p3a, descr=otherdescr)
         jump(p1a)
@@ -1933,7 +1933,7 @@
         guard_class(p3, ConstClass(node_vtable)) []
         setfield_gc(p3, p2, descr=otherdescr)
         p3a = new_with_vtable(ConstClass(node_vtable))
-        escape(p3a)
+        escape_n(p3a)
         p2a = new_with_vtable(ConstClass(node_vtable))
         jump(p2a, p3a)
         """
@@ -1955,7 +1955,7 @@
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
         p3a = new_with_vtable(ConstClass(node_vtable))
-        escape(p3a)
+        escape_n(p3a)
         setfield_gc(p1a, p2a, descr=nextdescr)
         setfield_gc(p1a, p3a, descr=otherdescr)
         jump(p1a)
@@ -1967,7 +1967,7 @@
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
         p3a = new_with_vtable(ConstClass(node_vtable))
-        escape(p3a)
+        escape_n(p3a)
         jump(p2a, p3a)
         """
         py.test.skip("XXX")
@@ -1992,7 +1992,7 @@
         guard_class(p1, ConstClass(node_vtable2)) []
         #
         p2 = new_with_vtable(ConstClass(node_vtable))
-        escape(p2)      # prevent it from staying Virtual
+        escape_n(p2)      # prevent it from staying Virtual
         jump(p2)
         """
         py.test.raises(InvalidLoop, self.optimize_loop,
@@ -2128,8 +2128,8 @@
         i4 = int_add(i2, 1)
         i4b = int_is_true(i4)
         guard_true(i4b) []
-        escape(i3)
-        escape(i4)
+        escape_n(i3)
+        escape_n(i4)
         guard_true(i1) []
         guard_true(i2) []
         jump(p1, p2)
@@ -2140,8 +2140,8 @@
         i3 = int_add(i1, 1)
         i3b = int_is_true(i3)
         guard_true(i3b) []
-        escape(i3)
-        escape(i3)
+        escape_n(i3)
+        escape_n(i3)
         guard_true(i1) []
         jump(p1, p2)
         """
@@ -2178,8 +2178,8 @@
         guard_no_overflow() []
         i4b = int_is_true(i4)
         guard_true(i4b) []
-        escape(i3)
-        escape(i4)
+        escape_n(i3)
+        escape_n(i4)
         jump(i1)
         """
         expected = """
@@ -2188,8 +2188,8 @@
         guard_no_overflow() []
         i3b = int_is_true(i3)
         guard_true(i3b) []
-        escape(i3)
-        escape(i3)
+        escape_n(i3)
+        escape_n(i3)
         jump(i1)
         """
         self.optimize_loop(ops, expected)
@@ -2295,12 +2295,12 @@
         ops = """
         [f0]
         f1 = float_mul(f0, 1.0)
-        f2 = escape(f1)
+        f2 = escape_f(f1)
         jump(f2)
         """
         expected = """
         [f0]
-        f2 = escape(f0)
+        f2 = escape_f(f0)
         jump(f2)
         """
         self.optimize_loop(ops, expected)
@@ -2308,12 +2308,12 @@
         ops = """
         [f0]
         f1 = float_mul(1.0, f0)
-        f2 = escape(f1)
+        f2 = escape_f(f1)
         jump(f2)
         """
         expected = """
         [f0]
-        f2 = escape(f0)
+        f2 = escape_f(f0)
         jump(f2)
         """
         self.optimize_loop(ops, expected)
@@ -2322,13 +2322,13 @@
         ops = """
         [f0]
         f1 = float_mul(f0, -1.0)
-        f2 = escape(f1)
+        f2 = escape_f(f1)
         jump(f2)
         """
         expected = """
         [f0]
         f1 = float_neg(f0)
-        f2 = escape(f1)
+        f2 = escape_f(f1)
         jump(f2)
         """
         self.optimize_loop(ops, expected)
@@ -2336,13 +2336,13 @@
         ops = """
         [f0]
         f1 = float_mul(-1.0, f0)
-        f2 = escape(f1)
+        f2 = escape_f(f1)
         jump(f2)
         """
         expected = """
         [f0]
         f1 = float_neg(f0)
-        f2 = escape(f1)
+        f2 = escape_f(f1)
         jump(f2)
         """
         self.optimize_loop(ops, expected)
@@ -2354,14 +2354,14 @@
         f2 = float_neg(f1)
         f3 = float_neg(f2)
         f4 = float_neg(f3)
-        escape(f4)
+        escape_n(f4)
         jump(f4)
         """
         expected = """
         [f0]
         # The backend removes this dead op.
         f1 = float_neg(f0)
-        escape(f0)
+        escape_n(f0)
         jump(f0)
         """
         self.optimize_loop(ops, expected)
@@ -2373,7 +2373,7 @@
         f2 = float_truediv(f1, 3.0)
         f3 = float_truediv(f2, -0.25)
         f4 = float_truediv(f3, 0.0)
-        f5 = escape(f4)
+        f5 = escape_f(f4)
         jump(f5)
         """
 
@@ -2383,7 +2383,7 @@
         f2 = float_truediv(f1, 3.0)
         f3 = float_mul(f2, -4.0)
         f4 = float_truediv(f3, 0.0)
-        f5 = escape(f4)
+        f5 = escape_f(f4)
         jump(f5)
         """
         self.optimize_loop(ops, expected)
@@ -2770,16 +2770,16 @@
         i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = call_i(i1, descr=nonwritedescr)
         i3 = getfield_gc_i(p1, descr=valuedescr)
-        escape(i1)
-        escape(i3)
+        escape_n(i1)
+        escape_n(i3)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
         i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = call_i(i1, descr=nonwritedescr)
-        escape(i1)
-        escape(i1)
+        escape_n(i1)
+        escape_n(i1)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -2792,10 +2792,10 @@
         i3 = call_i(i1, descr=writeadescr)
         i4 = getfield_gc_i(p1, descr=adescr)
         i5 = getfield_gc_i(p1, descr=bdescr)
-        escape(i1)
-        escape(i2)
-        escape(i4)
-        escape(i5)
+        escape_n(i1)
+        escape_n(i2)
+        escape_n(i4)
+        escape_n(i5)
         jump(p1, p2)
         """
         expected = """
@@ -2804,10 +2804,10 @@
         i2 = getfield_gc_i(p1, descr=bdescr)
         i3 = call_i(i1, descr=writeadescr)
         i4 = getfield_gc_i(p1, descr=adescr)
-        escape(i1)
-        escape(i2)
-        escape(i4)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
+        escape_n(i4)
+        escape_n(i2)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -2820,10 +2820,10 @@
         i3 = call_i(i1, descr=writeadescr)
         p5 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         p6 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
-        escape(p3)
-        escape(p4)
-        escape(p5)
-        escape(p6)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(p5)
+        escape_n(p6)
         jump(p1, p2, i1)
         """
         expected = """
@@ -2831,10 +2831,10 @@
         p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         p4 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
         i3 = call_i(i1, descr=writeadescr)
-        escape(p3)
-        escape(p4)
-        escape(p3)
-        escape(p4)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(p3)
+        escape_n(p4)
         jump(p1, p2, i1)
         """
         self.optimize_loop(ops, expected)
@@ -2849,12 +2849,12 @@
         p5 = getarrayitem_gc_r(p2, 0, descr=arraydescr2)
         p6 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
         i4 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
-        escape(p3)
-        escape(p4)
-        escape(p5)
-        escape(p6)
-        escape(i2)
-        escape(i4)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(p5)
+        escape_n(p6)
+        escape_n(i2)
+        escape_n(i4)
         jump(p1, p2, i1)
         """
         expected = """
@@ -2864,12 +2864,12 @@
         i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         i3 = call_i(i1, descr=writearraydescr)
         i4 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
-        escape(p3)
-        escape(p4)
-        escape(p3)
-        escape(p4)
-        escape(i2)
-        escape(i4)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(i2)
+        escape_n(i4)
         jump(p1, p2, i1)
         """
         self.optimize_loop(ops, expected)
@@ -2964,16 +2964,16 @@
         call_pure_results = {tuple(arg_consts): ConstInt(42)}
         ops = '''
         [i0, i1, i2]
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
         i3 = call_pure_i(123456, 4, 5, 6, descr=plaincalldescr)
         i4 = call_pure_i(123456, 4, i0, 6, descr=plaincalldescr)
         jump(i0, i3, i4)
         '''
         expected = '''
         [i0, i1, i2]
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
         i4 = call_i(123456, 4, i0, 6, descr=plaincalldescr)
         jump(i0, 42, i4)
         '''
@@ -2997,7 +2997,7 @@
         ops = """
         [p1]
         p2 = virtual_ref(p1, 5)
-        escape(p2)
+        escape_n(p2)
         virtual_ref_finish(p2, p1)
         jump(p1)
         """
@@ -3007,7 +3007,7 @@
         p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable))
         setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p2, p0, descr=virtualtokendescr)
-        escape(p2)
+        escape_n(p2)
         setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         jump(p1)
@@ -3146,7 +3146,7 @@
         [i1]
         p1 = new_with_vtable(ConstClass(node_vtable))
         p2 = virtual_ref(p1, 7)
-        escape(p2)
+        escape_n(p2)
         virtual_ref_finish(p2, p1)
         call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() []
@@ -3158,7 +3158,7 @@
         p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable))
         setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p2, p3, descr=virtualtokendescr)
-        escape(p2)
+        escape_n(p2)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p2, NULL, descr=virtualtokendescr)
@@ -3172,7 +3172,7 @@
         ops = """
         [i1, p1]
         p2 = virtual_ref(p1, 23)
-        escape(p2)
+        escape_n(p2)
         virtual_ref_finish(p2, p1)
         call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
@@ -3184,7 +3184,7 @@
         p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable))
         setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p2, p3, descr=virtualtokendescr)
-        escape(p2)
+        escape_n(p2)
         setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         call_may_force_n(i1, descr=mayforcevirtdescr)
@@ -4235,12 +4235,12 @@
         p1 = same_as_r(s"ab")
         p2 = same_as_r(s"cde")
         p3 = call_r(0, p1, p2, descr=strconcatdescr)
-        escape(p3)
+        escape_n(p3)
         jump()
         """
         expected = """
         []
-        escape(s"abcde")
+        escape_n(s"abcde")
         jump()
         """
         self.optimize_strunicode_loop(ops, expected)
@@ -4329,7 +4329,7 @@
         [p1, i1, i2, i3]
         p2 = call_r(0, p1, i1, i2, descr=strslicedescr)
         i4 = strgetitem(p2, i3)
-        escape(i4)
+        escape_n(i4)
         jump(p1, i1, i2, i3)
         """
         expected = """
@@ -4337,7 +4337,7 @@
         i6 = int_sub(i2, i1)      # killed by the backend
         i5 = int_add(i1, i3)
         i4 = strgetitem(p1, i5)
-        escape(i4)
+        escape_n(i4)
         jump(p1, i1, i2, i3)
         """
         self.optimize_strunicode_loop(ops, expected)
@@ -4350,12 +4350,12 @@
         strsetitem(p1, 1, i4)
         p2 = call_r(0, p1, 1, 2, descr=strslicedescr)
         i5 = strgetitem(p2, 0)
-        escape(i5)
+        escape_n(i5)
         jump(i3, i4)
         """
         expected = """
         [i3, i4]
-        escape(i4)
+        escape_n(i4)
         jump(i3, i4)
         """
         self.optimize_strunicode_loop(ops, expected)
@@ -4416,7 +4416,7 @@
         ops = """
         [p1, p2]
         i0 = call_i(0, p1, p2, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, p2)
         """
         self.optimize_strunicode_loop_extradescrs(ops, ops)
@@ -4426,7 +4426,7 @@
         [p1, p2, p3]
         p4 = call_r(0, p1, p2, descr=strconcatdescr)
         i0 = call_i(0, p3, p4, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, p2, p3)
         """
         expected = """
@@ -4438,7 +4438,7 @@
         copystrcontent(p1, p4, 0, 0, i1)
         copystrcontent(p2, p4, 0, i1, i2)
         i0 = call_i(0, p3, p4, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, p2, p3)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4448,14 +4448,14 @@
         [p1, i1, i2, p3]
         p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
         i0 = call_i(0, p4, p3, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i1, i2, p3)
         """
         expected = """
         [p1, i1, i2, p3]
         i3 = int_sub(i2, i1)
         i0 = call_i(0, p1, i1, i3, p3, descr=streq_slice_checknull_descr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i1, i2, p3)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4465,14 +4465,14 @@
         [p1, i1, i2, p3]
         p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
         i0 = call_i(0, p3, p4, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i1, i2, p3)
         """
         expected = """
         [p1, i1, i2, p3]
         i4 = int_sub(i2, i1)
         i0 = call_i(0, p1, i1, i4, p3, descr=streq_slice_checknull_descr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i1, i2, p3)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4483,7 +4483,7 @@
         guard_nonnull(p3) []
         p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
         i0 = call_i(0, p3, p4, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i1, i2, p3)
         """
         expected = """
@@ -4491,7 +4491,7 @@
         guard_nonnull(p3) []
         i4 = int_sub(i2, i1)
         i0 = call_i(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i1, i2, p3)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4501,14 +4501,14 @@
         [p1, i1, i2]
         p3 = call_r(0, p1, i1, i2, descr=strslicedescr)
         i0 = call_i(0, p3, s"x", descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i1, i2)
         """
         expected = """
         [p1, i1, i2]
         i3 = int_sub(i2, i1)
         i0 = call_i(0, p1, i1, i3, 120, descr=streq_slice_char_descr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i1, i2)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4520,14 +4520,14 @@
         p5 = newstr(1)
         strsetitem(p5, 0, i3)
         i0 = call_i(0, p5, p4, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i1, i2, i3)
         """
         expected = """
         [p1, i1, i2, i3]
         i4 = int_sub(i2, i1)
         i0 = call_i(0, p1, i1, i4, i3, descr=streq_slice_char_descr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i1, i2, i3)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4536,13 +4536,13 @@
         ops = """
         [p1]
         i0 = call_i(0, p1, NULL, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         expected = """
         [p1]
         i0 = ptr_eq(p1, NULL)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4551,13 +4551,13 @@
         ops = """
         [p1]
         i0 = call_i(0, NULL, p1, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         expected = """
         [p1]
         i0 = ptr_eq(p1, NULL)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4567,14 +4567,14 @@
         [p1]
         guard_nonnull(p1) []
         i0 = call_i(0, p1, s"hello world", descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         expected = """
         [p1]
         guard_nonnull(p1) []
         i0 = call_i(0, p1, s"hello world", descr=streq_nonnull_descr)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4584,7 +4584,7 @@
         [p1]
         guard_nonnull(p1) []
         i0 = call_i(0, p1, s"", descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         expected = """
@@ -4592,7 +4592,7 @@
         guard_nonnull(p1) []
         i1 = strlen(p1)
         i0 = int_eq(i1, 0)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4602,14 +4602,14 @@
         [p1]
         guard_nonnull(p1) []
         i0 = call_i(0, p1, s"x", descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         expected = """
         [p1]
         guard_nonnull(p1) []
         i0 = call_i(0, p1, 120, descr=streq_nonnull_char_descr)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4619,7 +4619,7 @@
         [p1, p2]
         p4 = call_r(0, p1, p2, descr=strconcatdescr)
         i0 = call_i(0, s"hello world", p4, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, p2)
         """
         expected = """
@@ -4631,7 +4631,7 @@
         copystrcontent(p1, p4, 0, 0, i1)
         copystrcontent(p2, p4, 0, i1, i2)
         i0 = call_i(0, s"hello world", p4, descr=streq_nonnull_descr)
-        escape(i0)
+        escape_n(i0)
         jump(p1, p2)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4641,12 +4641,12 @@
         [i1]
         p1 = newstr(0)
         i0 = call_i(0, p1, s"", descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(i1)
         """
         expected = """
         [i1]
-        escape(1)
+        escape_n(1)
         jump(i1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4657,13 +4657,13 @@
         p1 = newstr(1)
         strsetitem(p1, 0, i1)
         i0 = call_i(0, p1, s"x", descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(i1)
         """
         expected = """
         [i1]
         i0 = int_eq(i1, 120)     # ord('x')
-        escape(i0)
+        escape_n(i0)
         jump(i1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4675,7 +4675,7 @@
         strsetitem(p1, 0, i1)
         strsetitem(p1, 1, i2)
         i0 = call_i(0, p1, s"xy", descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(i1, i2)
         """
         expected = """
@@ -4684,7 +4684,7 @@
         strsetitem(p1, 0, i1)
         strsetitem(p1, 1, i2)
         i0 = call_i(0, p1, s"xy", descr=streq_lengthok_descr)
-        escape(i0)
+        escape_n(i0)
         jump(i1, i2)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4693,13 +4693,13 @@
         ops = """
         [p1]
         i0 = call_i(0, s"x", p1, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         expected = """
         [p1]
         i0 = call_i(0, p1, 120, descr=streq_checknull_char_descr)
-        escape(i0)
+        escape_n(i0)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4710,12 +4710,12 @@
         p1 = newstr(1)
         strsetitem(p1, 0, i1)
         i0 = call_i(0, s"xy", p1, descr=strequaldescr)
-        escape(i0)
+        escape_n(i0)
         jump(i1)
         """
         expected = """
         [i1]
-        escape(0)
+        escape_n(0)
         jump(i1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4724,12 +4724,12 @@
         ops = """
         []
         p0 = call_r(0, "xy", descr=s2u_descr)      # string -> unicode
-        escape(p0)
+        escape_n(p0)
         jump()
         """
         expected = """
         []
-        escape(u"xy")
+        escape_n(u"xy")
         jump()
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected)
@@ -4738,7 +4738,7 @@
         ops = """
         [p0]
         p1 = call_r(0, p0, descr=s2u_descr)      # string -> unicode
-        escape(p1)
+        escape_n(p1)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, ops)
@@ -4753,13 +4753,13 @@
         i2 = strgetitem(p0, i0)
         i3 = int_eq(i1, i2)
         guard_true(i3) []
-        escape(i2)
+        escape_n(i2)
         jump(p0, i0)
         """
         expected = """
         [p0, i0]
         i1 = strgetitem(p0, i0)
-        escape(i1)
+        escape_n(i1)
         jump(p0, i0)
         """
         self.optimize_strunicode_loop(ops, expected)
@@ -4788,7 +4788,7 @@
         [p0, i0]
         i1 = int_add(i0, 1)
         p1 = call_r(0, p0, i0, i1, descr=strslicedescr)
-        escape(p1)
+        escape_n(p1)
         jump(p0, i1)
         """
         expected = """
@@ -4797,7 +4797,7 @@
         p1 = newstr(1)
         i2 = strgetitem(p0, i0)
         strsetitem(p1, 0, i2)
-        escape(p1)
+        escape_n(p1)
         jump(p0, i1)
         """
         self.optimize_strunicode_loop(ops, expected)
@@ -4841,7 +4841,7 @@
         ops = """
         [i0]
         p0 = newstr(i0)
-        escape(p0)
+        escape_n(p0)
         i1 = strlen(p0)
         i2 = int_add(i1, 1)
         jump(i2)
@@ -4849,7 +4849,7 @@
         expected = """
         [i0]
         p0 = newstr(i0)
-        escape(p0)
+        escape_n(p0)
         i1 = int_add(i0, 1)
         jump(i1)
         """
@@ -5009,8 +5009,8 @@
         [i0, i1]
         p0 = new(descr=ssize)
         p1 = new(descr=ssize)
-        escape(p0)
-        escape(p1)
+        escape_n(p0)
+        escape_n(p1)
         setfield_gc(p0, i0, descr=adescr)
         setfield_gc(p1, i1, descr=adescr)
         i2 = getfield_gc_i(p0, descr=adescr)
@@ -5019,9 +5019,9 @@
         expected = """
         [i0, i1]
         p0 = new(descr=ssize)
-        escape(p0)
+        escape_n(p0)
         p1 = new(descr=ssize)
-        escape(p1)
+        escape_n(p1)
         setfield_gc(p0, i0, descr=adescr)
         setfield_gc(p1, i1, descr=adescr)
         jump(i0, i0)
@@ -5192,14 +5192,14 @@
         p1 = newstr(1)
         strsetitem(p1, 0, i0)
         p2 = newstr(1)
-        escape(p2)
+        escape_n(p2)
         copystrcontent(p1, p2, 0, 0, 1)
         finish()
         """
         expected = """
         [i0]
         p2 = newstr(1)
-        escape(p2)
+        escape_n(p2)
         strsetitem(p2, 0, i0)
         finish()
         """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1,14 +1,15 @@
 import py
 from rpython.rlib.objectmodel import instantiate
 from rpython.jit.metainterp import compile, resume
-from rpython.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, TreeLoop
+from rpython.jit.metainterp.history import AbstractDescr, ConstInt, TreeLoop
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import build_opt_chain
 from rpython.jit.metainterp.optimizeopt.test.test_util import (
     LLtypeMixin, BaseTest, convert_old_style_to_targets)
 from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import \
     FakeMetaInterpStaticData
-from rpython.jit.metainterp.resoperation import rop, opname, oparity
+from rpython.jit.metainterp.resoperation import rop, opname, oparity,\
+     InputArgInt
 
 
 def test_build_opt_chain():
@@ -134,10 +135,10 @@
     def test_simple(self):
         ops = """
         []
-        f = escape()
+        f = escape_f()
         f0 = float_sub(f, 1.0)
         guard_value(f0, 0.0) [f0]
-        escape(f)
+        escape_n(f)
         jump()
         """
         self.optimize_loop(ops, ops)
@@ -178,9 +179,9 @@
         self.optimize_loop(ops, expected)
 
     def test_constfold_all(self):
-        from rpython.jit.metainterp.executor import execute_nonspec
+        from rpython.jit.metainterp.executor import _execute_nonspec
         import random
-        for opnum in range(rop.INT_ADD, rop.SAME_AS+1):
+        for opnum in range(rop.INT_ADD, rop.SAME_AS_I+1):
             try:
                 op = opname[opnum]
             except KeyError:
@@ -193,15 +194,14 @@
             ops = """
             []
             i1 = %s(%s)
-            escape(i1)
+            escape_n(i1)
             jump()
             """ % (op.lower(), ', '.join(map(str, args)))
-            argboxes = [BoxInt(a) for a in args]
-            expected_value = execute_nonspec(self.cpu, None, opnum,
-                                             argboxes).getint()
+            argboxes = [InputArgInt(a) for a in args]
+            expected_value = _execute_nonspec(self.cpu, None, opnum, argboxes)
             expected = """
             []
-            escape(%d)
+            escape_n(%d)
             jump()
             """ % expected_value
             self.optimize_loop(ops, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -18,7 +18,7 @@
 from rpython.jit.metainterp.counter import DeterministicJitCounter
 from rpython.config.translationoption import get_combined_translation_config
 from rpython.jit.metainterp.resoperation import rop, ResOperation, InputArgRef
-from rpython.jit.metainterp.optimizeopt.unroll import Inliner
+
 
 def test_sort_descrs():
     class PseudoDescr(AbstractDescr):
@@ -343,9 +343,9 @@
         op.setfailargs(boxes)
     def __eq__(self, other):
         return type(self) is type(other)      # xxx obscure
-    def clone_if_mutable(self):
+    def clone_if_mutable(self, memo):
         res = Storage(self.metainterp_sd, self.original_greenkey)
-        self.copy_all_attributes_into(res)
+        self.copy_all_attributes_into(res, memo)
         return res
 
 def _sortboxes(boxes):
@@ -367,7 +367,7 @@
     def add_guard_future_condition(self, res):
         # invent a GUARD_FUTURE_CONDITION to not have to change all tests
         if res.operations[-1].getopnum() == rop.JUMP:
-            guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None, descr=self.invent_fail_descr(None, -1, []))
+            guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], descr=self.invent_fail_descr(None, -1, []))
             res.operations.insert(-1, guard)
 
     def invent_fail_descr(self, model, opnum, fail_args):
@@ -414,7 +414,10 @@
 
         jump_args = jumpop.getarglist()[:]
         operations = operations[:-1]
-        cloned_operations = [op.clone() for op in operations]
+        memo = compile.Memo()
+        cloned_operations = [op.clone(memo) for op in operations]
+        for op in cloned_operations:
+            op.is_source_op = True
 
         preamble = TreeLoop('preamble')
         preamble.inputargs = inputargs
@@ -427,11 +430,10 @@
 
         assert preamble.operations[-1].getopnum() == rop.LABEL
 
-        inliner = Inliner(inputargs, jump_args)
         loop.operations = [preamble.operations[-1]] + \
-                          [inliner.inline_op(op, clone=False) for op in cloned_operations] + \
-                          [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jump_args],
-                                        None, descr=token)]
+                          cloned_operations + \
+                          [ResOperation(rop.JUMP, [memo.get(a, a) for a in jump_args],
+                                        descr=token)]
                           #[inliner.inline_op(jumpop)]
         assert loop.operations[-1].getopnum() == rop.JUMP
         assert loop.operations[0].getopnum() == rop.LABEL
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,8 +1,7 @@
 import sys
 
-from rpython.jit.metainterp.compile import ResumeGuardDescr
+from rpython.jit.metainterp.compile import Memo
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
-from rpython.jit.metainterp.inliner import Inliner
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
@@ -115,7 +114,7 @@
 
             if start_label and self.jump_to_start_label(start_label, stop_label):
                 # Initial label matches, jump to it
-                jumpop = ResOperation(rop.JUMP, stop_label.getarglist(), None,
+                jumpop = ResOperation(rop.JUMP, stop_label.getarglist(),
                                       descr=start_label.getdescr())
                 if self.short:
                     # Construct our short preamble
@@ -318,9 +317,10 @@
                                       'end of this bridge does not do that.')
 
             args[short_inputargs[i]] = jmp_to_short_args[i]
-        self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
-        self._inline_short_preamble(self.short, self.short_inliner,
-                                    patchguardop, self.short_boxes.assumed_classes)
+        self.memo = Memo(short_inputargs, jmp_to_short_args)
+        self._inline_short_preamble(self.short, self.memo,
+                                    patchguardop,
+                                    self.short_boxes.assumed_classes)
 
         # Import boxes produced in the preamble but used in the loop
         newoperations = self.optimizer.get_newoperations()
@@ -341,7 +341,7 @@
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
-        self.short.append(ResOperation(rop.JUMP, short_jumpargs, None, descr=jumpop.getdescr()))
+        self.short.append(ResOperation(rop.JUMP, short_jumpargs, descr=jumpop.getdescr()))
 
         # Verify that the virtual state at the end of the loop is one
         # that is compatible with the virtual state at the start of the loop
@@ -397,21 +397,20 @@
             else:
                 newargs[i] = a.clonebox()
                 boxmap[a] = newargs[i]
-        inliner = Inliner(short_inputargs, newargs)
+        memo = Memo(short_inputargs, newargs)
         target_token.assumed_classes = {}
         for i in range(len(short)):
             op = short[i]
-            newop = inliner.inline_op(op)
-            if op.result and op.result in self.short_boxes.assumed_classes:
-                target_token.assumed_classes[newop.result] = self.short_boxes.assumed_classes[op.result]
+            newop = op.clone(memo)
+            if op.type != 'v' and op in self.short_boxes.assumed_classes:
+                target_token.assumed_classes[newop] = self.short_boxes.assumed_classes[op]
             short[i] = newop
 
         # Forget the values to allow them to be freed
         for box in short[0].getarglist():
             box.forget_value()
         for op in short:
-            if op.result:
-                op.result.forget_value()
+            op.forget_value()
         target_token.short_preamble = self.short
         target_token.exported_state = None
 
@@ -492,7 +491,7 @@
 
 
     def _import_op(self, op, inputargs, short_jumpargs, jumpargs):
-        self.boxes_created_this_iteration[op.result] = None
+        self.boxes_created_this_iteration[op] = None
         args = op.getarglist()
         if op.is_guard():
             args = args + op.getfailargs()
@@ -551,7 +550,7 @@
             args = target.virtual_state.make_inputargs(values, self.optimizer,
                                                        keyboxes=True)
             short_inputargs = target.short_preamble[0].getarglist()
-            inliner = Inliner(short_inputargs, args)
+            memo = Memo(short_inputargs, args)
 
             for guard in extra_guards:
                 if guard.is_guard():
@@ -561,7 +560,7 @@
 
             try:
                 # NB: the short_preamble ends with a jump
-                self._inline_short_preamble(target.short_preamble, inliner, patchguardop, target.assumed_classes)
+                self._inline_short_preamble(target.short_preamble, memo, patchguardop, target.assumed_classes)
             except InvalidLoop:
                 #debug_print("Inlining failed unexpectedly",
                 #            "jumping to preamble instead")
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -242,6 +242,9 @@
     def returns_bool_result(self):
         return self._cls_has_bool_result
 
+    def forget_value(self):
+        pass
+
 
 # ===================
 # Top of the hierachy
@@ -358,6 +361,9 @@
 
     getvalue = getref_base
 
+    def forget_value(self):
+        self._resref = lltype.nullptr(llmemory.GCREF.TO)
+
     def setref_base(self, refval):
         self._resref = refval
 
@@ -626,7 +632,7 @@
     'INT_INVERT/1/i',
     'INT_FORCE_GE_ZERO/1/i',
     #
-    'SAME_AS/1/rfi',      # gets a Const or a Box, turns it into another Box
+    'SAME_AS/1/ifr',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1/i',
     'CAST_INT_TO_PTR/1/r',
     #
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -14,9 +14,6 @@
 
 class ESCAPE_OP(N_aryOp, ResOpWithDescr):
 
-    OPNUM = -123
-
-    type = 'i'
     is_source_op = True
 
     def getopnum(self):
@@ -25,14 +22,38 @@
     def getopname(self):
         return 'escape'
 
-    def clone(self):
-        op = ESCAPE_OP()
-        op.initarglist(self.getarglist()[:])
+    def clone(self, memo):
+        op = self.__class__()
+        op.initarglist([memo.get(arg, arg) for arg in self.getarglist()])
+        memo.set(self, op)
         return op
 
+class ESCAPE_OP_I(ESCAPE_OP):
+    type = 'i'
+    OPNUM = -123
+
+class ESCAPE_OP_F(ESCAPE_OP):
+    type = 'f'
+    OPNUM = -124
+
+class ESCAPE_OP_N(ESCAPE_OP):
+    type = 'v'
+    OPNUM = -125
+
+class ESCAPE_OP_R(ESCAPE_OP):
+    type = 'r'
+    OPNUM = -126
+
+ALL_ESCAPE_OPS = {
+    ESCAPE_OP_I.OPNUM: ESCAPE_OP_I,
+    ESCAPE_OP_F.OPNUM: ESCAPE_OP_F,
+    ESCAPE_OP_N.OPNUM: ESCAPE_OP_N,
+    ESCAPE_OP_R.OPNUM: ESCAPE_OP_R
+}
+
 class FORCE_SPILL(UnaryOp, PlainResOp):
 
-    OPNUM = -124
+    OPNUM = -127
     is_source_op = True
 
     def getopnum(self):
@@ -224,8 +245,14 @@
         try:
             opnum = getattr(rop, opname.upper())
         except AttributeError:
-            if opname == 'escape':
-                opnum = ESCAPE_OP.OPNUM
+            if opname == 'escape_i':
+                opnum = ESCAPE_OP_I.OPNUM
+            elif opname == 'escape_f':
+                opnum = ESCAPE_OP_F.OPNUM
+            elif opname == 'escape_n':
+                opnum = ESCAPE_OP_N.OPNUM
+            elif opname == 'escape_r':
+                opnum = ESCAPE_OP_R.OPNUM
             elif opname == 'force_spill':
                 opnum = FORCE_SPILL.OPNUM
             else:
@@ -268,8 +295,8 @@
         return opnum, args, descr, fail_args
 
     def create_op(self, opnum, args, descr):
-        if opnum == ESCAPE_OP.OPNUM:
-            op = ESCAPE_OP()
+        if opnum in ALL_ESCAPE_OPS:
+            op = ALL_ESCAPE_OPS[opnum]()
             op.initarglist(args)
             assert descr is None
             return op

From noreply at buildbot.pypy.org  Wed Dec  3 20:30:18 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed,  3 Dec 2014 20:30:18 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: First WIP for the Scipy India talk
Message-ID: <20141203193018.986481D34B6@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: extradoc
Changeset: r5473:6790ce64ab6c
Date: 2014-12-03 17:40 +0100
http://bitbucket.org/pypy/extradoc/changeset/6790ce64ab6c/

Log:	First WIP for the Scipy India talk

diff --git a/talk/scipyindia2014/talk.rst b/talk/scipyindia2014/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/scipyindia2014/talk.rst
@@ -0,0 +1,79 @@
+=============================
+PyPy and the scientific stack
+=============================
+
+Introduction
+------------
+
+* PyPy contributor
+
+* Hired to work on NumPyPy
+
+* Interested in library compatibility
+
+* @rguillebert on twitter, feel free to send me questions
+
+* Software consultant
+
+* Feel free to interrupt me
+
+PyPy
+----
+
+* PyPy is an implementation of the Python language
+
+* Speed is one of its main advantages
+
+* Compatibility is very important to us
+
+Speed
+-----
+
+* XXX : Insert speed.pypy.org screenshot
+
+Python
+------
+
+* Python is a great language
+
+* Very dynamic
+
+* Easy to introspect (pdb is just another Python module)
+
+* Considered slow
+
+How to get performance out of Python code ?
+-------------------------------------------
+
+* Rewrite your code in C
+
+* Rewrite your code in Cython
+
+* Rewrite your code in some subset/dialect of Python like Numba
+
+* Just write Python and use PyPy, pay only the cost of what you use
+
+PyPy And C
+----------
+
+* PyPy is pretty good at interacting with C code with cffi
+
+* C extensions written using the Python C API can work, but they're slow and support is incomplete
+
+* We have ideas to help with that in some use cases
+
+Python C API
+------------
+
+* Leaks way too many implementation details (refcounting, PyObject structure fields)
+
+* Makes it hard to improve Python while supporting 100% of the API
+
+* Should we have a new C API ?
+
+NumPyPy
+-------
+
+* ~80% of the numpy tests are passing
+
+* XXX

From noreply at buildbot.pypy.org  Wed Dec  3 21:05:30 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed,  3 Dec 2014 21:05:30 +0100 (CET)
Subject: [pypy-commit] pypy default: simplify setslice with no shape
Message-ID: <20141203200530.0E3CC1D350A@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74797:e2e286fb7245
Date: 2014-12-03 15:03 -0500
http://bitbucket.org/pypy/pypy/changeset/e2e286fb7245/

Log:	simplify setslice with no shape

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -89,17 +89,14 @@
 
 def setslice(space, shape, target, source):
     if not shape:
-        # XXX - simplify
-        target_iter, target_state = target.create_iter(shape)
-        source_iter, source_state = source.create_iter(shape)
         dtype = target.dtype
-        val = source_iter.getitem(source_state)
+        val = source.getitem(source.start)
         if dtype.is_str_or_unicode():
             val = dtype.coerce(space, val)
         else:
             val = val.convert_to(space, dtype)
-        target_iter.setitem(target_state, val)
-        return target        
+        target.setitem(target.start, val)
+        return target
     return _setslice(space, shape, target, source)
 
 def _setslice(space, shape, target, source):

From noreply at buildbot.pypy.org  Wed Dec  3 23:11:31 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed,  3 Dec 2014 23:11:31 +0100 (CET)
Subject: [pypy-commit] pypy default: avoid tracking an index in setslice,
 update/enable its test_zjit
Message-ID: <20141203221131.30B631C07E8@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74798:f1b314da580e
Date: 2014-12-03 17:01 -0500
http://bitbucket.org/pypy/pypy/changeset/f1b314da580e/

Log:	avoid tracking an index in setslice, update/enable its test_zjit

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -104,6 +104,7 @@
     # array implementations, not arrays
     target_iter, target_state = target.create_iter(shape)
     source_iter, source_state = source.create_iter(shape)
+    source_iter.track_index = False
     dtype = target.dtype
     shapelen = len(shape)
     while not target_iter.done(target_state):
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -479,20 +479,27 @@
         a = |30|
         b = |10|
         b[1] = 5.5
-        c = b + b
-        a[0:30:3] = c
+        a[0:30:3] = b
         a -> 3
         """
 
     def test_setslice(self):
         result = self.run("setslice")
-        assert result == 11.0
-        py.test.skip("don't run for now")
+        assert result == 5.5
         self.check_trace_count(1)
-        self.check_simple_loop({'raw_load': 2, 'float_add': 1,
-                                'raw_store': 1, 'int_add': 2,
-                                'int_eq': 1, 'guard_false': 1, 'jump': 1,
-                                'arraylen_gc': 1})
+        self.check_simple_loop({
+            'getarrayitem_gc': 1,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'guard_true': 1,
+            'int_add': 4,
+            'int_ge': 1,
+            'int_lt': 1,
+            'jump': 1,
+            'raw_load': 1,
+            'raw_store': 1,
+            'setarrayitem_gc': 1,
+        })
 
     def define_virtual_slice():
         return """

From noreply at buildbot.pypy.org  Wed Dec  3 23:37:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  3 Dec 2014 23:37:52 +0100 (CET)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20141203223752.444E21D2C37@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r556:90302869d491
Date: 2014-12-03 23:38 +0100
http://bitbucket.org/pypy/pypy.org/changeset/90302869d491/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 55.3
+      value: 55.6
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $58082 of $105000 (55.3%)
+   $58367 of $105000 (55.6%)
    <div id="progressbar">
    </div>
 
diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 84.8
+      value: 85.2
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Numpy / 0.95 -->
-   $50904 of $60000 (84.8%)
+   $51099 of $60000 (85.2%)
    <div id="progressbar">
    </div>
 
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -9,7 +9,7 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 24.6
+      value: 26.8
    });
   });
 </script>
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $19674 of $80000 (24.6%)
+   $21455 of $80000 (26.8%)
    <div id="progressbar">
    </div>
 

From noreply at buildbot.pypy.org  Thu Dec  4 01:28:19 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 01:28:19 +0100 (CET)
Subject: [pypy-commit] pypy default: rename iter.update to iter.indices for
	clarity
Message-ID: <20141204002819.991681C363D@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74799:1287046cabea
Date: 2014-12-03 17:38 -0500
http://bitbucket.org/pypy/pypy/changeset/1287046cabea/

Log:	rename iter.update to iter.indices for clarity

diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py
--- a/pypy/module/micronumpy/flatiter.py
+++ b/pypy/module/micronumpy/flatiter.py
@@ -41,8 +41,8 @@
         return space.wrap(self.state.index)
 
     def descr_coords(self, space):
-        self.state = self.iter.update(self.state)
-        return space.newtuple([space.wrap(c) for c in self.state.indices])
+        coords = self.iter.indices(self.state)
+        return space.newtuple([space.wrap(c) for c in coords])
 
     def descr_iter(self):
         return self
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -87,12 +87,12 @@
 
 
 class IterState(object):
-    _immutable_fields_ = ['iterator', 'index', 'indices', 'offset']
+    _immutable_fields_ = ['iterator', 'index', '_indices', 'offset']
 
     def __init__(self, iterator, index, indices, offset):
         self.iterator = iterator
         self.index = index
-        self.indices = indices
+        self._indices = indices
         self.offset = offset
 
 
@@ -144,7 +144,7 @@
             indices = [0] * len(self.shape_m1)
         else:
             assert state.iterator is self
-            indices = state.indices
+            indices = state._indices
             for i in xrange(self.ndim_m1, -1, -1):
                 indices[i] = 0
         return IterState(self, 0, indices, self.array.start)
@@ -155,7 +155,7 @@
         index = state.index
         if self.track_index:
             index += 1
-        indices = state.indices
+        indices = state._indices
         offset = state.offset
         if self.contiguous:
             offset += self.array.dtype.elsize
@@ -184,20 +184,20 @@
         return IterState(self, index, None, offset)
 
     @jit.unroll_safe
-    def update(self, state):
+    def indices(self, state):
         assert state.iterator is self
         assert self.track_index
+        indices = state._indices
         if not self.contiguous:
-            return state
+            return indices
         current = state.index
-        indices = state.indices
         for i in xrange(len(self.shape_m1)):
             if self.factors[i] != 0:
                 indices[i] = current / self.factors[i]
                 current %= self.factors[i]
             else:
                 indices[i] = 0
-        return IterState(self, state.index, indices, state.offset)
+        return indices
 
     def done(self, state):
         assert state.iterator is self
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -247,8 +247,8 @@
                                             dtype=dtype)
         assert not arr_iter.done(arr_state)
         w_val = arr_iter.getitem(arr_state).convert_to(space, dtype)
-        out_state = out_iter.update(out_state)
-        if out_state.indices[axis] == 0:
+        out_indices = out_iter.indices(out_state)
+        if out_indices[axis] == 0:
             if identity is not None:
                 w_val = func(dtype, identity, w_val)
         else:
@@ -380,9 +380,9 @@
     while not arr_iter.done(arr_state):
         nonzero_driver.jit_merge_point(shapelen=shapelen, dims=dims, dtype=dtype)
         if arr_iter.getitem_bool(arr_state):
-            arr_state = arr_iter.update(arr_state)
+            arr_indices = arr_iter.indices(arr_state)
             for d in dims:
-                res_iter.setitem(res_state, box(arr_state.indices[d]))
+                res_iter.setitem(res_state, box(arr_indices[d]))
                 res_state = res_iter.next(res_state)
         arr_state = arr_iter.next(arr_state)
     return res
diff --git a/pypy/module/micronumpy/test/test_iterators.py b/pypy/module/micronumpy/test/test_iterators.py
--- a/pypy/module/micronumpy/test/test_iterators.py
+++ b/pypy/module/micronumpy/test/test_iterators.py
@@ -31,16 +31,14 @@
         s = i.next(s)
         assert s.offset == 3
         assert not i.done(s)
-        assert s.indices == [0,0]
-        s = i.update(s)
-        assert s.indices == [0,3]
+        assert s._indices == [0,0]
+        assert i.indices(s) == [0,3]
         #cause a dimension overflow
         s = i.next(s)
         s = i.next(s)
         assert s.offset == 5
-        assert s.indices == [0,3]
-        s = i.update(s)
-        assert s.indices == [1,0]
+        assert s._indices == [0,3]
+        assert i.indices(s) == [1,0]
 
         #Now what happens if the array is transposed? strides[-1] != 1
         # therefore layout is non-contiguous
@@ -56,12 +54,12 @@
         s = i.next(s)
         assert s.offset == 9
         assert not i.done(s)
-        assert s.indices == [0,3]
+        assert s._indices == [0,3]
         #cause a dimension overflow
         s = i.next(s)
         s = i.next(s)
         assert s.offset == 1
-        assert s.indices == [1,0]
+        assert s._indices == [1,0]
 
     def test_iterator_goto(self):
         shape = [3, 5]
@@ -74,9 +72,9 @@
         assert not i.contiguous
         s = i.reset()
         assert s.index == 0
-        assert s.indices == [0, 0]
+        assert s._indices == [0, 0]
         assert s.offset == a.start
         s = i.goto(11)
         assert s.index == 11
-        assert s.indices is None
+        assert s._indices is None
         assert s.offset == a.start + 5

From noreply at buildbot.pypy.org  Thu Dec  4 01:28:20 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 01:28:20 +0100 (CET)
Subject: [pypy-commit] pypy default: optimize iterators for single dim arrays
Message-ID: <20141204002820.CD6C41C363D@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74800:268850625399
Date: 2014-12-03 18:15 -0500
http://bitbucket.org/pypy/pypy/changeset/268850625399/

Log:	optimize iterators for single dim arrays

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -159,6 +159,8 @@
         offset = state.offset
         if self.contiguous:
             offset += self.array.dtype.elsize
+        elif self.ndim_m1 == 0:
+            offset += self.strides[0]
         else:
             for i in xrange(self.ndim_m1, -1, -1):
                 idx = indices[i]
@@ -176,6 +178,8 @@
         offset = self.array.start
         if self.contiguous:
             offset += index * self.array.dtype.elsize
+        elif self.ndim_m1 == 0:
+            offset += index * self.strides[0]
         else:
             current = index
             for i in xrange(len(self.shape_m1)):
@@ -188,7 +192,7 @@
         assert state.iterator is self
         assert self.track_index
         indices = state._indices
-        if not self.contiguous:
+        if not (self.contiguous or self.ndim_m1 == 0):
             return indices
         current = state.index
         for i in xrange(len(self.shape_m1)):
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -102,18 +102,15 @@
         assert result == 3 + 3
         self.check_trace_count(1)
         self.check_simple_loop({
+            'arraylen_gc': 1,
             'float_add': 1,
-            'getarrayitem_gc': 1,
             'guard_false': 1,
             'guard_not_invalidated': 1,
-            'guard_true': 1,
-            'int_add': 5,
+            'int_add': 4,
             'int_ge': 1,
-            'int_lt': 1,
             'jump': 1,
             'raw_load': 2,
             'raw_store': 1,
-            'setarrayitem_gc': 1,
         })
 
     def define_pow():
@@ -127,24 +124,22 @@
         assert result == 3 ** 2
         self.check_trace_count(1)
         self.check_simple_loop({
+            'arraylen_gc': 1,
             'call': 1,
             'float_add': 1,
             'float_eq': 3,
             'float_mul': 2,
             'float_ne': 1,
-            'getarrayitem_gc': 1,
             'getarrayitem_raw': 1,     # read the errno
             'guard_false': 4,
             'guard_not_invalidated': 1,
-            'guard_true': 3,
-            'int_add': 5,
+            'guard_true': 2,
+            'int_add': 4,
             'int_ge': 1,
             'int_is_true': 1,
-            'int_lt': 1,
             'jump': 1,
             'raw_load': 2,
             'raw_store': 1,
-            'setarrayitem_gc': 1,
             'setarrayitem_raw': 1,     # write the errno
         })
 
@@ -162,18 +157,15 @@
         self.check_trace_count(2)  # extra one for the astype
         del get_stats().loops[0]   # we don't care about it
         self.check_simple_loop({
+            'arraylen_gc': 1,
             'call': 1,
-            'getarrayitem_gc': 1,
             'guard_false': 1,
             'guard_not_invalidated': 1,
-            'guard_true': 1,
-            'int_add': 5,
+            'int_add': 4,
             'int_ge': 1,
-            'int_lt': 1,
             'jump': 1,
             'raw_load': 2,
             'raw_store': 1,
-            'setarrayitem_gc': 1,
         })
 
     def define_sum():
@@ -390,18 +382,15 @@
         assert result == 18
         self.check_trace_count(1)
         self.check_simple_loop({
+            'arraylen_gc': 2,
             'float_add': 1,
-            'getarrayitem_gc': 2,
             'guard_false': 1,
             'guard_not_invalidated': 1,
-            'guard_true': 2,
-            'int_add': 6,
+            'int_add': 4,
             'int_ge': 1,
-            'int_lt': 2,
             'jump': 1,
             'raw_load': 2,
             'raw_store': 1,
-            'setarrayitem_gc': 2,
         })
 
     def define_take():
@@ -488,17 +477,14 @@
         assert result == 5.5
         self.check_trace_count(1)
         self.check_simple_loop({
-            'getarrayitem_gc': 1,
+            'arraylen_gc': 1,
             'guard_false': 1,
             'guard_not_invalidated': 1,
-            'guard_true': 1,
-            'int_add': 4,
+            'int_add': 3,
             'int_ge': 1,
-            'int_lt': 1,
             'jump': 1,
             'raw_load': 1,
             'raw_store': 1,
-            'setarrayitem_gc': 1,
         })
 
     def define_virtual_slice():
@@ -618,12 +604,13 @@
             'getarrayitem_gc_pure': 9,
             'getfield_gc_pure': 49,
             'guard_class': 4,
-            'guard_false': 13,
+            'guard_false': 15,
             'guard_not_invalidated': 2,
             'guard_true': 14,
             'int_add': 17,
             'int_ge': 4,
             'int_is_true': 3,
+            'int_is_zero': 2,
             'int_le': 5,
             'int_lt': 8,
             'int_sub': 3,

From noreply at buildbot.pypy.org  Thu Dec  4 01:28:22 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 01:28:22 +0100 (CET)
Subject: [pypy-commit] pypy default: fix test_pypy_c for iterator changes
Message-ID: <20141204002822.0E9251C363D@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74801:651d67d1059e
Date: 2014-12-03 19:27 -0500
http://bitbucket.org/pypy/pypy/changeset/651d67d1059e/

Log:	fix test_pypy_c for iterator changes

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -99,7 +99,7 @@
             i128 = int_mul(i117, i59)
             i129 = int_add(i55, i128)
             f149 = raw_load(i100, i129, descr=<ArrayF 8>)
-            p150 = getfield_gc_pure(p123, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_indices .+>)
+            p150 = getfield_gc_pure(p123, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
             i151 = int_add(i117, 1)
             setarrayitem_gc(p150, 1, 0, descr=<ArrayS .+>)
             setarrayitem_gc(p150, 0, 0, descr=<ArrayS .+>)
@@ -112,7 +112,7 @@
             setfield_gc(p156, p49, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_iterator .+>)
             setfield_gc(p156, i55, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
             setfield_gc(p156, 0, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index .+>)
-            setfield_gc(p156, p150, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_indices .+>)
+            setfield_gc(p156, p150, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
             setfield_gc(p16, p156, descr=<FieldP pypy.module.micronumpy.flatiter.W_FlatIterator.inst_state .+>)
             }}}
             jump(p0, p1, p3, p6, p7, p12, p14, p16, i151, f149, p26, i44, i50, i59, i55, i100, p156, p49, descr=...)
@@ -140,7 +140,7 @@
             i132 = int_add(i53, i131)
             guard_not_invalidated(descr=...)
             raw_store(i103, i132, 42.000000, descr=<ArrayF 8>)
-            p152 = getfield_gc_pure(p126, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_indices .+>)
+            p152 = getfield_gc_pure(p126, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
             i153 = int_add(i120, 1)
             i154 = getfield_raw(#, descr=<FieldS pypysig_long_struct.c_value 0>)
             setarrayitem_gc(p152, 1, 0, descr=<ArrayS .+>)
@@ -152,7 +152,7 @@
             setfield_gc(p158, p47, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_iterator .+>)
             setfield_gc(p158, i53, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
             setfield_gc(p158, 0, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index .+>)
-            setfield_gc(p158, p152, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_indices .+>)
+            setfield_gc(p158, p152, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
             setfield_gc(p16, p158, descr=<FieldP pypy.module.micronumpy.flatiter.W_FlatIterator.inst_state .+>)
             }}}
             jump(p0, p1, p3, p6, p7, p12, p14, p16, i153, i42, i48, i57, i53, p47, i103, p158, descr=...)

From noreply at buildbot.pypy.org  Thu Dec  4 03:03:51 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 03:03:51 +0100 (CET)
Subject: [pypy-commit] pypy default: silence pyflakes
Message-ID: <20141204020351.704941D2C37@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74802:6bd0b3c411b0
Date: 2014-12-03 19:48 -0500
http://bitbucket.org/pypy/pypy/changeset/6bd0b3c411b0/

Log:	silence pyflakes

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -167,7 +167,7 @@
         if len(args_w) >= 1:
             for w_arg in args_w:
                 try:
-                    idx = support.index_w(space, w_arg)
+                    support.index_w(space, w_arg)
                 except OperationError:
                     raise oefmt(space.w_TypeError, "an integer is required")
             raise oefmt(space.w_ValueError, "axes don't match array")

From noreply at buildbot.pypy.org  Thu Dec  4 03:03:52 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 03:03:52 +0100 (CET)
Subject: [pypy-commit] pypy default: move unnecessary additions to base
 iterator from nditer branches to subclass
Message-ID: <20141204020352.D20971D2DD0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74803:73e3f0512a73
Date: 2014-12-03 21:02 -0500
http://bitbucket.org/pypy/pypy/changeset/73e3f0512a73/

Log:	move unnecessary additions to base iterator from nditer branches to
	subclass

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -41,16 +41,6 @@
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.flagsobj import _update_contiguous_flags
 
-class OpFlag(object):
-    def __init__(self):
-        self.rw = ''
-        self.broadcast = True
-        self.force_contig = False
-        self.force_align = False
-        self.native_byte_order = False
-        self.tmp_copy = ''
-        self.allocate = False
-
 
 class PureShapeIter(object):
     def __init__(self, shape, idx_w):
@@ -99,14 +89,12 @@
 class ArrayIter(object):
     _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
                           'strides[*]', 'backstrides[*]', 'factors[*]',
-                          'slice_shape', 'slice_stride', 'slice_backstride',
-                          'track_index', 'operand_type', 'slice_operand_type']
+                          'track_index']
 
     track_index = True
 
     @jit.unroll_safe
-    def __init__(self, array, size, shape, strides, backstrides, op_flags=OpFlag()):
-        from pypy.module.micronumpy import concrete
+    def __init__(self, array, size, shape, strides, backstrides):
         assert len(shape) == len(strides) == len(backstrides)
         _update_contiguous_flags(array)
         self.contiguous = (array.flags & NPY.ARRAY_C_CONTIGUOUS and
@@ -118,12 +106,6 @@
         self.shape_m1 = [s - 1 for s in shape]
         self.strides = strides
         self.backstrides = backstrides
-        self.slice_shape = 1
-        self.slice_stride = -1
-        if strides:
-            self.slice_stride = strides[-1]
-        self.slice_backstride = 1
-        self.slice_operand_type = concrete.SliceArray
 
         ndim = len(shape)
         factors = [0] * ndim
@@ -133,10 +115,6 @@
             else:
                 factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i]
         self.factors = factors
-        if op_flags.rw == 'r':
-            self.operand_type = concrete.ConcreteNonWritableArrayWithBase
-        else:
-            self.operand_type = concrete.ConcreteArrayWithBase
 
     @jit.unroll_safe
     def reset(self, state=None):
@@ -220,12 +198,6 @@
         assert state.iterator is self
         self.array.setitem(state.offset, elem)
 
-    def getoperand(self, st, base):
-        impl = self.operand_type
-        res = impl([], self.array.dtype, self.array.order, [], [],
-                   self.array.storage, base)
-        res.start = st.offset
-        return res
 
 def AxisIter(array, shape, axis, cumulative):
     strides = array.get_strides()
@@ -249,42 +221,3 @@
         size /= shape[axis]
     shape[axis] = backstrides[axis] = 0
     return ArrayIter(array, size, shape, array.strides, backstrides)
-
-class SliceIter(ArrayIter):
-    '''
-    used with external loops, getitem and setitem return a SliceArray
-    view into the original array
-    '''
-    _immutable_fields_ = ['base', 'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]']
-
-    def __init__(self, array, size, shape, strides, backstrides, slice_shape,
-                 slice_stride, slice_backstride, op_flags, base):
-        from pypy.module.micronumpy import concrete
-        ArrayIter.__init__(self, array, size, shape, strides, backstrides, op_flags)
-        self.slice_shape = slice_shape
-        self.slice_stride = slice_stride
-        self.slice_backstride = slice_backstride
-        self.base = base
-        if op_flags.rw == 'r':
-            self.slice_operand_type = concrete.NonWritableSliceArray
-        else:
-            self.slice_operand_type = concrete.SliceArray
-
-    def getitem(self, state):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def getitem_bool(self, state):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def setitem(self, state, elem):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def getoperand(self, state, base):
-        assert state.iterator is self
-        impl = self.slice_operand_type
-        arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
-                   [self.slice_shape], self.array, self.base)
-        return arr
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -6,7 +6,7 @@
 from pypy.module.micronumpy import ufuncs, support, concrete
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.descriptor import decode_w_dtype
-from pypy.module.micronumpy.iterators import ArrayIter, SliceIter, OpFlag
+from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
                                             shape_agreement, shape_agreement_multiple)
 
@@ -36,6 +36,16 @@
     return ret
 
 
+class OpFlag(object):
+    def __init__(self):
+        self.rw = ''
+        self.broadcast = True
+        self.force_contig = False
+        self.force_align = False
+        self.native_byte_order = False
+        self.tmp_copy = ''
+        self.allocate = False
+
 def parse_op_flag(space, lst):
     op_flag = OpFlag()
     for w_item in lst:
@@ -142,11 +152,75 @@
         raise NotImplementedError('not implemented yet')
 
 
-def get_iter(space, order, arr, shape, dtype, op_flags):
+class OperandIter(ArrayIter):
+    _immutable_fields_ = ['slice_shape', 'slice_stride', 'slice_backstride',
+                          'operand_type', 'base']
+
+    def getitem(self, state):
+        # XXX cannot be called - must return a boxed value
+        assert False
+
+    def getitem_bool(self, state):
+        # XXX cannot be called - must return a boxed value
+        assert False
+
+    def setitem(self, state, elem):
+        # XXX cannot be called - must return a boxed value
+        assert False
+
+
+class ConcreteIter(OperandIter):
+    def __init__(self, array, size, shape, strides, backstrides,
+                 op_flags, base):
+        OperandIter.__init__(self, array, size, shape, strides, backstrides)
+        self.slice_shape = 1
+        self.slice_stride = -1
+        if strides:
+            self.slice_stride = strides[-1]
+        self.slice_backstride = 1
+        if op_flags.rw == 'r':
+            self.operand_type = concrete.ConcreteNonWritableArrayWithBase
+        else:
+            self.operand_type = concrete.ConcreteArrayWithBase
+        self.base = base
+
+    def getoperand(self, state):
+        assert state.iterator is self
+        impl = self.operand_type
+        #assert issubclass(impl, concrete.ConcreteArrayWithBase)
+        res = impl([], self.array.dtype, self.array.order, [], [],
+                   self.array.storage, self.base)
+        res.start = state.offset
+        return res
+
+
+class SliceIter(OperandIter):
+    def __init__(self, array, size, shape, strides, backstrides, slice_shape,
+                 slice_stride, slice_backstride, op_flags, base):
+        OperandIter.__init__(self, array, size, shape, strides, backstrides)
+        self.slice_shape = slice_shape
+        self.slice_stride = slice_stride
+        self.slice_backstride = slice_backstride
+        if op_flags.rw == 'r':
+            self.operand_type = concrete.NonWritableSliceArray
+        else:
+            self.operand_type = concrete.SliceArray
+        self.base = base
+
+    def getoperand(self, state):
+        assert state.iterator is self
+        impl = self.operand_type
+        #assert issubclass(impl, concrete.SliceArray)
+        arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
+                   [self.slice_shape], self.array, self.base)
+        return arr
+
+
+def get_iter(space, order, arr, shape, dtype, op_flags, base):
     imp = arr.implementation
     backward = is_backward(imp, order)
     if arr.is_scalar():
-        return ArrayIter(imp, 1, [], [], [], op_flags=op_flags)
+        return ConcreteIter(imp, 1, [], [], [], op_flags, base)
     if (imp.strides[0] < imp.strides[-1] and not backward) or \
        (imp.strides[0] > imp.strides[-1] and backward):
         # flip the strides. Is this always true for multidimension?
@@ -161,7 +235,7 @@
         backstrides = imp.backstrides
     r = calculate_broadcast_strides(strides, backstrides, imp.shape,
                                     shape, backward)
-    return ArrayIter(imp, imp.get_size(), shape, r[0], r[1], op_flags=op_flags)
+    return ConcreteIter(imp, imp.get_size(), shape, r[0], r[1], op_flags, base)
 
 def calculate_ndim(op_in, oa_ndim):
     if oa_ndim >=0:
@@ -398,7 +472,7 @@
         self.iters = []
         for i in range(len(self.seq)):
             it = get_iter(space, self.order, self.seq[i], self.shape,
-                          self.dtypes[i], self.op_flags[i])
+                          self.dtypes[i], self.op_flags[i], self)
             it.contiguous = False
             self.iters.append((it, it.reset()))
 
@@ -437,7 +511,7 @@
         return space.wrap(self)
 
     def getitem(self, it, st):
-        res = it.getoperand(st, self)
+        res = it.getoperand(st)
         return W_NDimArray(res)
 
     def descr_getitem(self, space, w_idx):

From noreply at buildbot.pypy.org  Thu Dec  4 03:08:04 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 03:08:04 +0100 (CET)
Subject: [pypy-commit] pypy default: cleanup comments
Message-ID: <20141204020804.B7F2A1D2C37@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74804:cab353e0b9a5
Date: 2014-12-03 21:08 -0500
http://bitbucket.org/pypy/pypy/changeset/cab353e0b9a5/

Log:	cleanup comments

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -157,15 +157,15 @@
                           'operand_type', 'base']
 
     def getitem(self, state):
-        # XXX cannot be called - must return a boxed value
+        # cannot be called - must return a boxed value
         assert False
 
     def getitem_bool(self, state):
-        # XXX cannot be called - must return a boxed value
+        # cannot be called - must return a boxed value
         assert False
 
     def setitem(self, state, elem):
-        # XXX cannot be called - must return a boxed value
+        # cannot be called - must return a boxed value
         assert False
 
 
@@ -187,7 +187,6 @@
     def getoperand(self, state):
         assert state.iterator is self
         impl = self.operand_type
-        #assert issubclass(impl, concrete.ConcreteArrayWithBase)
         res = impl([], self.array.dtype, self.array.order, [], [],
                    self.array.storage, self.base)
         res.start = state.offset
@@ -210,7 +209,6 @@
     def getoperand(self, state):
         assert state.iterator is self
         impl = self.operand_type
-        #assert issubclass(impl, concrete.SliceArray)
         arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
                    [self.slice_shape], self.array, self.base)
         return arr

From noreply at buildbot.pypy.org  Thu Dec  4 05:38:58 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 05:38:58 +0100 (CET)
Subject: [pypy-commit] pypy default: optimize call2 when lhs or rhs is a
	scalar
Message-ID: <20141204043858.D59A91C1147@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74805:378bfef11c09
Date: 2014-12-03 23:30 -0500
http://bitbucket.org/pypy/pypy/changeset/378bfef11c09/

Log:	optimize call2 when lhs or rhs is a scalar

diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py
--- a/pypy/module/micronumpy/flatiter.py
+++ b/pypy/module/micronumpy/flatiter.py
@@ -22,6 +22,9 @@
     def get_shape(self):
         return self.shape
 
+    def get_size(self):
+        return self.base().get_size()
+
     def create_iter(self, shape=None, backward_broadcast=False):
         assert isinstance(self.base(), W_NDimArray)
         return self.base().create_iter()
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -42,23 +42,38 @@
 
     # TODO handle __array_priorities__ and maybe flip the order
 
+    if w_lhs.get_size() == 1:
+        w_left = w_lhs.get_scalar_value().convert_to(space, calc_dtype)
+        left_iter = left_state = None
+    else:
+        w_left = None
+        left_iter, left_state = w_lhs.create_iter(shape)
+        left_iter.track_index = False
+
+    if w_rhs.get_size() == 1:
+        w_right = w_rhs.get_scalar_value().convert_to(space, calc_dtype)
+        right_iter = right_state = None
+    else:
+        w_right = None
+        right_iter, right_state = w_rhs.create_iter(shape)
+        right_iter.track_index = False
+
     if out is None:
         out = W_NDimArray.from_shape(space, shape, res_dtype,
                                      w_instance=lhs_for_subtype)
-    left_iter, left_state = w_lhs.create_iter(shape)
-    right_iter, right_state = w_rhs.create_iter(shape)
     out_iter, out_state = out.create_iter(shape)
-    left_iter.track_index = right_iter.track_index = False
     shapelen = len(shape)
     while not out_iter.done(out_state):
         call2_driver.jit_merge_point(shapelen=shapelen, func=func,
                                      calc_dtype=calc_dtype, res_dtype=res_dtype)
-        w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
-        w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
+        if left_iter:
+            w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
+            left_state = left_iter.next(left_state)
+        if right_iter:
+            w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
+            right_state = right_iter.next(right_state)
         out_iter.setitem(out_state, func(calc_dtype, w_left, w_right).convert_to(
             space, res_dtype))
-        left_state = left_iter.next(left_state)
-        right_state = right_iter.next(right_state)
         out_state = out_iter.next(out_state)
     return out
 
@@ -68,11 +83,12 @@
     reds='auto')
 
 def call1(space, shape, func, calc_dtype, res_dtype, w_obj, out):
+    obj_iter, obj_state = w_obj.create_iter(shape)
+    obj_iter.track_index = False
+
     if out is None:
         out = W_NDimArray.from_shape(space, shape, res_dtype, w_instance=w_obj)
-    obj_iter, obj_state = w_obj.create_iter(shape)
     out_iter, out_state = out.create_iter(shape)
-    obj_iter.track_index = False
     shapelen = len(shape)
     while not out_iter.done(out_state):
         call1_driver.jit_merge_point(shapelen=shapelen, func=func,
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -102,14 +102,13 @@
         assert result == 3 + 3
         self.check_trace_count(1)
         self.check_simple_loop({
-            'arraylen_gc': 1,
             'float_add': 1,
             'guard_false': 1,
             'guard_not_invalidated': 1,
-            'int_add': 4,
+            'int_add': 3,
             'int_ge': 1,
             'jump': 1,
-            'raw_load': 2,
+            'raw_load': 1,
             'raw_store': 1,
         })
 
@@ -124,21 +123,18 @@
         assert result == 3 ** 2
         self.check_trace_count(1)
         self.check_simple_loop({
-            'arraylen_gc': 1,
             'call': 1,
-            'float_add': 1,
-            'float_eq': 3,
+            'float_eq': 2,
             'float_mul': 2,
-            'float_ne': 1,
             'getarrayitem_raw': 1,     # read the errno
-            'guard_false': 4,
+            'guard_false': 2,
             'guard_not_invalidated': 1,
             'guard_true': 2,
-            'int_add': 4,
+            'int_add': 3,
             'int_ge': 1,
             'int_is_true': 1,
             'jump': 1,
-            'raw_load': 2,
+            'raw_load': 1,
             'raw_store': 1,
             'setarrayitem_raw': 1,     # write the errno
         })
@@ -157,14 +153,13 @@
         self.check_trace_count(2)  # extra one for the astype
         del get_stats().loops[0]   # we don't care about it
         self.check_simple_loop({
-            'arraylen_gc': 1,
             'call': 1,
             'guard_false': 1,
             'guard_not_invalidated': 1,
-            'int_add': 4,
+            'int_add': 3,
             'int_ge': 1,
             'jump': 1,
-            'raw_load': 2,
+            'raw_load': 1,
             'raw_store': 1,
         })
 

From noreply at buildbot.pypy.org  Thu Dec  4 05:55:30 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 05:55:30 +0100 (CET)
Subject: [pypy-commit] pypy default: enable test_multidim in test_zjit
Message-ID: <20141204045530.15ACC1D2859@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74806:5ae2c3c20777
Date: 2014-12-03 23:54 -0500
http://bitbucket.org/pypy/pypy/changeset/5ae2c3c20777/

Log:	enable test_multidim in test_zjit

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -420,13 +420,19 @@
     def test_multidim(self):
         result = self.run('multidim')
         assert result == 8
-        py.test.skip("don't run for now")
         # int_add might be 1 here if we try slightly harder with
         # reusing indexes or some optimization
-        self.check_simple_loop({'float_add': 1, 'raw_load': 2,
-                                'guard_false': 1, 'int_add': 1, 'int_ge': 1,
-                                'jump': 1, 'raw_store': 1,
-                                'arraylen_gc': 1})
+        self.check_trace_count(1)
+        self.check_simple_loop({
+            'float_add': 1,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'int_add': 4,
+            'int_ge': 1,
+            'jump': 1,
+            'raw_load': 2,
+            'raw_store': 1,
+        })
 
     def define_multidim_slice():
         return """

From noreply at buildbot.pypy.org  Thu Dec  4 06:20:12 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 06:20:12 +0100 (CET)
Subject: [pypy-commit] pypy default: enable test_multidim_slice in test_zjit
Message-ID: <20141204052012.8E8701D2859@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74807:5f2acc45e585
Date: 2014-12-04 00:20 -0500
http://bitbucket.org/pypy/pypy/changeset/5f2acc45e585/

Log:	enable test_multidim_slice in test_zjit

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -445,10 +445,47 @@
     def test_multidim_slice(self):
         result = self.run('multidim_slice')
         assert result == 12
-        py.test.skip("improve")
         # XXX the bridge here is scary. Hopefully jit-targets will fix that,
         #     otherwise it looks kind of good
-        self.check_simple_loop({})
+        self.check_trace_count(2)
+        self.check_simple_loop({
+            'float_add': 1,
+            'getarrayitem_gc': 2,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'guard_true': 2,
+            'int_add': 6,
+            'int_ge': 1,
+            'int_lt': 2,
+            'jump': 1,
+            'raw_load': 2,
+            'raw_store': 1,
+            'setarrayitem_gc': 2,
+        })
+        self.check_resops({
+            'float_add': 3,
+            'getarrayitem_gc': 7,
+            'getarrayitem_gc_pure': 14,
+            'getfield_gc_pure': 69,
+            'guard_class': 5,
+            'guard_false': 20,
+            'guard_nonnull': 6,
+            'guard_nonnull_class': 1,
+            'guard_not_invalidated': 3,
+            'guard_true': 16,
+            'guard_value': 2,
+            'int_add': 24,
+            'int_ge': 4,
+            'int_is_true': 6,
+            'int_is_zero': 4,
+            'int_le': 5,
+            'int_lt': 7,
+            'int_sub': 2,
+            'jump': 2,
+            'raw_load': 5,
+            'raw_store': 3,
+            'setarrayitem_gc': 8,
+        })
 
     def define_broadcast():
         return """

From noreply at buildbot.pypy.org  Thu Dec  4 06:36:47 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 06:36:47 +0100 (CET)
Subject: [pypy-commit] pypy default: enable test_broadcast too
Message-ID: <20141204053647.1C2AA1D29E9@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74808:d1a536a138a3
Date: 2014-12-04 00:36 -0500
http://bitbucket.org/pypy/pypy/changeset/d1a536a138a3/

Log:	enable test_broadcast too

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -498,8 +498,42 @@
     def test_broadcast(self):
         result = self.run("broadcast")
         assert result == 10
-        py.test.skip("improve")
-        self.check_simple_loop({})
+        self.check_trace_count(2)
+        self.check_simple_loop({
+            'float_add': 1,
+            'getarrayitem_gc': 1,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'guard_true': 1,
+            'int_add': 5,
+            'int_ge': 1,
+            'int_lt': 1,
+            'jump': 1,
+            'raw_load': 2,
+            'raw_store': 1,
+            'setarrayitem_gc': 1,
+        })
+        self.check_resops({
+            'float_add': 2,
+            'getarrayitem_gc': 2,
+            'getarrayitem_gc_pure': 2,
+            'getfield_gc_pure': 36,
+            'guard_class': 3,
+            'guard_false': 7,
+            'guard_nonnull': 2,
+            'guard_not_invalidated': 2,
+            'guard_true': 8,
+            'int_add': 11,
+            'int_ge': 2,
+            'int_is_true': 3,
+            'int_is_zero': 1,
+            'int_le': 1,
+            'int_lt': 2,
+            'jump': 1,
+            'raw_load': 4,
+            'raw_store': 2,
+            'setarrayitem_gc': 2,
+        })
 
     def define_setslice():
         return """

From noreply at buildbot.pypy.org  Thu Dec  4 08:03:16 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu,  4 Dec 2014 08:03:16 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Progress
Message-ID: <20141204070316.5B4C61C100F@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: extradoc
Changeset: r5474:70b9b4be4e0d
Date: 2014-12-04 02:13 +0100
http://bitbucket.org/pypy/extradoc/changeset/70b9b4be4e0d/

Log:	Progress

diff --git a/talk/scipyindia2014/talk.rst b/talk/scipyindia2014/talk.rst
--- a/talk/scipyindia2014/talk.rst
+++ b/talk/scipyindia2014/talk.rst
@@ -26,11 +26,6 @@
 
 * Compatibility is very important to us
 
-Speed
------
-
-* XXX : Insert speed.pypy.org screenshot
-
 Python
 ------
 
@@ -42,6 +37,16 @@
 
 * Considered slow
 
+Speed
+-----
+
+* XXX : Insert speed.pypy.org screenshot
+
+Demo
+----
+
+* Real-time edge detection
+
 How to get performance out of Python code ?
 -------------------------------------------
 
@@ -58,6 +63,12 @@
 
 * PyPy is pretty good at interacting with C code with cffi
 
+* With CFFI you can call C code from Python and expose Python functions to C 
+
+  - this means that you can create your own C API in pure python !
+
+* It is now possible to embed PyPy in a C application (uWSGI)
+
 * C extensions written using the Python C API can work, but they're slow and support is incomplete
 
 * We have ideas to help with that in some use cases

From noreply at buildbot.pypy.org  Thu Dec  4 08:03:17 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu,  4 Dec 2014 08:03:17 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Add the rst2beamer boilerplate
Message-ID: <20141204070317.B7D3F1C100F@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: extradoc
Changeset: r5475:f56bf45a654e
Date: 2014-12-04 02:25 +0100
http://bitbucket.org/pypy/extradoc/changeset/f56bf45a654e/

Log:	Add the rst2beamer boilerplate

diff --git a/talk/scipyindia2014/Makefile b/talk/scipyindia2014/Makefile
new file mode 100644
--- /dev/null
+++ b/talk/scipyindia2014/Makefile
@@ -0,0 +1,18 @@
+# you can find rst2beamer.py here:
+# https://bitbucket.org/antocuni/env/raw/default/bin/rst2beamer.py
+
+# WARNING: to work, it needs this patch for docutils
+# https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414
+
+talk.pdf: talk.rst author.latex stylesheet.latex
+	python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	#/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
+	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
+	#sed 's/\\maketitle/\\input{title.latex}/' -i talk.latex || exit
+	pdflatex talk.latex  || exit
+
+view: talk.pdf
+	evince talk.pdf &
+
+xpdf: talk.pdf
+	xpdf talk.pdf &
diff --git a/talk/scipyindia2014/author.latex b/talk/scipyindia2014/author.latex
new file mode 100644
--- /dev/null
+++ b/talk/scipyindia2014/author.latex
@@ -0,0 +1,9 @@
+\definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0}
+
+\title[PyPy and the scientific stack]{PyPy and the scientific stack}
+\author[rguillebert]
+{Romain Guillebert\\
+\includegraphics[width=80px]{../img/py-web-new.png}}
+
+\institute{Scipy India 2014}
+\date{December 7th, 2014}
diff --git a/talk/scipyindia2014/beamerdefs.txt b/talk/scipyindia2014/beamerdefs.txt
new file mode 100644
--- /dev/null
+++ b/talk/scipyindia2014/beamerdefs.txt
@@ -0,0 +1,108 @@
+.. colors
+.. ===========================
+
+.. role:: green
+.. role:: red
+
+
+.. general useful commands
+.. ===========================
+
+.. |pause| raw:: latex
+
+   \pause
+
+.. |small| raw:: latex
+
+   {\small
+
+.. |end_small| raw:: latex
+
+   }
+
+.. |scriptsize| raw:: latex
+
+   {\scriptsize
+
+.. |end_scriptsize| raw:: latex
+
+   }
+
+.. |strike<| raw:: latex
+
+   \sout{
+
+.. closed bracket
+.. ===========================
+
+.. |>| raw:: latex
+
+   }
+
+
+.. example block
+.. ===========================
+
+.. |example<| raw:: latex
+
+   \begin{exampleblock}{
+
+
+.. |end_example| raw:: latex
+
+   \end{exampleblock}
+
+
+
+.. alert block
+.. ===========================
+
+.. |alert<| raw:: latex
+
+   \begin{alertblock}{
+
+
+.. |end_alert| raw:: latex
+
+   \end{alertblock}
+
+
+
+.. columns
+.. ===========================
+
+.. |column1| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.45\textwidth}
+
+.. |column2| raw:: latex
+
+      \end{column}
+      \begin{column}{0.45\textwidth}
+
+
+.. |end_columns| raw:: latex
+
+      \end{column}
+   \end{columns}
+
+
+
+.. |snake| image:: ../../img/py-web-new.png
+           :scale: 15%
+           
+
+
+.. nested blocks
+.. ===========================
+
+.. |nested| raw:: latex
+
+   \begin{columns}
+      \begin{column}{0.85\textwidth}
+
+.. |end_nested| raw:: latex
+
+      \end{column}
+   \end{columns}
diff --git a/talk/scipyindia2014/stylesheet.latex b/talk/scipyindia2014/stylesheet.latex
new file mode 100644
--- /dev/null
+++ b/talk/scipyindia2014/stylesheet.latex
@@ -0,0 +1,9 @@
+\setbeamercovered{transparent}
+\setbeamertemplate{navigation symbols}{}
+
+\definecolor{darkgreen}{rgb}{0, 0.5, 0.0}
+\newcommand{\docutilsrolegreen}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\docutilsrolered}[1]{\color{red}#1\normalcolor}
+
+\newcommand{\green}[1]{\color{darkgreen}#1\normalcolor}
+\newcommand{\red}[1]{\color{red}#1\normalcolor}
diff --git a/talk/scipyindia2014/talk.rst b/talk/scipyindia2014/talk.rst
--- a/talk/scipyindia2014/talk.rst
+++ b/talk/scipyindia2014/talk.rst
@@ -65,7 +65,7 @@
 
 * With CFFI you can call C code from Python and expose Python functions to C 
 
-  - this means that you can create your own C API in pure python !
+  - This means you can create your own C API in pure python !
 
 * It is now possible to embed PyPy in a C application (uWSGI)
 

From noreply at buildbot.pypy.org  Thu Dec  4 08:03:18 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu,  4 Dec 2014 08:03:18 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Progress
Message-ID: <20141204070318.D627A1C100F@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: extradoc
Changeset: r5476:dc90c5848fd0
Date: 2014-12-04 07:28 +0530
http://bitbucket.org/pypy/extradoc/changeset/dc90c5848fd0/

Log:	Progress

diff --git a/talk/scipyindia2014/talk.rst b/talk/scipyindia2014/talk.rst
--- a/talk/scipyindia2014/talk.rst
+++ b/talk/scipyindia2014/talk.rst
@@ -56,16 +56,25 @@
 
 * Rewrite your code in some subset/dialect of Python like Numba
 
-* Just write Python and use PyPy, pay only the cost of what you use
+* Just write Python and use PyPy, only pay the cost of what you use
 
-PyPy And C
-----------
+PyPy and C (1/2)
+----------------
 
-* PyPy is pretty good at interacting with C code with cffi
+* PyPy is pretty good at interacting with C code now, thanks to CFFI
+
+* CFFI is the easiest tool to I've used so far
+
+* Very fast on PyPy, fast enough on CPython
+
+* Used by NumPyPy
 
 * With CFFI you can call C code from Python and expose Python functions to C 
 
-  - This means you can create your own C API in pure python !
+  - This means you can create your own C API in pure Python !
+
+PyPy and C (2/2)
+----------------
 
 * It is now possible to embed PyPy in a C application (uWSGI)
 

From noreply at buildbot.pypy.org  Thu Dec  4 08:13:50 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 08:13:50 +0100 (CET)
Subject: [pypy-commit] pypy default: test any/all in test_zjit
Message-ID: <20141204071350.BE3901C07BB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74809:0961d3cfbdde
Date: 2014-12-04 02:09 -0500
http://bitbucket.org/pypy/pypy/changeset/0961d3cfbdde/

Log:	test any/all in test_zjit

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -277,21 +277,45 @@
 
     def define_any():
         return """
-        a = [0,0,0,0,0,0,0,0,0,0,0]
-        a[8] = -12
-        b = a + a
-        any(b)
+        a = [0,0,0,0,0,0,0,1,0,0,0]
+        any(a)
         """
 
     def test_any(self):
         result = self.run("any")
         assert result == 1
-        py.test.skip("don't run for now")
-        self.check_simple_loop({"raw_load": 2, "float_add": 1,
-                                "int_and": 1, "int_add": 1,
-                                'cast_float_to_int': 1,
-                                "int_ge": 1, "jump": 1,
-                                "guard_false": 2, 'arraylen_gc': 1})
+        self.check_trace_count(1)
+        self.check_simple_loop({
+            'cast_float_to_int': 1,
+            'guard_false': 2,
+            'guard_not_invalidated': 1,
+            'int_add': 2,
+            'int_and': 1,
+            'int_ge': 1,
+            'jump': 1,
+            'raw_load': 1,
+        })
+
+    def define_all():
+        return """
+        a = [1,1,1,1,1,1,1,1]
+        all(a)
+        """
+
+    def test_all(self):
+        result = self.run("all")
+        assert result == 1
+        self.check_simple_loop({
+            'cast_float_to_int': 1,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'guard_true': 1,
+            'int_add': 2,
+            'int_and': 1,
+            'int_ge': 1,
+            'jump': 1,
+            'raw_load': 1,
+        })
 
     def define_already_forced():
         return """

From noreply at buildbot.pypy.org  Thu Dec  4 08:13:52 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 08:13:52 +0100 (CET)
Subject: [pypy-commit] pypy default: whitespace
Message-ID: <20141204071352.09F031C07BB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74810:092797de8785
Date: 2014-12-04 02:13 -0500
http://bitbucket.org/pypy/pypy/changeset/092797de8785/

Log:	whitespace

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -286,7 +286,6 @@
     def descr_hash(self, space):
         return space.wrap(self._compute_hash(space, 0x345678))
 
-
     def descr_str(self, space):
         if self.fields:
             return space.str(self.descr_get_descr(space))
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -392,6 +392,7 @@
         t5 = dtype([('x', '<f4'), ('y', '<i4')])
         t6 = dtype([('y', '<i4'), ('x', '<f4')])
         assert hash(t5) != hash(t6)
+
     def test_pickle(self):
         import numpy as np
         from numpypy import array, dtype

From noreply at buildbot.pypy.org  Thu Dec  4 09:48:59 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Thu,  4 Dec 2014 09:48:59 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Add new topics
Message-ID: <20141204084859.8C5651C100F@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: extradoc
Changeset: r5477:b636f7bce759
Date: 2014-12-04 14:18 +0530
http://bitbucket.org/pypy/extradoc/changeset/b636f7bce759/

Log:	Add new topics

diff --git a/talk/scipyindia2014/talk.rst b/talk/scipyindia2014/talk.rst
--- a/talk/scipyindia2014/talk.rst
+++ b/talk/scipyindia2014/talk.rst
@@ -96,4 +96,15 @@
 
 * ~80% of the numpy tests are passing
 
-* XXX
+* Most of numpy is there
+
+* XXX is missing
+
+NumPyPy performance
+-------------------
+
+PyMetabiosis
+------------
+
+JitPy
+-----

From noreply at buildbot.pypy.org  Thu Dec  4 13:18:14 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 13:18:14 +0100 (CET)
Subject: [pypy-commit] creflect default: clean-ups, small stuff
Message-ID: <20141204121814.1F9A01C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r144:aab3d6122236
Date: 2014-12-04 12:15 +0100
http://bitbucket.org/cffi/creflect/changeset/aab3d6122236/

Log:	clean-ups, small stuff

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,6 @@
 setup(
     ext_modules=[
         Extension(name = 'zeffir',
-                  sources=['zeffir/zeffir.c',
-                           'creflect/creflect_cdecl.c'],
+                  sources=['zeffir/zeffir.c']
         )
     ])
diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -99,7 +99,7 @@
 
 static _crx_type_t *zef_get_void_type(_crx_builder_t *cb)
 {
-    return _zef_primitive(cb, -1, "void", CT_VOID | CT_IS_OPAQUE);
+    return _zef_primitive(cb, -1, "void", CT_VOID);
 }
 
 static _crx_type_t *zef_get_char_type(_crx_builder_t *cb)
@@ -285,7 +285,7 @@
 
 static _crx_type_t *zef_get_unknown_type(_crx_builder_t *cb, const char *name)
 {
-    abort();
+    return _zef_primitive(cb, -1, name, CT_UNKNOWN);
 }
 
 static void zef_complete(_crx_builder_t *cb, _crx_type_t *t,
diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -8,13 +8,13 @@
 #define CT_ARRAY             32    /* array */
 #define CT_STRUCT            64    /* struct */
 #define CT_UNION            128    /* union */
-#define CT_FUNCTIONPTR      256    /* pointer to function */
+#define CT_UNKNOWN          256    /* unknown type */
 #define CT_VOID             512    /* void */
 
 /* other flags that may also be set in addition to the base flag: */
 #define CT_CAST_ANYTHING         1024    /* 'char *' and 'void *' only */
 #define CT_PRIMITIVE_FITS_LONG   2048
-#define CT_IS_OPAQUE             4096
+//#define CT_IS_OPAQUE           4096    /* == (ct_size < 0) */
 #define CT_IS_ENUM               8192
 #define CT_IS_PTR_TO_OWNED      16384
 #define CT_IS_LONGDOUBLE        65536
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -86,6 +86,8 @@
 
     lib = lib_create(path);
     Py_DECREF(path);
+    if (lib == NULL)
+        return NULL;
 
     if (load_creflect_main(self, lib) < 0) {
         Py_DECREF(lib);
@@ -108,22 +110,53 @@
     return Py_None;
 }
 
+static CTypeDescrObject *_ffi_type(ZefFFIObject *ffi, PyObject *arg)
+{
+    /* Returns the CTypeDescrObject from the user-supplied 'arg'.
+       Does not return a new reference!
+    */
+    if (PyString_Check(arg)) {
+        PyObject *x = PyDict_GetItem(ffi->types_dict, arg);
+        if (x != NULL && CTypeDescr_Check(x))
+            return (CTypeDescrObject *)x;
+
+        CTypeDescrObject *ct = parse_c_decl(ffi, PyString_AS_STRING(arg));
+        if (ct == NULL)
+            return NULL;
+
+        x = (PyObject *)ct;
+        if (PyDict_SetItem(ffi->types_dict, arg, x) < 0)
+            ct = NULL;
+        Py_DECREF(x);
+        return ct;
+    }
+    else if (CTypeDescr_Check(arg)) {
+        return (CTypeDescrObject *)arg;
+    }
+    else {
+        PyErr_SetString(PyExc_TypeError, "expected a string or a CType object");
+        return NULL;
+    }
+}
+
+static PyObject *ffi_sizeof(ZefFFIObject *self, PyObject *arg)
+{
+    CTypeDescrObject *ct = _ffi_type(self, arg);
+    if (ct == NULL)
+        return NULL;
+
+    return PyInt_FromSsize_t(ct->ct_size);
+}
+
 static PyObject *ffi_typeof(ZefFFIObject *self, PyObject *arg)
 {
     if (!PyString_Check(arg)) {
-        PyErr_SetString(PyExc_TypeError, "XXX");
+        PyErr_SetString(PyExc_TypeError, "expected a string");
         return NULL;
     }
 
-    PyObject *x = PyDict_GetItem(self->types_dict, arg);
-    if (x != NULL) {
-        Py_INCREF(x);
-        return x;
-    }
-
-    x = (PyObject *)parse_c_decl(self, PyString_AS_STRING(arg));
-    if (x != NULL)
-        PyDict_SetItem(self->types_dict, arg, x);
+    PyObject *x = (PyObject *)_ffi_type(self, arg);
+    Py_XINCREF(x);
     return x;
 }
 
@@ -131,6 +164,7 @@
     {"close_library", ffi_close_library,         METH_VARARGS | METH_STATIC},
     {"load_library",  (PyCFunction)ffi_load_library,
                                                  METH_VARARGS | METH_KEYWORDS},
+    {"sizeof",        (PyCFunction)ffi_sizeof,   METH_O},
     {"typeof",        (PyCFunction)ffi_typeof,   METH_O},
     {NULL}
 };
diff --git a/zeffir/test/function.crx b/zeffir/test/function.crx
--- a/zeffir/test/function.crx
+++ b/zeffir/test/function.crx
@@ -1,11 +1,11 @@
+int simple_function(int x)
+{
+    return x + 1;
+}
+
+
 // CREFLECT: start
 
 int simple_function(int);
 
 // CREFLECT: end
-
-
-int simple_function(int x)
-{
-    return x + 1;
-}
diff --git a/zeffir/test/test_ctype.py b/zeffir/test/test_ctype.py
--- a/zeffir/test/test_ctype.py
+++ b/zeffir/test/test_ctype.py
@@ -10,6 +10,8 @@
 def test_typeof_char():
     ffi = support.new_ffi()
     assert repr(ffi.typeof("char")) == "<ctype 'char'>"
+    assert ffi.sizeof("char") == 1
+    assert ffi.sizeof(ffi.typeof("char")) == 1
 
 def test_typeof_bool():
     ffi = support.new_ffi()

From noreply at buildbot.pypy.org  Thu Dec  4 13:18:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 13:18:15 +0100 (CET)
Subject: [pypy-commit] creflect default: Finding the closest integer type
 matching the size: move that logic out
Message-ID: <20141204121815.3C68B1C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r145:e7243eb19cf3
Date: 2014-12-04 13:15 +0100
http://bitbucket.org/cffi/creflect/changeset/e7243eb19cf3/

Log:	Finding the closest integer type matching the size: move that logic
	out of creflect_debug_print.c, because it is generally useful

diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -77,8 +77,16 @@
 #undef _CRX_SELF
 
 
-#define _CRX_INT_TYPE(cb, expr, guessname)                               \
-    _crx_int_type(cb, expr > 0, sizeof(expr), expr == 1, guessname)
+enum {
+    _crx_sc_char      = 1,
+    _crx_sc_short     = 2,
+    _crx_sc_int       = 3,
+    _crx_sc_long      = 4,
+    _crx_sc_long_long = 5,
+};
+
+#define _CRX_INT_TYPE(cb, expr, sizeclass)                              \
+    _crx_int_type(cb, expr > 0, sizeof(expr), expr == 1, sizeclass)
 
 #define _CRX_INT_CONST(cb, expr, vp, pn)                 \
     _crx_int_const(cb, vp, pn,                    \
@@ -102,14 +110,28 @@
 __attribute__((unused))
 static _crx_type_t *_crx_int_type(_crx_builder_t *cb, int expr_positive,
                                   size_t size_of_expr, int expr_equal_one,
-                                  const char *guessname)
+                                  int sizeclass)
 {
+    if (size_of_expr == 1 && expr_equal_one)
+        return cb->get_bool_type(cb);
+
+    static size_t all_sizes[] = { 0, sizeof(char), sizeof(short), sizeof(int),
+                                  sizeof(long), sizeof(long long), (size_t)-1 };
+    while (all_sizes[sizeclass] != size_of_expr) {
+        if (all_sizes[sizeclass] > size_of_expr)
+            sizeclass--;
+        else
+            sizeclass++;
+    }
+    if (sizeclass < 1) sizeclass = 1;
+    if (sizeclass > 5) sizeclass = 5;
+
+    static const char *all_names[] = { NULL, "char", "short", "int",
+                                       "long", "long long" };
     if (!expr_positive)
-        return cb->get_signed_type(cb, size_of_expr, guessname);
-    else if (size_of_expr == 1 && expr_equal_one)
-        return cb->get_bool_type(cb);
+        return cb->get_signed_type(cb, size_of_expr, all_names[sizeclass]);
     else
-        return cb->get_unsigned_type(cb, size_of_expr, guessname);
+        return cb->get_unsigned_type(cb, size_of_expr, all_names[sizeclass]);
 }
 
 __attribute__((unused))
diff --git a/creflect/creflect_debug_print.c b/creflect/creflect_debug_print.c
--- a/creflect/creflect_debug_print.c
+++ b/creflect/creflect_debug_print.c
@@ -67,63 +67,52 @@
     return newtype("_Bool");
 }
 
+static void badsize(const char *g, size_t sz)
+{
+    printf("type '%s' has incorrect size specification: %zd\n", g, sz);
+    abort();
+}
+
 static _crx_type_t *tst_get_signed_type(_crx_builder_t *cb, size_t sz,
                                         const char *g)
 {
-    int skip = 0;
-    if (sizeof(long) == sizeof(long long))
-        if (strcmp(g, "long long") == 0)
-            skip = 4;
-    if (sizeof(int) == sizeof(long))
-        if (strcmp(g, "long") == 0 || strcmp(g, "long long") == 0)
-            skip = 3;
+#define TT(in_name, out_name)                           \
+    if (strcmp(g, #in_name) == 0) {                     \
+        if (sz != sizeof(out_name)) badsize(g, sz);     \
+        return newtype(#out_name);                      \
+    }
 
-#define TT(name)   if (--skip && sz == sizeof(name)) { return newtype(#name); }
-    TT(signed char);
-    TT(short);
-    TT(int);
-    TT(long);
-    TT(long long);
+    TT(char, signed char)
+    TT(short, short)
+    TT(int, int)
+    TT(long, long)
+    TT(long long, long long)
 
-    printf("cannot find signed type with %zd bytes\n", sz);
+    printf("cannot find signed type '%s'\n", g);
     abort();
 }
 
 static _crx_type_t *tst_get_unsigned_type(_crx_builder_t *cb, size_t sz,
                                           const char *g)
 {
-    int skip = 0;
-    if (sizeof(long) == sizeof(long long))
-        if (strcmp(g, "long long") == 0)
-            skip = 4;
-    if (sizeof(int) == sizeof(long))
-        if (strcmp(g, "long") == 0 || strcmp(g, "long long") == 0)
-            skip = 3;
+    TT(char, unsigned char)
+    TT(short, unsigned short)
+    TT(int, unsigned int)
+    TT(long, unsigned long)
+    TT(long long, unsigned long long)
 
-    TT(unsigned char);
-    TT(unsigned short);
-    TT(unsigned int);
-    TT(unsigned long);
-    TT(unsigned long long);
-
-    printf("cannot find unsigned type with %zd bytes\n", sz);
+    printf("cannot find unsigned type '%s'\n", g);
     abort();
 }
 
 static _crx_type_t *tst_get_float_type(_crx_builder_t *cb, size_t sz,
                                        const char *g)
 {
-    int skip = 0;
-    if (sizeof(double) == sizeof(long double))
-        if (strcmp(g, "long double") == 0)
-            skip = 2;
+    TT(float, float)
+    TT(double, double)
+    TT(long double, long double)
 
-    TT(float);
-    TT(double);
-    TT(long double);
-#undef TT
-
-    printf("cannot find float type with %zd bytes\n", sz);
+    printf("cannot find float type '%s'\n", g);
     abort();
 }
 
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -134,6 +134,17 @@
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
 
+    def get_simplified_integer_type(self):
+        assert self.is_integer_type()
+        name = self.name
+        if name.startswith('unsigned '):
+            name = name[9:]
+        elif name.startswith('signed '):
+            name = name[7:]
+        if name == '_Bool':
+            name = 'char'
+        return name
+
     def inspect_type(self, block, inspect, qualifiers):
         if isinstance(inspect, TypeInspector):
             star_p1 = inspect.fetch_star_p1()
@@ -157,10 +168,9 @@
             if not (qualifiers & CRX_CONST):
                 block.writeline("%s = -1;%s" % (star_p1, comment3))
             if self.is_integer_type():
-                hint = self.name.split()
-                if hint[0] in ('signed', 'unsigned'):
-                    hint = hint[1:]
-                expr = '_CRX_INT_TYPE(cb, %s, "%s")' % (star_p1, " ".join(hint))
+                hint = self.get_simplified_integer_type()
+                hint = hint.replace(' ', '_')
+                expr = '_CRX_INT_TYPE(cb, %s, _crx_sc_%s)' % (star_p1, hint)
             elif self.is_char_type():
                 errmsg = "numeric type '%s' is not a char" % (
                     inspect.get_comment_type(0, False),)
@@ -189,12 +199,16 @@
                                          inspect.varname,
                                          qualtype.get_c_name()))
                 block.writeline("(void)p1;")
-            if self.is_signed_type():
+            if self.name == '_Bool':
+                expr = 'cb->get_bool_type(cb)'
+            elif self.is_signed_type():
+                hint = self.get_simplified_integer_type()
                 expr = 'cb->get_signed_type(cb, sizeof(%s), "%s")' % (
-                    self.name, self.name)
+                    self.name, hint)
             elif self.is_unsigned_type():
+                hint = self.get_simplified_integer_type()
                 expr = 'cb->get_unsigned_type(cb, sizeof(%s), "%s")' % (
-                    self.name, self.name)
+                    self.name, hint)
             elif self.is_char_type():
                 expr = 'cb->get_char_type(cb)'
             elif self.is_float_type():
diff --git a/creflect/test/codegen/003.c b/creflect/test/codegen/003.c
--- a/creflect/test/codegen/003.c
+++ b/creflect/test/codegen/003.c
@@ -11,7 +11,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, *p1, "int");
+        t1 = _CRX_INT_TYPE(cb, *p1, _crx_sc_int);
         cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = int
     }
diff --git a/creflect/test/codegen/003b.c b/creflect/test/codegen/003b.c
--- a/creflect/test/codegen/003b.c
+++ b/creflect/test/codegen/003b.c
@@ -11,7 +11,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, *p1, "long");
+        t1 = _CRX_INT_TYPE(cb, *p1, _crx_sc_long);
         cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = long
     }
diff --git a/creflect/test/codegen/003c.c b/creflect/test/codegen/003c.c
--- a/creflect/test/codegen/003c.c
+++ b/creflect/test/codegen/003c.c
@@ -11,7 +11,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, *p1, "long long");
+        t1 = _CRX_INT_TYPE(cb, *p1, _crx_sc_long_long);
         cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = long long
     }
diff --git a/creflect/test/codegen/003d.c b/creflect/test/codegen/003d.c
--- a/creflect/test/codegen/003d.c
+++ b/creflect/test/codegen/003d.c
@@ -11,7 +11,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, *p1, "char");
+        t1 = _CRX_INT_TYPE(cb, *p1, _crx_sc_char);
         cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = signed char
     }
diff --git a/creflect/test/codegen/003f.c b/creflect/test/codegen/003f.c
--- a/creflect/test/codegen/003f.c
+++ b/creflect/test/codegen/003f.c
@@ -11,7 +11,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, *p1, "long long");
+        t1 = _CRX_INT_TYPE(cb, *p1, _crx_sc_long_long);
         cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = unsigned long long
     }
diff --git a/creflect/test/codegen/003h.c b/creflect/test/codegen/003h.c
--- a/creflect/test/codegen/003h.c
+++ b/creflect/test/codegen/003h.c
@@ -15,7 +15,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, *p1, "long long");
+        t1 = _CRX_INT_TYPE(cb, *p1, _crx_sc_long_long);
         cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = int
     }
diff --git a/creflect/test/codegen/003i.c b/creflect/test/codegen/003i.c
--- a/creflect/test/codegen/003i.c
+++ b/creflect/test/codegen/003i.c
@@ -11,7 +11,7 @@
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
         *p1 = -1;  /* check that 'num_t' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, *p1, "char");
+        t1 = _CRX_INT_TYPE(cb, *p1, _crx_sc_char);
         cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = signed char
     }
diff --git a/creflect/test/codegen/004.c b/creflect/test/codegen/004.c
--- a/creflect/test/codegen/004.c
+++ b/creflect/test/codegen/004.c
@@ -13,7 +13,7 @@
         *p1 = (void *)b;    /* check that 'num_t' is a pointer type */
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
         **p1 = -1;  /* check that '*num_t' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, **p1, "int");
+        t1 = _CRX_INT_TYPE(cb, **p1, _crx_sc_int);
         t2 = cb->get_pointer_type(cb, t1, 0);
         cb->define_type(cb, "num_t", t2, 0);
 #expect TYPEDEF num_t = PTR int
diff --git a/creflect/test/codegen/004b.c b/creflect/test/codegen/004b.c
--- a/creflect/test/codegen/004b.c
+++ b/creflect/test/codegen/004b.c
@@ -15,7 +15,7 @@
         **p1 = (void *)b;    /* check that '*num_t' is a pointer type */
         (void)(***p1 << 1);  /* check that '**num_t' is an integer type */
         ***p1 = -1;  /* check that '**num_t' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, ***p1, "int");
+        t1 = _CRX_INT_TYPE(cb, ***p1, _crx_sc_int);
         t2 = cb->get_pointer_type(cb, t1, 0);
         t3 = cb->get_pointer_type(cb, t2, 0);
         cb->define_type(cb, "num_t", t3, 0);
diff --git a/creflect/test/codegen/005.c b/creflect/test/codegen/005.c
--- a/creflect/test/codegen/005.c
+++ b/creflect/test/codegen/005.c
@@ -15,7 +15,7 @@
         }
         (void)(**p1 << 1);  /* check that 'foo_t[]' is an integer type */
         **p1 = -1;  /* check that 'foo_t[]' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, **p1, "int");
+        t1 = _CRX_INT_TYPE(cb, **p1, _crx_sc_int);
         t2 = cb->get_array_type(cb, t1, sizeof(*p1) / sizeof(**p1));
         cb->define_type(cb, "foo_t", t2, 0);
 #expect TYPEDEF foo_t = ARRAY[27] int
diff --git a/creflect/test/codegen/005b.c b/creflect/test/codegen/005b.c
--- a/creflect/test/codegen/005b.c
+++ b/creflect/test/codegen/005b.c
@@ -17,7 +17,7 @@
         **p1 = (void *)b;    /* check that 'foo_t[]' is a pointer type */
         (void)(***p1 << 1);  /* check that '*foo_t[]' is an integer type */
         ***p1 = -1;  /* check that '*foo_t[]' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, ***p1, "int");
+        t1 = _CRX_INT_TYPE(cb, ***p1, _crx_sc_int);
         t2 = cb->get_pointer_type(cb, t1, 0);
         t3 = cb->get_array_type(cb, t2, sizeof(*p1) / sizeof(**p1));
         cb->define_type(cb, "foo_t", t3, 0);
diff --git a/creflect/test/codegen/005c.c b/creflect/test/codegen/005c.c
--- a/creflect/test/codegen/005c.c
+++ b/creflect/test/codegen/005c.c
@@ -17,7 +17,7 @@
         }
         (void)(***p1 << 1);  /* check that '(*foo_t)[]' is an integer type */
         ***p1 = -1;  /* check that '(*foo_t)[]' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, ***p1, "int");
+        t1 = _CRX_INT_TYPE(cb, ***p1, _crx_sc_int);
         t2 = cb->get_array_type(cb, t1, sizeof(**p1) / sizeof(***p1));
         t3 = cb->get_pointer_type(cb, t2, 0);
         cb->define_type(cb, "foo_t", t3, 0);
diff --git a/creflect/test/codegen/005d.c b/creflect/test/codegen/005d.c
--- a/creflect/test/codegen/005d.c
+++ b/creflect/test/codegen/005d.c
@@ -37,7 +37,7 @@
         *********p1 = (void *)b;    /* check that '*(***foo_t[][])[][]' is a pointer type */
         (void)(**********p1 << 1);  /* check that '**(***foo_t[][])[][]' is an integer type */
         **********p1 = -1;  /* check that '**(***foo_t[][])[][]' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, **********p1, "int");
+        t1 = _CRX_INT_TYPE(cb, **********p1, _crx_sc_int);
         t2 = cb->get_pointer_type(cb, t1, 0);
         t3 = cb->get_pointer_type(cb, t2, 0);
         t4 = cb->get_array_type(cb, t3, sizeof(*******p1) / sizeof(********p1));
diff --git a/creflect/test/codegen/006.c b/creflect/test/codegen/006.c
--- a/creflect/test/codegen/006.c
+++ b/creflect/test/codegen/006.c
@@ -11,7 +11,7 @@
         memset(b, -1, sizeof(b));
         p1 = (void *)b;
         (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
-        t1 = _CRX_INT_TYPE(cb, *p1, "int");
+        t1 = _CRX_INT_TYPE(cb, *p1, _crx_sc_int);
         cb->define_type(cb, "num_t", t1, _CRX_CONST);
 #expect TYPEDEF num_t = CONST unsigned int
     }
diff --git a/creflect/test/codegen/006b.c b/creflect/test/codegen/006b.c
--- a/creflect/test/codegen/006b.c
+++ b/creflect/test/codegen/006b.c
@@ -17,7 +17,7 @@
         }
         (void)(**p1 << 1);  /* check that '*num_t' is an integer type */
         **p1 = -1;  /* check that '*num_t' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, **p1, "int");
+        t1 = _CRX_INT_TYPE(cb, **p1, _crx_sc_int);
         t2 = cb->get_pointer_type(cb, t1, 0);
         cb->define_type(cb, "num_t", t2, _CRX_CONST);
 #expect TYPEDEF num_t = CONST PTR int
diff --git a/creflect/test/codegen/006c.c b/creflect/test/codegen/006c.c
--- a/creflect/test/codegen/006c.c
+++ b/creflect/test/codegen/006c.c
@@ -21,7 +21,7 @@
         }
         (void)(***p1 << 1);  /* check that '(*foo_t)[]' is an integer type */
         ***p1 = -1;  /* check that '(*foo_t)[]' is not declared 'const' */
-        t1 = _CRX_INT_TYPE(cb, ***p1, "int");
+        t1 = _CRX_INT_TYPE(cb, ***p1, _crx_sc_int);
         t2 = cb->get_array_type(cb, t1, sizeof(**p1) / sizeof(***p1));
         t3 = cb->get_pointer_type(cb, t2, 0);
         cb->define_type(cb, "foo_t", t3, _CRX_CONST);
diff --git a/creflect/test/codegen/func-001b.c b/creflect/test/codegen/func-001b.c
--- a/creflect/test/codegen/func-001b.c
+++ b/creflect/test/codegen/func-001b.c
@@ -41,7 +41,7 @@
     _crx_qual_type a2[2];
     _crx_qual_type a3[2];
     {
-        t1 = cb->get_unsigned_type(cb, sizeof(unsigned int), "unsigned int");
+        t1 = cb->get_unsigned_type(cb, sizeof(unsigned int), "int");
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
         a1[0].type = t2;
         a1[0].qualifiers = 0;
diff --git a/creflect/test/codegen/struct-001.c b/creflect/test/codegen/struct-001.c
--- a/creflect/test/codegen/struct-001.c
+++ b/creflect/test/codegen/struct-001.c
@@ -17,7 +17,7 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
         p1->aa = -1;  /* check that 'struct foo_s::aa' is not declared 'const' */
-        t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, p1->aa, _crx_sc_int);
         d1[0].name = "aa";
         d1[0].type = t2;
         d1[0].qualifiers = 0;
@@ -32,7 +32,7 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->bb << 1);  /* check that 'struct foo_s::bb' is an integer type */
         p1->bb = -1;  /* check that 'struct foo_s::bb' is not declared 'const' */
-        t3 = _CRX_INT_TYPE(cb, p1->bb, "int");
+        t3 = _CRX_INT_TYPE(cb, p1->bb, _crx_sc_int);
         d1[1].name = "bb";
         d1[1].type = t3;
         d1[1].qualifiers = 0;
diff --git a/creflect/test/codegen/struct-001b.c b/creflect/test/codegen/struct-001b.c
--- a/creflect/test/codegen/struct-001b.c
+++ b/creflect/test/codegen/struct-001b.c
@@ -16,7 +16,7 @@
         memset(b, -1, sizeof(b));
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'struct foo_s::aa' is an integer type */
-        t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, p1->aa, _crx_sc_int);
         d1[0].name = "aa";
         d1[0].type = t2;
         d1[0].qualifiers = _CRX_CONST;
diff --git a/creflect/test/codegen/struct-003.c b/creflect/test/codegen/struct-003.c
--- a/creflect/test/codegen/struct-003.c
+++ b/creflect/test/codegen/struct-003.c
@@ -18,7 +18,7 @@
         p1->aa = (void *)b;    /* check that 'struct foo_s::aa' is a pointer type */
         (void)(*p1->aa << 1);  /* check that '*struct foo_s::aa' is an integer type */
         *p1->aa = -1;  /* check that '*struct foo_s::aa' is not declared 'const' */
-        t2 = _CRX_INT_TYPE(cb, *p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, *p1->aa, _crx_sc_int);
         t3 = cb->get_pointer_type(cb, t2, 0);
         d1[0].name = "aa";
         d1[0].type = t3;
diff --git a/creflect/test/codegen/struct-004.c b/creflect/test/codegen/struct-004.c
--- a/creflect/test/codegen/struct-004.c
+++ b/creflect/test/codegen/struct-004.c
@@ -20,7 +20,7 @@
         }
         (void)(*p1->aa << 1);  /* check that 'struct foo_s::aa[]' is an integer type */
         *p1->aa = -1;  /* check that 'struct foo_s::aa[]' is not declared 'const' */
-        t2 = _CRX_INT_TYPE(cb, *p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, *p1->aa, _crx_sc_int);
         t3 = cb->get_array_type(cb, t2, sizeof(p1->aa) / sizeof(*p1->aa));
         d1[0].name = "aa";
         d1[0].type = t3;
diff --git a/creflect/test/codegen/struct-005.c b/creflect/test/codegen/struct-005.c
--- a/creflect/test/codegen/struct-005.c
+++ b/creflect/test/codegen/struct-005.c
@@ -14,7 +14,7 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that 'foo_t::aa' is an integer type */
         p1->aa = -1;  /* check that 'foo_t::aa' is not declared 'const' */
-        t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, p1->aa, _crx_sc_int);
         d1[0].name = "aa";
         d1[0].type = t2;
         d1[0].qualifiers = 0;
diff --git a/creflect/test/codegen/struct-005b.c b/creflect/test/codegen/struct-005b.c
--- a/creflect/test/codegen/struct-005b.c
+++ b/creflect/test/codegen/struct-005b.c
@@ -14,7 +14,7 @@
         p1 = (void *)(((char *)b) - o);
         (void)(p1->aa << 1);  /* check that '*foo_p::aa' is an integer type */
         p1->aa = -1;  /* check that '*foo_p::aa' is not declared 'const' */
-        t2 = _CRX_INT_TYPE(cb, p1->aa, "int");
+        t2 = _CRX_INT_TYPE(cb, p1->aa, _crx_sc_int);
         d1[0].name = "aa";
         d1[0].type = t2;
         d1[0].qualifiers = 0;

From noreply at buildbot.pypy.org  Thu Dec  4 13:18:16 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 13:18:16 +0100 (CET)
Subject: [pypy-commit] creflect default: type definitions
Message-ID: <20141204121816.4F5DC1C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r146:910d0436c9ff
Date: 2014-12-04 13:18 +0100
http://bitbucket.org/cffi/creflect/changeset/910d0436c9ff/

Log:	type definitions

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -20,9 +20,6 @@
     PyObject *result;
     char *p;
 
-    if (PyErr_Occurred())
-        return NULL;
-
     base_name_len = (ct != NULL ? strlen(ct->ct_name) : 0);
     extra_name_len = strlen(extra_text);
     result = PyString_FromStringAndSize(NULL, base_name_len + extra_name_len);
@@ -72,6 +69,9 @@
 static _crx_type_t *_zef_primitive(_crx_builder_t *cb, Py_ssize_t size,
                                    const char *name, int flags)
 {
+    if (PyErr_Occurred())
+        return NULL;
+
     PyObject *name_obj;
     CTypeDescrObject *ct;
 
@@ -165,6 +165,9 @@
 static _crx_type_t *zef_get_pointer_type(_crx_builder_t *cb,
                                          _crx_type_t *totype, int toquals)
 {
+    if (PyErr_Occurred())
+        return NULL;
+
     const char *extra;
     PyObject *name_obj;
     CTypeDescrObject *ct;
@@ -205,6 +208,9 @@
 static _crx_type_t *zef_get_array_type(_crx_builder_t *cb, _crx_type_t *ctitem,
                                        size_t len)
 {
+    if (PyErr_Occurred())
+        return NULL;
+
     CTypeDescrObject *ct;
     PyObject *name_obj;
     char extra_text[32];
@@ -304,7 +310,11 @@
 static void zef_define_type(_crx_builder_t *cb, const char *name,
                             _crx_type_t *type, int quals)
 {
-    abort();
+    if (PyErr_Occurred())
+        return;
+
+    PyObject *types_dict = ((zeffir_builder_t *)cb)->types_dict;
+    PyDict_SetItemString(types_dict, name, (PyObject *)type);
 }
 
 static void zef_define_var(_crx_builder_t *cb, const char *name,
@@ -317,6 +327,9 @@
                             _crx_type_t *ret, _crx_qual_type args[], int nargs,
                             _crx_trampoline0_fn trampl, void *directcall)
 {
+    if (PyErr_Occurred())
+        return;
+
     PyObject *l_dict = ((zeffir_builder_t *)cb)->l_dict;
     PyObject *l_libname_obj = ((zeffir_builder_t *)cb)->lib->l_libname_obj;
 
@@ -333,6 +346,9 @@
 static void zef_define_num_const(_crx_builder_t *cb, const char *name,
                                  _crx_type_t *ct, _crx_num_const_t *value)
 {
+    if (PyErr_Occurred())
+        return;
+
     PyObject *l_dict = ((zeffir_builder_t *)cb)->l_dict;
 
     assert(ct->ct_flags & CT_PRIMITIVE_ANY);
diff --git a/zeffir/test/ctype.crx b/zeffir/test/ctype.crx
new file mode 100644
--- /dev/null
+++ b/zeffir/test/ctype.crx
@@ -0,0 +1,9 @@
+typedef long long foo_t;
+
+
+
+// CREFLECT: start
+
+typedef int foo_t;
+
+// CREFLECT: end
diff --git a/zeffir/test/test_ctype.py b/zeffir/test/test_ctype.py
--- a/zeffir/test/test_ctype.py
+++ b/zeffir/test/test_ctype.py
@@ -65,3 +65,11 @@
     assert repr(ffi.typeof("int [ ]")) == "<ctype 'int[]'>"
     assert repr(ffi.typeof("int * [ ]")) == "<ctype 'int *[]'>"
     assert repr(ffi.typeof("int ( * ) [ ]")) == "<ctype 'int(*)[]'>"
+
+def test_simple_typedef():
+    ffi, lib = support.compile_and_open('ctype')
+    assert ffi.sizeof("foo_t") == ffi.sizeof("long long")
+    if ffi.sizeof("long") == ffi.sizeof("long long"):
+        assert repr(ffi.typeof("foo_t")) == "<ctype 'long'>"
+    else:
+        assert repr(ffi.typeof("foo_t")) == "<ctype 'long long'>"

From noreply at buildbot.pypy.org  Thu Dec  4 13:21:50 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 13:21:50 +0100 (CET)
Subject: [pypy-commit] creflect default: a test
Message-ID: <20141204122150.58F581C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r147:6b080d10e199
Date: 2014-12-04 13:22 +0100
http://bitbucket.org/cffi/creflect/changeset/6b080d10e199/

Log:	a test

diff --git a/creflect/test/codegen/003j.c b/creflect/test/codegen/003j.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/003j.c
@@ -0,0 +1,18 @@
+typedef _Bool num_t;
+
+# ____________________________________________________________
+
+void test003j(_crx_builder_t *cb)
+{
+    _crx_type_t *t1;
+    {
+        num_t *p1;
+        char b[sizeof(*p1)];
+        p1 = (void *)b;
+        (void)(*p1 << 1);  /* check that 'num_t' is an integer type */
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
+        t1 = _CRX_INT_TYPE(cb, *p1, _crx_sc_char);
+        cb->define_type(cb, "num_t", t1, 0);
+#expect TYPEDEF num_t = _Bool
+    }
+}

From noreply at buildbot.pypy.org  Thu Dec  4 13:28:21 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 13:28:21 +0100 (CET)
Subject: [pypy-commit] creflect default: guess the float types,
	like we do the integer types
Message-ID: <20141204122821.B57111C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r148:3471b7c2e683
Date: 2014-12-04 13:28 +0100
http://bitbucket.org/cffi/creflect/changeset/3471b7c2e683/

Log:	guess the float types, like we do the integer types

diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -83,13 +83,20 @@
     _crx_sc_int       = 3,
     _crx_sc_long      = 4,
     _crx_sc_long_long = 5,
+
+    _crx_sc_float       = 1,
+    _crx_sc_double      = 2,
+    _crx_sc_long_double = 3,
 };
 
 #define _CRX_INT_TYPE(cb, expr, sizeclass)                              \
     _crx_int_type(cb, expr > 0, sizeof(expr), expr == 1, sizeclass)
 
-#define _CRX_INT_CONST(cb, expr, vp, pn)                 \
-    _crx_int_const(cb, vp, pn,                    \
+#define _CRX_FLOAT_TYPE(cb, expr, sizeclass)            \
+    _crx_float_type(cb, sizeof(expr), sizeclass)
+
+#define _CRX_INT_CONST(cb, expr, vp, pn)                \
+    _crx_int_const(cb, vp, pn,                          \
         "integer constant '" #expr "' is too large",    \
         !(((expr) * 0 + 4) << (sizeof(int)*8-2)),       \
         !(((expr) * 0L + 4L) << (sizeof(long)*8-2)),    \
@@ -98,9 +105,9 @@
         (expr) == (long long)(expr),                    \
         (unsigned long long)(expr))
 
-#define _CRX_CHAR_CONST(cb, expr, vp)                                    \
-    _crx_char_const(cb, vp,                                       \
-        "char constant '" #expr "' is too large",                       \
+#define _CRX_CHAR_CONST(cb, expr, vp)                                     \
+    _crx_char_const(cb, vp,                                               \
+        "char constant '" #expr "' is too large",                         \
         (expr) == (signed char)(expr) || (expr) == (unsigned char)(expr), \
         (char)(expr))
 
@@ -135,6 +142,25 @@
 }
 
 __attribute__((unused))
+static _crx_type_t *_crx_float_type(_crx_builder_t *cb, size_t size_of_expr,
+                                    int sizeclass)
+{
+    static size_t all_sizes[] = { 0, sizeof(float), sizeof(double),
+                                  sizeof(long double), (size_t)-1 };
+    while (all_sizes[sizeclass] != size_of_expr) {
+        if (all_sizes[sizeclass] > size_of_expr)
+            sizeclass--;
+        else
+            sizeclass++;
+    }
+    if (sizeclass < 1) sizeclass = 1;
+    if (sizeclass > 3) sizeclass = 3;
+
+    static const char *all_names[] = { NULL, "float", "double", "long double" };
+    return cb->get_float_type(cb, size_of_expr, all_names[sizeclass]);
+}
+
+__attribute__((unused))
 static _crx_type_t *_crx_int_const(_crx_builder_t *cb, _crx_num_const_t *vp,
                                    int preference_number,
                                    const char *toobig,
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -186,8 +186,8 @@
                 block.writeline('    cb->error(cb, "%s");' % (errmsg,))
                 block.writeline("    return;")
                 block.writeline("}")
-                expr = 'cb->get_float_type(cb, sizeof(%s), "%s")' % (
-                    star_p1, self.name)
+                hint = self.name.replace(' ', '_')
+                expr = '_CRX_FLOAT_TYPE(cb, %s, _crx_sc_%s)' % (star_p1, hint)
             else:
                 raise AssertionError
         else:
diff --git a/creflect/test/codegen/003g.c b/creflect/test/codegen/003g.c
--- a/creflect/test/codegen/003g.c
+++ b/creflect/test/codegen/003g.c
@@ -16,7 +16,7 @@
             cb->error(cb, "numeric type 'num_t' is an integer, not a float");
             return;
         }
-        t1 = cb->get_float_type(cb, sizeof(*p1), "double");
+        t1 = _CRX_FLOAT_TYPE(cb, *p1, _crx_sc_double);
         cb->define_type(cb, "num_t", t1, 0);
 #expect TYPEDEF num_t = double
     }
diff --git a/creflect/test/codegen/003k.c b/creflect/test/codegen/003k.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/003k.c
@@ -0,0 +1,27 @@
+typedef double num_t;
+
+# ____________________________________________________________
+
+#define num_t  float
+
+# ____________________________________________________________
+
+void test003k(_crx_builder_t *cb)
+{
+    _crx_type_t *t1;
+    {
+        num_t *p1;
+        char b[sizeof(*p1)];
+        memset(b, 0, sizeof(b));
+        p1 = (void *)b;
+        (void)(*p1 / 2.0);  /* check that 'num_t' is a numeric type */
+        *p1 = -1;  /* check that 'num_t' is not declared 'const' */
+        if ((1 + *p1 * 0) / 2 == 0) {
+            cb->error(cb, "numeric type 'num_t' is an integer, not a float");
+            return;
+        }
+        t1 = _CRX_FLOAT_TYPE(cb, *p1, _crx_sc_double);
+        cb->define_type(cb, "num_t", t1, 0);
+#expect TYPEDEF num_t = float
+    }
+}

From noreply at buildbot.pypy.org  Thu Dec  4 13:30:59 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 13:30:59 +0100 (CET)
Subject: [pypy-commit] creflect default: add passing tests
Message-ID: <20141204123059.1DA911C1147@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r149:bee544ef3a06
Date: 2014-12-04 13:31 +0100
http://bitbucket.org/cffi/creflect/changeset/bee544ef3a06/

Log:	add passing tests

diff --git a/zeffir/test/ctype.crx b/zeffir/test/ctype.crx
--- a/zeffir/test/ctype.crx
+++ b/zeffir/test/ctype.crx
@@ -1,9 +1,14 @@
 typedef long long foo_t;
-
+typedef char rfoo_t;
+typedef long double bar_t;
+typedef float rbar_t;
 
 
 // CREFLECT: start
 
 typedef int foo_t;
+typedef long long rfoo_t;
+typedef float bar_t;
+typedef long double rbar_t;
 
 // CREFLECT: end
diff --git a/zeffir/test/test_ctype.py b/zeffir/test/test_ctype.py
--- a/zeffir/test/test_ctype.py
+++ b/zeffir/test/test_ctype.py
@@ -73,3 +73,15 @@
         assert repr(ffi.typeof("foo_t")) == "<ctype 'long'>"
     else:
         assert repr(ffi.typeof("foo_t")) == "<ctype 'long long'>"
+    #
+    assert ffi.sizeof("rfoo_t") == ffi.sizeof("char")
+    assert repr(ffi.typeof("rfoo_t")) == "<ctype 'signed char'>"
+    #
+    assert ffi.sizeof("bar_t") == ffi.sizeof("long double")
+    if ffi.sizeof("long double") == ffi.sizeof("double"):
+        assert repr(ffi.typeof("bar_t")) == "<ctype 'double'>"
+    else:
+        assert repr(ffi.typeof("bar_t")) == "<ctype 'long double'>"
+    #
+    assert ffi.sizeof("rbar_t") == ffi.sizeof("float")
+    assert repr(ffi.typeof("rbar_t")) == "<ctype 'float'>"

From noreply at buildbot.pypy.org  Thu Dec  4 19:12:48 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 19:12:48 +0100 (CET)
Subject: [pypy-commit] creflect default: unknown types
Message-ID: <20141204181248.EA8261C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r150:121223620cdd
Date: 2014-12-04 16:08 +0100
http://bitbucket.org/cffi/creflect/changeset/121223620cdd/

Log:	unknown types

diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -134,7 +134,7 @@
         return (CTypeDescrObject *)arg;
     }
     else {
-        PyErr_SetString(PyExc_TypeError, "expected a string or a CType object");
+        PyErr_SetString(PyExc_TypeError, "expected a string or a ctype object");
         return NULL;
     }
 }
@@ -145,6 +145,11 @@
     if (ct == NULL)
         return NULL;
 
+    if (ct->ct_size < 0) {
+        PyErr_Format(PyExc_ValueError, "don't know the size of ctype '%s': "
+                     "incomplete type", ct->ct_name);
+        return NULL;
+    }
     return PyInt_FromSsize_t(ct->ct_size);
 }
 
diff --git a/zeffir/test/ctype.crx b/zeffir/test/ctype.crx
--- a/zeffir/test/ctype.crx
+++ b/zeffir/test/ctype.crx
@@ -11,4 +11,6 @@
 typedef float bar_t;
 typedef long double rbar_t;
 
+typedef ... unknown_t;
+
 // CREFLECT: end
diff --git a/zeffir/test/test_ctype.py b/zeffir/test/test_ctype.py
--- a/zeffir/test/test_ctype.py
+++ b/zeffir/test/test_ctype.py
@@ -1,3 +1,4 @@
+import py
 import support
 
 
@@ -85,3 +86,8 @@
     #
     assert ffi.sizeof("rbar_t") == ffi.sizeof("float")
     assert repr(ffi.typeof("rbar_t")) == "<ctype 'float'>"
+
+def test_unknown_type():
+    ffi, lib = support.compile_and_open('ctype')
+    assert repr(ffi.typeof("unknown_t")) == "<ctype '$unknown_t'>"
+    py.test.raises(ValueError, ffi.sizeof, "unknown_t")

From noreply at buildbot.pypy.org  Thu Dec  4 19:12:50 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 19:12:50 +0100 (CET)
Subject: [pypy-commit] creflect default: progress
Message-ID: <20141204181250.4E04B1C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r151:558efbc52db3
Date: 2014-12-04 17:49 +0100
http://bitbucket.org/cffi/creflect/changeset/558efbc52db3/

Log:	progress

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -286,7 +286,20 @@
 
 static _crx_qual_type zef_get_user_type(_crx_builder_t *cb, const char *name)
 {
-    abort();
+    PyObject *name_obj;
+    _crx_qual_type result = { NULL, 0 };
+
+    if (PyErr_Occurred())
+        return result;
+
+    name_obj = combine_type_name(cb, NULL, name);
+    if (name_obj == NULL)
+        return result;
+
+    result.type = get_cached_type(cb, name_obj);
+    if (result.type == NULL)
+        PyErr_Format(ZefError, "undefined type: '%s'", name);
+    return result;
 }
 
 static _crx_type_t *zef_get_unknown_type(_crx_builder_t *cb, const char *name)
@@ -363,7 +376,8 @@
 
 static void zef_error(_crx_builder_t *cb, const char *msg)
 {
-    abort();
+    if (!PyErr_Occurred())
+        PyErr_SetString(ZefError, msg);
 }
 
 static int load_creflect_main(ZefFFIObject *ffi, ZefLibObject *lib)
@@ -463,6 +477,25 @@
 
     _crx_qual_type result;
     const char *err = creflect_decl_parser(&builder.cb, str, &result);
+    if (PyErr_Occurred()) {
+        if (PyErr_ExceptionMatches(ZefError)) {
+            PyObject *exc, *val, *tb, *ps;
+            PyErr_Fetch(&exc, &val, &tb);
+            ps = PyObject_Str(val);
+            if (ps != NULL && PyString_Check(ps)) {
+                PyErr_Format(exc, "at pos %zd: %s",
+                             (size_t)(err - str), PyString_AS_STRING(ps));
+                Py_XDECREF(tb);
+                Py_XDECREF(val);
+                Py_XDECREF(exc);
+            }
+            else {
+                PyErr_Restore(exc, val, tb);
+            }
+            Py_XDECREF(ps);
+        }
+        return NULL;
+    }
     if (err != NULL)
         abort();
 
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -146,7 +146,7 @@
         return NULL;
 
     if (ct->ct_size < 0) {
-        PyErr_Format(PyExc_ValueError, "don't know the size of ctype '%s': "
+        PyErr_Format(ZefError, "don't know the size of ctype '%s': "
                      "incomplete type", ct->ct_name);
         return NULL;
     }
diff --git a/zeffir/test/test_ctype.py b/zeffir/test/test_ctype.py
--- a/zeffir/test/test_ctype.py
+++ b/zeffir/test/test_ctype.py
@@ -90,4 +90,15 @@
 def test_unknown_type():
     ffi, lib = support.compile_and_open('ctype')
     assert repr(ffi.typeof("unknown_t")) == "<ctype '$unknown_t'>"
-    py.test.raises(ValueError, ffi.sizeof, "unknown_t")
+    py.test.raises(ffi.error, ffi.sizeof, "unknown_t")
+    py.test.raises(ffi.error, ffi.sizeof, "  unknown_t  ")
+    assert ffi.sizeof("unknown_t*") == ffi.sizeof("void *")
+
+def test_undefined_type():
+    ffi = support.new_ffi()
+    py.test.raises(ffi.error, ffi.typeof, "abcdef_missing_t")
+
+def test_parse_error():
+    ffi = support.new_ffi()
+    e = py.test.raises(ffi.error, ffi.typeof, "int int")
+    assert str(e.value) == "at pos 4: unexpected symbol"
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -53,4 +53,7 @@
     Py_INCREF(ZefError);
     if (PyModule_AddObject(m, "error", ZefError) < 0)
         return;
+
+    if (PyDict_SetItemString(ZefFFI_Type.tp_dict, "error", ZefError) < 0)
+        return;
 }

From noreply at buildbot.pypy.org  Thu Dec  4 19:14:50 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 19:14:50 +0100 (CET)
Subject: [pypy-commit] pypy default: enable axissum test_zjit
Message-ID: <20141204181450.09D5C1C07BB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74811:39760476cedc
Date: 2014-12-04 12:16 -0500
http://bitbucket.org/pypy/pypy/changeset/39760476cedc/

Log:	enable axissum test_zjit

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -188,18 +188,48 @@
     def test_axissum(self):
         result = self.run("axissum")
         assert result == 30
-        py.test.skip("don't run for now")
         # XXX note - the bridge here is fairly crucial and yet it's pretty
         #            bogus. We need to improve the situation somehow.
-        self.check_simple_loop({'raw_load': 2,
-                                'raw_store': 1,
-                                'arraylen_gc': 2,
-                                'guard_true': 1,
-                                'int_lt': 1,
-                                'jump': 1,
-                                'float_add': 1,
-                                'int_add': 3,
-                                })
+        self.check_trace_count(2)
+        self.check_simple_loop({
+            'float_add': 1,
+            'getarrayitem_gc': 2,
+            'guard_false': 2,
+            'guard_not_invalidated': 1,
+            'guard_true': 1,
+            'int_add': 5,
+            'int_ge': 1,
+            'int_is_zero': 1,
+            'int_lt': 1,
+            'jump': 1,
+            'raw_load': 2,
+            'raw_store': 1,
+            'setarrayitem_gc': 1,
+        })
+        self.check_resops({
+            'float_add': 2,
+            'getarrayitem_gc': 5,
+            'getarrayitem_gc_pure': 7,
+            'getfield_gc_pure': 56,
+            'guard_class': 3,
+            'guard_false': 11,
+            'guard_nonnull': 8,
+            'guard_nonnull_class': 3,
+            'guard_not_invalidated': 2,
+            'guard_true': 12,
+            'guard_value': 4,
+            'int_add': 17,
+            'int_ge': 4,
+            'int_is_true': 4,
+            'int_is_zero': 4,
+            'int_le': 2,
+            'int_lt': 3,
+            'int_sub': 1,
+            'jump': 2,
+            'raw_load': 4,
+            'raw_store': 2,
+            'setarrayitem_gc': 4,
+        })
 
     def define_reduce():
         return """

From noreply at buildbot.pypy.org  Thu Dec  4 19:14:51 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 19:14:51 +0100 (CET)
Subject: [pypy-commit] pypy default: avoid tracking an iterator index in
	axis_reduce
Message-ID: <20141204181451.4FF6F1C07BB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74812:e9c67f6fba33
Date: 2014-12-04 12:23 -0500
http://bitbucket.org/pypy/pypy/changeset/e9c67f6fba33/

Log:	avoid tracking an iterator index in axis_reduce

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -239,10 +239,9 @@
             state = x_state
     return out
 
-axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce',
-                                    greens=['shapelen',
-                                            'func', 'dtype'],
-                                    reds='auto')
+axis_reduce_driver = jit.JitDriver(name='numpy_axis_reduce',
+                                   greens=['shapelen', 'func', 'dtype'],
+                                   reds='auto')
 
 def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity, cumulative,
                    temp):
@@ -255,14 +254,16 @@
         temp_iter = out_iter  # hack
         temp_state = out_state
     arr_iter, arr_state = arr.create_iter()
+    arr_iter.track_index = False
     if identity is not None:
         identity = identity.convert_to(space, dtype)
     shapelen = len(shape)
     while not out_iter.done(out_state):
-        axis_reduce__driver.jit_merge_point(shapelen=shapelen, func=func,
-                                            dtype=dtype)
-        assert not arr_iter.done(arr_state)
+        axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func,
+                                           dtype=dtype)
         w_val = arr_iter.getitem(arr_state).convert_to(space, dtype)
+        arr_state = arr_iter.next(arr_state)
+
         out_indices = out_iter.indices(out_state)
         if out_indices[axis] == 0:
             if identity is not None:
@@ -270,6 +271,7 @@
         else:
             cur = temp_iter.getitem(temp_state)
             w_val = func(dtype, cur, w_val)
+
         out_iter.setitem(out_state, w_val)
         out_state = out_iter.next(out_state)
         if cumulative:
@@ -277,7 +279,6 @@
             temp_state = temp_iter.next(temp_state)
         else:
             temp_state = out_state
-        arr_state = arr_iter.next(arr_state)
     return out
 
 
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -197,7 +197,7 @@
             'guard_false': 2,
             'guard_not_invalidated': 1,
             'guard_true': 1,
-            'int_add': 5,
+            'int_add': 4,
             'int_ge': 1,
             'int_is_zero': 1,
             'int_lt': 1,
@@ -212,13 +212,13 @@
             'getarrayitem_gc_pure': 7,
             'getfield_gc_pure': 56,
             'guard_class': 3,
-            'guard_false': 11,
-            'guard_nonnull': 8,
+            'guard_false': 12,
+            'guard_nonnull': 11,
             'guard_nonnull_class': 3,
             'guard_not_invalidated': 2,
-            'guard_true': 12,
-            'guard_value': 4,
-            'int_add': 17,
+            'guard_true': 10,
+            'guard_value': 5,
+            'int_add': 13,
             'int_ge': 4,
             'int_is_true': 4,
             'int_is_zero': 4,

From noreply at buildbot.pypy.org  Thu Dec  4 19:14:52 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 19:14:52 +0100 (CET)
Subject: [pypy-commit] pypy default: enable sum/prod test_zjits
Message-ID: <20141204181452.7D5081C07BB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74813:624d6b4b60ce
Date: 2014-12-04 12:31 -0500
http://bitbucket.org/pypy/pypy/changeset/624d6b4b60ce/

Log:	enable sum/prod test_zjits

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -166,17 +166,22 @@
     def define_sum():
         return """
         a = |30|
-        b = a + a
-        sum(b)
+        sum(a)
         """
 
     def test_sum(self):
         result = self.run("sum")
-        assert result == 2 * sum(range(30))
-        py.test.skip("don't run for now")
-        self.check_simple_loop({"raw_load": 2, "float_add": 2,
-                                "int_add": 1, "int_ge": 1, "guard_false": 1,
-                                "jump": 1, 'arraylen_gc': 1})
+        assert result == sum(range(30))
+        self.check_trace_count(1)
+        self.check_simple_loop({
+            'float_add': 1,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'int_add': 2,
+            'int_ge': 1,
+            'jump': 1,
+            'raw_load': 1,
+        })
 
     def define_axissum():
         return """
@@ -260,8 +265,7 @@
     def define_prod():
         return """
         a = |30|
-        b = a + a
-        prod(b)
+        prod(a)
         """
 
     def test_prod(self):
@@ -270,11 +274,16 @@
         for i in range(30):
             expected *= i * 2
         assert result == expected
-        py.test.skip("don't run for now")
-        self.check_simple_loop({"raw_load": 2, "float_add": 1,
-                                "float_mul": 1, "int_add": 1,
-                                "int_ge": 1, "guard_false": 1, "jump": 1,
-                                'arraylen_gc': 1})
+        self.check_trace_count(1)
+        self.check_simple_loop({
+            'float_mul': 1,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'int_add': 2,
+            'int_ge': 1,
+            'jump': 1,
+            'raw_load': 1,
+        })
 
     def define_max():
         return """

From noreply at buildbot.pypy.org  Thu Dec  4 19:14:53 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 19:14:53 +0100 (CET)
Subject: [pypy-commit] pypy default: enable min/max in test_zjit
Message-ID: <20141204181453.C7BE61C07BB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74814:ac1f001ff1c0
Date: 2014-12-04 13:13 -0500
http://bitbucket.org/pypy/pypy/changeset/ac1f001ff1c0/

Log:	enable min/max in test_zjit

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -289,30 +289,62 @@
         return """
         a = |30|
         a[13] = 128
-        b = a + a
-        max(b)
+        max(a)
         """
 
     def test_max(self):
         result = self.run("max")
-        assert result == 256
-        py.test.skip("not there yet, getting though")
-        self.check_simple_loop({"raw_load": 2, "float_add": 1,
-                                "float_mul": 1, "int_add": 1,
-                                "int_lt": 1, "guard_true": 1, "jump": 1})
+        assert result == 128
+        self.check_trace_count(3)
+        self.check_simple_loop({
+            'float_ge': 1,
+            'float_ne': 1,
+            'guard_false': 3,
+            'guard_not_invalidated': 1,
+            'int_add': 2,
+            'int_ge': 1,
+            'jump': 1,
+            'raw_load': 1,
+        })
+        self.check_resops({
+            'float_ge': 2,
+            'float_ne': 2,
+            'getfield_gc_pure': 34,
+            'guard_class': 1,
+            'guard_false': 8,
+            'guard_nonnull': 2,
+            'guard_nonnull_class': 2,
+            'guard_not_invalidated': 2,
+            'guard_true': 7,
+            'guard_value': 2,
+            'int_add': 8,
+            'int_ge': 4,
+            'int_is_true': 3,
+            'jump': 3,
+            'raw_load': 2,
+        })
+
+    def define_min():
+        return """
+        a = |30|
+        a[13] = -128
+        min(a)
+        """
 
     def test_min(self):
-        py.test.skip("broken, investigate")
-        result = self.run("""
-        a = |30|
-        a[15] = -12
-        b = a + a
-        min(b)
-        """)
-        assert result == -24
-        self.check_simple_loop({"raw_load": 2, "float_add": 1,
-                                "float_mul": 1, "int_add": 1,
-                                "int_lt": 1, "guard_true": 1, "jump": 1})
+        result = self.run("min")
+        assert result == -128
+        self.check_trace_count(1)
+        self.check_simple_loop({
+            'float_le': 1,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'guard_true': 1,
+            'int_add': 2,
+            'int_ge': 1,
+            'jump': 1,
+            'raw_load': 1,
+        })
 
     def define_any():
         return """

From noreply at buildbot.pypy.org  Thu Dec  4 19:21:49 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 19:21:49 +0100 (CET)
Subject: [pypy-commit] cffi default: Merged in
 podshumok/cffi/podshumok/add-source_extnsion-param-to-verifier-1417477338819
 (pull request #54)
Message-ID: <20141204182149.AC7F41C100F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r1584:c5c87b3b1c00
Date: 2014-12-04 19:22 +0100
http://bitbucket.org/cffi/cffi/changeset/c5c87b3b1c00/

Log:	Merged in podshumok/cffi/podshumok/add-source_extnsion-param-to-
	verifier-1417477338819 (pull request #54)

	add source_extnsion param to Verifier

diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -16,7 +16,8 @@
 class Verifier(object):
 
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
-                 ext_package=None, tag='', force_generic_engine=False, **kwds):
+                 ext_package=None, tag='', force_generic_engine=False,
+                 source_extension='.c', **kwds):
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
@@ -43,7 +44,7 @@
                                               k1, k2)
         suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or os.environ.get('CFFI_TMPDIR') or _caller_dir_pycache()
-        self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
+        self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension)
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
         self.ext_package = ext_package
         self._has_source = False

From noreply at buildbot.pypy.org  Thu Dec  4 19:22:20 2014
From: noreply at buildbot.pypy.org (podshumok)
Date: Thu,  4 Dec 2014 19:22:20 +0100 (CET)
Subject: [pypy-commit] cffi
 podshumok/add-source_extnsion-param-to-verifier-1417477338819: add
 source_extnsion param to Verifier
Message-ID: <20141204182220.629E71C100F@cobra.cs.uni-duesseldorf.de>

Author: podshumok <k.ignatov at neodialog.ru>
Branch: podshumok/add-source_extnsion-param-to-verifier-1417477338819
Changeset: r1583:d82365905c32
Date: 2014-12-01 23:42 +0000
http://bitbucket.org/cffi/cffi/changeset/d82365905c32/

Log:	add source_extnsion param to Verifier

diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -16,7 +16,8 @@
 class Verifier(object):
 
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
-                 ext_package=None, tag='', force_generic_engine=False, **kwds):
+                 ext_package=None, tag='', force_generic_engine=False,
+                 source_extension='.c', **kwds):
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
@@ -43,7 +44,7 @@
                                               k1, k2)
         suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or os.environ.get('CFFI_TMPDIR') or _caller_dir_pycache()
-        self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
+        self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension)
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
         self.ext_package = ext_package
         self._has_source = False

From noreply at buildbot.pypy.org  Thu Dec  4 19:27:09 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 19:27:09 +0100 (CET)
Subject: [pypy-commit] cffi default: Document c5c87b3b1c00
Message-ID: <20141204182709.DE1B31D3605@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1585:2b86c5dc1331
Date: 2014-12-04 19:27 +0100
http://bitbucket.org/cffi/cffi/changeset/2b86c5dc1331/

Log:	Document c5c87b3b1c00

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -667,6 +667,15 @@
    check.  Be sure to have other means of clearing the ``tmpdir``
    whenever you change your sources.
 
+.. versionadded:: 0.8.7
+   You can give C++ source code in ``ffi.verify()``::
+
+     ext = ffi.verify(r'''
+         extern "C" {
+             int somefunc(int somearg) { return real_cpp_func(somearg); }
+         }
+     ''', source_extension='.cpp', extra_compile_args=['-std=c++11'])
+
 This function returns a "library" object that gets closed when it goes
 out of scope.  Make sure you keep the library object around as long as
 needed.

From noreply at buildbot.pypy.org  Thu Dec  4 22:41:23 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 22:41:23 +0100 (CET)
Subject: [pypy-commit] pypy default: add test_pypy_c for flatiter next
Message-ID: <20141204214123.8DAAE1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74815:d62d6a8c4809
Date: 2014-12-04 15:00 -0500
http://bitbucket.org/pypy/pypy/changeset/d62d6a8c4809/

Log:	add test_pypy_c for flatiter next

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -78,6 +78,46 @@
             jump(p0, p1, p3, p6, p7, p12, p14, f86, p18, i87, i62, p41, i58, p47, i40, i64, i70, descr=...)
         """)
 
+    def test_array_flatiter_next(self):
+        def main():
+            import _numpypy.multiarray as np
+            arr = np.zeros((1024, 16)) + 42
+            ai = arr.flat
+            i = 0
+            while i < arr.size:
+                a = next(ai)
+                i += 1
+            return a
+        log = self.run(main, [])
+        assert log.result == 42.0
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i86 = int_lt(i79, i45)
+            guard_true(i86, descr=...)
+            guard_not_invalidated(descr=...)
+            i87 = getfield_gc_pure(p85, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index .+>)
+            i88 = int_ge(i87, i59)
+            guard_false(i88, descr=...)
+            i89 = getfield_gc_pure(p85, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
+            f90 = raw_load(i67, i89, descr=<ArrayF 8>)
+            i91 = int_add(i87, 1)
+            p92 = getfield_gc_pure(p85, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
+            i93 = int_add(i89, i76)
+            i94 = int_add(i79, 1)
+            i95 = getfield_raw(#, descr=<FieldS pypysig_long_struct.c_value 0>)
+            i96 = int_lt(i95, 0)
+            guard_false(i96, descr=...)
+            p97 = new_with_vtable(#)
+            {{{
+            setfield_gc(p97, p56, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_iterator .+>)
+            setfield_gc(p97, p92, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
+            setfield_gc(p97, i91, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index .+>)
+            setfield_gc(p97, i93, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
+            setfield_gc(p16, p97, descr=<FieldP pypy.module.micronumpy.flatiter.W_FlatIterator.inst_state .+>)
+            }}}
+            jump(p0, p1, p3, p6, p12, p14, p16, i94, f90, p26, i45, p97, i59, p56, i67, i76, descr=...)
+        """)
+
     def test_array_flatiter_getitem_single(self):
         def main():
             import _numpypy.multiarray as np

From noreply at buildbot.pypy.org  Thu Dec  4 22:41:24 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 22:41:24 +0100 (CET)
Subject: [pypy-commit] pypy default: support mutate flag to iterator
	next/reset
Message-ID: <20141204214124.D6D6D1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74816:e920a686f245
Date: 2014-12-04 14:32 -0500
http://bitbucket.org/pypy/pypy/changeset/e920a686f245/

Log:	support mutate flag to iterator next/reset

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -77,7 +77,7 @@
 
 
 class IterState(object):
-    _immutable_fields_ = ['iterator', 'index', '_indices', 'offset']
+    _immutable_fields_ = ['iterator', '_indices']
 
     def __init__(self, iterator, index, indices, offset):
         self.iterator = iterator
@@ -117,7 +117,8 @@
         self.factors = factors
 
     @jit.unroll_safe
-    def reset(self, state=None):
+    def reset(self, state=None, mutate=False):
+        index = 0
         if state is None:
             indices = [0] * len(self.shape_m1)
         else:
@@ -125,10 +126,14 @@
             indices = state._indices
             for i in xrange(self.ndim_m1, -1, -1):
                 indices[i] = 0
-        return IterState(self, 0, indices, self.array.start)
+        offset = self.array.start
+        if not mutate:
+            return IterState(self, index, indices, offset)
+        state.index = index
+        state.offset = offset
 
     @jit.unroll_safe
-    def next(self, state):
+    def next(self, state, mutate=False):
         assert state.iterator is self
         index = state.index
         if self.track_index:
@@ -149,7 +154,10 @@
                 else:
                     indices[i] = 0
                     offset -= self.backstrides[i]
-        return IterState(self, index, indices, offset)
+        if not mutate:
+            return IterState(self, index, indices, offset)
+        state.index = index
+        state.offset = offset
 
     @jit.unroll_safe
     def goto(self, index):
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -215,7 +215,8 @@
             'float_add': 2,
             'getarrayitem_gc': 5,
             'getarrayitem_gc_pure': 7,
-            'getfield_gc_pure': 56,
+            'getfield_gc': 5,
+            'getfield_gc_pure': 51,
             'guard_class': 3,
             'guard_false': 12,
             'guard_nonnull': 11,
@@ -309,7 +310,8 @@
         self.check_resops({
             'float_ge': 2,
             'float_ne': 2,
-            'getfield_gc_pure': 34,
+            'getfield_gc': 4,
+            'getfield_gc_pure': 30,
             'guard_class': 1,
             'guard_false': 8,
             'guard_nonnull': 2,
@@ -561,7 +563,8 @@
             'float_add': 3,
             'getarrayitem_gc': 7,
             'getarrayitem_gc_pure': 14,
-            'getfield_gc_pure': 69,
+            'getfield_gc': 6,
+            'getfield_gc_pure': 63,
             'guard_class': 5,
             'guard_false': 20,
             'guard_nonnull': 6,
@@ -612,7 +615,8 @@
             'float_add': 2,
             'getarrayitem_gc': 2,
             'getarrayitem_gc_pure': 2,
-            'getfield_gc_pure': 36,
+            'getfield_gc': 6,
+            'getfield_gc_pure': 30,
             'guard_class': 3,
             'guard_false': 7,
             'guard_nonnull': 2,
@@ -769,7 +773,8 @@
             'float_mul': 2,
             'getarrayitem_gc': 4,
             'getarrayitem_gc_pure': 9,
-            'getfield_gc_pure': 49,
+            'getfield_gc': 7,
+            'getfield_gc_pure': 42,
             'guard_class': 4,
             'guard_false': 15,
             'guard_not_invalidated': 2,

From noreply at buildbot.pypy.org  Thu Dec  4 22:41:26 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 22:41:26 +0100 (CET)
Subject: [pypy-commit] pypy default: use mutating iterator state in flatiter
Message-ID: <20141204214126.09ACD1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74817:cdcc29b5a5eb
Date: 2014-12-04 15:00 -0500
http://bitbucket.org/pypy/pypy/changeset/cdcc29b5a5eb/

Log:	use mutating iterator state in flatiter

diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py
--- a/pypy/module/micronumpy/flatiter.py
+++ b/pypy/module/micronumpy/flatiter.py
@@ -57,7 +57,7 @@
         if self.iter.done(self.state):
             raise OperationError(space.w_StopIteration, space.w_None)
         w_res = self.iter.getitem(self.state)
-        self.state = self.iter.next(self.state)
+        self.iter.next(self.state, mutate=True)
         return w_res
 
     def descr_getitem(self, space, w_idx):
@@ -74,7 +74,7 @@
                                          base.get_order(), w_instance=base)
             return loop.flatiter_getitem(res, self.iter, state, step)
         finally:
-            self.state = self.iter.reset(self.state)
+            self.iter.reset(self.state, mutate=True)
 
     def descr_setitem(self, space, w_idx, w_value):
         if not (space.isinstance_w(w_idx, space.w_int) or
@@ -94,7 +94,7 @@
             arr = convert_to_array(space, w_value)
             loop.flatiter_setitem(space, dtype, arr, self.iter, state, step, length)
         finally:
-            self.state = self.iter.reset(self.state)
+            self.iter.reset(self.state, mutate=True)
 
 
 W_FlatIterator.typedef = TypeDef("numpy.flatiter",
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -95,27 +95,18 @@
             i86 = int_lt(i79, i45)
             guard_true(i86, descr=...)
             guard_not_invalidated(descr=...)
-            i87 = getfield_gc_pure(p85, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index .+>)
             i88 = int_ge(i87, i59)
             guard_false(i88, descr=...)
-            i89 = getfield_gc_pure(p85, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
             f90 = raw_load(i67, i89, descr=<ArrayF 8>)
             i91 = int_add(i87, 1)
-            p92 = getfield_gc_pure(p85, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
             i93 = int_add(i89, i76)
             i94 = int_add(i79, 1)
             i95 = getfield_raw(#, descr=<FieldS pypysig_long_struct.c_value 0>)
+            setfield_gc(p97, i91, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index .+>)
+            setfield_gc(p97, i93, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
             i96 = int_lt(i95, 0)
             guard_false(i96, descr=...)
-            p97 = new_with_vtable(#)
-            {{{
-            setfield_gc(p97, p56, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_iterator .+>)
-            setfield_gc(p97, p92, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
-            setfield_gc(p97, i91, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index .+>)
-            setfield_gc(p97, i93, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
-            setfield_gc(p16, p97, descr=<FieldP pypy.module.micronumpy.flatiter.W_FlatIterator.inst_state .+>)
-            }}}
-            jump(p0, p1, p3, p6, p12, p14, p16, i94, f90, p26, i45, p97, i59, p56, i67, i76, descr=...)
+            jump(p0, p1, p3, p6, p12, p14, p16, i94, f90, p26, i45, i91, i59, p97, p96, i67, i93, i76, descr=...)
         """)
 
     def test_array_flatiter_getitem_single(self):
@@ -139,23 +130,15 @@
             i128 = int_mul(i117, i59)
             i129 = int_add(i55, i128)
             f149 = raw_load(i100, i129, descr=<ArrayF 8>)
-            p150 = getfield_gc_pure(p123, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
             i151 = int_add(i117, 1)
             setarrayitem_gc(p150, 1, 0, descr=<ArrayS .+>)
             setarrayitem_gc(p150, 0, 0, descr=<ArrayS .+>)
+            setfield_gc(p156, i55, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
             guard_not_invalidated(descr=...)
             i154 = getfield_raw(#, descr=<FieldS pypysig_long_struct.c_value 0>)
             i155 = int_lt(i154, 0)
             guard_false(i155, descr=...)
-            p156 = new_with_vtable(...)
-            {{{
-            setfield_gc(p156, p49, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_iterator .+>)
-            setfield_gc(p156, i55, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
-            setfield_gc(p156, 0, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index .+>)
-            setfield_gc(p156, p150, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
-            setfield_gc(p16, p156, descr=<FieldP pypy.module.micronumpy.flatiter.W_FlatIterator.inst_state .+>)
-            }}}
-            jump(p0, p1, p3, p6, p7, p12, p14, p16, i151, f149, p26, i44, i50, i59, i55, i100, p156, p49, descr=...)
+            jump(p0, p1, p3, p6, p7, p12, p14, p16, i151, f149, p26, i44, i50, i59, i55, i100, p150, p156, descr=...)
         """)
 
     def test_array_flatiter_setitem_single(self):
@@ -180,20 +163,12 @@
             i132 = int_add(i53, i131)
             guard_not_invalidated(descr=...)
             raw_store(i103, i132, 42.000000, descr=<ArrayF 8>)
-            p152 = getfield_gc_pure(p126, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
             i153 = int_add(i120, 1)
             i154 = getfield_raw(#, descr=<FieldS pypysig_long_struct.c_value 0>)
             setarrayitem_gc(p152, 1, 0, descr=<ArrayS .+>)
             setarrayitem_gc(p152, 0, 0, descr=<ArrayS .+>)
+            setfield_gc(p158, i53, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
             i157 = int_lt(i154, 0)
             guard_false(i157, descr=...)
-            p158 = new_with_vtable(...)
-            {{{
-            setfield_gc(p158, p47, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_iterator .+>)
-            setfield_gc(p158, i53, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
-            setfield_gc(p158, 0, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index .+>)
-            setfield_gc(p158, p152, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices .+>)
-            setfield_gc(p16, p158, descr=<FieldP pypy.module.micronumpy.flatiter.W_FlatIterator.inst_state .+>)
-            }}}
-            jump(p0, p1, p3, p6, p7, p12, p14, p16, i153, i42, i48, i57, i53, p47, i103, p158, descr=...)
+            jump(p0, p1, p3, p6, p7, p12, p14, p16, i153, i42, i48, i57, i53, p47, i103, p152, p158, descr=...)
         """)

From noreply at buildbot.pypy.org  Thu Dec  4 22:45:27 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 22:45:27 +0100 (CET)
Subject: [pypy-commit] creflect default: Accept (and ignore) variable names
 after "long" (they were already
Message-ID: <20141204214527.B1F731D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r152:f9f839119f6e
Date: 2014-12-04 22:42 +0100
http://bitbucket.org/cffi/creflect/changeset/f9f839119f6e/

Log:	Accept (and ignore) variable names after "long" (they were already
	accepted after "int")

diff --git a/creflect/creflect_cdecl.c b/creflect/creflect_cdecl.c
--- a/creflect/creflect_cdecl.c
+++ b/creflect/creflect_cdecl.c
@@ -444,7 +444,6 @@
         case TOK_VOID:
         case TOK__BOOL:
         case TOK_FLOAT:
-        case TOK_IDENTIFIER:
         case TOK_STRUCT:
         case TOK_UNION:
             parse_error(tok, "invalid combination of types");
diff --git a/creflect/test/test_c_decl_parser.py b/creflect/test/test_c_decl_parser.py
--- a/creflect/test/test_c_decl_parser.py
+++ b/creflect/test/test_c_decl_parser.py
@@ -43,6 +43,8 @@
     parse("unsigned int", "unsigned int")
     parse("long", "long")
     parse("long int", "long")
+    parse("long int a", "long")
+    parse("long a", "long")
     parse("short int", "short")
     parse("long long int", "long long")
     parse("unsigned long long *", "PTR unsigned long long")
@@ -82,7 +84,7 @@
     parse_error("long char", "invalid combination of types", 5)
     parse_error("short char", "invalid combination of types", 6)
     parse_error("signed void", "invalid combination of types", 7)
-    parse_error("unsigned foobar_t", "invalid combination of types", 9)
+    parse_error("unsigned struct", "invalid combination of types", 9)
     #
     parse_error("", "identifier expected", 0)
     parse_error("]", "identifier expected", 0)

From noreply at buildbot.pypy.org  Thu Dec  4 22:45:28 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 22:45:28 +0100 (CET)
Subject: [pypy-commit] creflect default: function types
Message-ID: <20141204214528.D506A1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r153:bfdb3cb4b9b4
Date: 2014-12-04 22:45 +0100
http://bitbucket.org/cffi/creflect/changeset/bfdb3cb4b9b4/

Log:	function types

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -10,32 +10,38 @@
     PyObject *types_dict;
 } zeffir_builder_t;
 
-static PyObject *combine_type_name(_crx_builder_t *cb, CTypeDescrObject *ct,
-                                   const char *extra_text)
+static PyObject *combine_type_name_l(CTypeDescrObject *ct,
+                                     size_t extra_text_len)
 {
-    /* Build a PyString with the name given by combining 'ct' and 'extra_text'.
-       'ct' can be NULL.  If an error occurs (or has already occurred),
-       returns NULL. */
-    size_t base_name_len, extra_name_len;
+    size_t base_name_len;
     PyObject *result;
     char *p;
 
-    base_name_len = (ct != NULL ? strlen(ct->ct_name) : 0);
-    extra_name_len = strlen(extra_text);
-    result = PyString_FromStringAndSize(NULL, base_name_len + extra_name_len);
+    base_name_len = strlen(ct->ct_name);
+    result = PyString_FromStringAndSize(NULL, base_name_len + extra_text_len);
     if (result == NULL)
         return NULL;
 
     p = PyString_AS_STRING(result);
-    if (ct != NULL) {
-        memcpy(p, ct->ct_name, ct->ct_name_position);
+    memcpy(p, ct->ct_name, ct->ct_name_position);
+    p += ct->ct_name_position;
+    p += extra_text_len;
+    memcpy(p, ct->ct_name + ct->ct_name_position,
+           base_name_len - ct->ct_name_position);
+    return result;
+}
+
+static PyObject *combine_type_name(CTypeDescrObject *ct, const char *extra_text)
+{
+    /* Build a PyString with the name given by combining 'ct' and 'extra_text'.
+       'ct' can be NULL.  If an error occurs returns NULL. */
+    size_t extra_text_len = strlen(extra_text);
+    PyObject *result = combine_type_name_l(ct, extra_text_len);
+
+    if (result != NULL) {
+        char *p = PyString_AS_STRING(result);
         p += ct->ct_name_position;
-    }
-    memcpy(p, extra_text, extra_name_len);
-    if (ct != NULL) {
-        p += extra_name_len;
-        memcpy(p, ct->ct_name + ct->ct_name_position,
-               base_name_len - ct->ct_name_position);
+        memcpy(p, extra_text, extra_text_len);
     }
     return result;
 }
@@ -75,7 +81,7 @@
     PyObject *name_obj;
     CTypeDescrObject *ct;
 
-    name_obj = combine_type_name(cb, NULL, name);
+    name_obj = PyString_FromString(name);
     if (name_obj == NULL)
         return NULL;
 
@@ -151,7 +157,63 @@
                                           _crx_qual_type args[], int nargs,
                                           _crx_trampoline1_fn trampl)
 {
-    abort();
+    if (PyErr_Occurred())
+        return NULL;
+
+    PyObject *name_obj;
+    CTypeDescrObject *ct;
+    size_t extra_len;
+
+    if (nargs == 0) {
+        name_obj = combine_type_name(ret, "(void)");
+        if (name_obj == NULL)
+            return NULL;
+    }
+    else {
+        char *p;
+        int i;
+        extra_len = 1 + 1;   /* "("  ")" */
+        extra_len += (nargs - 1) * 2;   /* ", " */
+        for (i = 0; i < nargs; i++) {
+            extra_len += strlen(args[i].type->ct_name);
+        }
+        name_obj = combine_type_name_l(ret, extra_len);
+        if (name_obj == NULL)
+            return NULL;
+        p = PyString_AS_STRING(name_obj);
+        p += ret->ct_name_position;
+        *p++ = '(';
+        for (i = 0; i < nargs; i++) {
+            size_t len = strlen(args[i].type->ct_name);
+            if (i > 0) {
+                *p++ = ',';
+                *p++ = ' ';
+            }
+            memcpy(p, args[i].type->ct_name, len);
+            p += len;
+        }
+        *p++ = ')';
+        assert(p - PyString_AS_STRING(name_obj) - ret->ct_name_position
+               == extra_len);
+    }
+
+    ct = get_cached_type(cb, name_obj);
+    if (ct && (ct->ct_flags == CT_FUNCTION))
+        goto done;
+
+    ct = ctypedescr_new(name_obj, ret->ct_name_position);
+    if (ct == NULL)
+        goto done;
+
+    ct->ct_size = -1;
+    ct->ct_flags = CT_FUNCTION;
+    /* XXX more... */
+
+    put_cached_type(cb, name_obj, ct);
+
+ done:
+    Py_DECREF(name_obj);
+    return ct;
 }
 
 static _crx_type_t *zef_get_ellipsis_function_type(_crx_builder_t *cb,
@@ -172,12 +234,12 @@
     PyObject *name_obj;
     CTypeDescrObject *ct;
 
-    if (totype->ct_flags & CT_ARRAY)
+    if (totype->ct_flags & (CT_ARRAY | CT_FUNCTION))
         extra = "(*)";
     else
         extra = " *";
 
-    name_obj = combine_type_name(cb, totype, extra);
+    name_obj = combine_type_name(totype, extra);
     if (name_obj == NULL)
         return NULL;
 
@@ -228,7 +290,7 @@
     else
         sprintf(extra_text, "[%zd]", length);
 
-    name_obj = combine_type_name(cb, ctitem, extra_text);
+    name_obj = combine_type_name(ctitem, extra_text);
     if (name_obj == NULL)
         return NULL;
 
@@ -292,7 +354,7 @@
     if (PyErr_Occurred())
         return result;
 
-    name_obj = combine_type_name(cb, NULL, name);
+    name_obj = PyString_FromString(name);
     if (name_obj == NULL)
         return result;
 
diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -10,11 +10,11 @@
 #define CT_UNION            128    /* union */
 #define CT_UNKNOWN          256    /* unknown type */
 #define CT_VOID             512    /* void */
+#define CT_FUNCTION        1024    /* function */
 
 /* other flags that may also be set in addition to the base flag: */
-#define CT_CAST_ANYTHING         1024    /* 'char *' and 'void *' only */
-#define CT_PRIMITIVE_FITS_LONG   2048
-//#define CT_IS_OPAQUE           4096    /* == (ct_size < 0) */
+#define CT_CAST_ANYTHING         2048    /* 'char *' and 'void *' only */
+#define CT_PRIMITIVE_FITS_LONG   4096
 #define CT_IS_ENUM               8192
 #define CT_IS_PTR_TO_OWNED      16384
 #define CT_IS_LONGDOUBLE        65536
diff --git a/zeffir/test/test_ctype.py b/zeffir/test/test_ctype.py
--- a/zeffir/test/test_ctype.py
+++ b/zeffir/test/test_ctype.py
@@ -67,6 +67,13 @@
     assert repr(ffi.typeof("int * [ ]")) == "<ctype 'int *[]'>"
     assert repr(ffi.typeof("int ( * ) [ ]")) == "<ctype 'int(*)[]'>"
 
+def test_typeof_function():
+    ffi = support.new_ffi()
+    assert repr(ffi.typeof("int()")) == "<ctype 'int(void)'>"
+    assert repr(ffi.typeof("int*(void*)")) == "<ctype 'int *(void *)'>"
+    assert repr(ffi.typeof("int(long a,short b)"))=="<ctype 'int(long, short)'>"
+    assert repr(ffi.typeof("int(*)(long)"))=="<ctype 'int(*)(long)'>"
+
 def test_simple_typedef():
     ffi, lib = support.compile_and_open('ctype')
     assert ffi.sizeof("foo_t") == ffi.sizeof("long long")

From noreply at buildbot.pypy.org  Thu Dec  4 22:49:23 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 22:49:23 +0100 (CET)
Subject: [pypy-commit] creflect default: functions with '...'
Message-ID: <20141204214923.483A41D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r154:47c327664396
Date: 2014-12-04 22:49 +0100
http://bitbucket.org/cffi/creflect/changeset/47c327664396/

Log:	functions with '...'

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -153,9 +153,9 @@
     return _zef_primitive(cb, sz, name, flags);
 }
 
-static _crx_type_t *zef_get_function_type(_crx_builder_t *cb, _crx_type_t *ret,
-                                          _crx_qual_type args[], int nargs,
-                                          _crx_trampoline1_fn trampl)
+static _crx_type_t *_zef_function(_crx_builder_t *cb, _crx_type_t *ret,
+                                  _crx_qual_type args[], int nargs,
+                                  int dotdotdot)
 {
     if (PyErr_Occurred())
         return NULL;
@@ -163,8 +163,9 @@
     PyObject *name_obj;
     CTypeDescrObject *ct;
     size_t extra_len;
+    int nargsall = nargs + dotdotdot;
 
-    if (nargs == 0) {
+    if (nargsall == 0) {
         name_obj = combine_type_name(ret, "(void)");
         if (name_obj == NULL)
             return NULL;
@@ -173,10 +174,12 @@
         char *p;
         int i;
         extra_len = 1 + 1;   /* "("  ")" */
-        extra_len += (nargs - 1) * 2;   /* ", " */
+        extra_len += (nargsall - 1) * 2;   /* ", " */
         for (i = 0; i < nargs; i++) {
             extra_len += strlen(args[i].type->ct_name);
         }
+        if (dotdotdot)
+            extra_len += 3;  /* "..." */
         name_obj = combine_type_name_l(ret, extra_len);
         if (name_obj == NULL)
             return NULL;
@@ -192,6 +195,14 @@
             memcpy(p, args[i].type->ct_name, len);
             p += len;
         }
+        if (dotdotdot) {
+            if (nargs > 0) {
+                *p++ = ',';
+                *p++ = ' ';
+            }
+            memcpy(p, "...", 3);
+            p += 3;
+        }
         *p++ = ')';
         assert(p - PyString_AS_STRING(name_obj) - ret->ct_name_position
                == extra_len);
@@ -216,12 +227,19 @@
     return ct;
 }
 
+static _crx_type_t *zef_get_function_type(_crx_builder_t *cb, _crx_type_t *ret,
+                                          _crx_qual_type args[], int nargs,
+                                          _crx_trampoline1_fn trampl)
+{
+    return _zef_function(cb, ret, args, nargs, 0);
+}
+
 static _crx_type_t *zef_get_ellipsis_function_type(_crx_builder_t *cb,
                                                    _crx_type_t *ret,
                                                    _crx_qual_type args[],
                                                    int nargs)
 {
-    abort();
+    return _zef_function(cb, ret, args, nargs, 1);
 }
 
 static _crx_type_t *zef_get_pointer_type(_crx_builder_t *cb,
diff --git a/zeffir/test/test_ctype.py b/zeffir/test/test_ctype.py
--- a/zeffir/test/test_ctype.py
+++ b/zeffir/test/test_ctype.py
@@ -74,6 +74,11 @@
     assert repr(ffi.typeof("int(long a,short b)"))=="<ctype 'int(long, short)'>"
     assert repr(ffi.typeof("int(*)(long)"))=="<ctype 'int(*)(long)'>"
 
+def test_typedef_function_dotdotdot():
+    ffi = support.new_ffi()
+    assert repr(ffi.typeof("int(...)")) == "<ctype 'int(...)'>"
+    assert repr(ffi.typeof("int*(void*,...)")) == "<ctype 'int *(void *, ...)'>"
+
 def test_simple_typedef():
     ffi, lib = support.compile_and_open('ctype')
     assert ffi.sizeof("foo_t") == ffi.sizeof("long long")

From noreply at buildbot.pypy.org  Thu Dec  4 23:01:19 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 23:01:19 +0100 (CET)
Subject: [pypy-commit] pypy default: add a test_pypy_c for array.any()
Message-ID: <20141204220119.A03CF1D2B03@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74818:ff4204895657
Date: 2014-12-04 17:00 -0500
http://bitbucket.org/pypy/pypy/changeset/ff4204895657/

Log:	add a test_pypy_c for array.any()

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -3,6 +3,30 @@
 
 
 class TestMicroNumPy(BaseTestPyPyC):
+    def test_array_any(self):
+        def main():
+            import _numpypy.multiarray as np
+            arr = np.array([False] * 1500)
+            return arr.any()
+        log = self.run(main, [])
+        assert log.result == False
+        assert len(log.loops) == 1
+        loop = log._filter(log.loops[0])
+        assert loop.match("""
+            i33 = raw_load(i9, i29, descr=<ArrayU 1>)
+            guard_false(i33, descr=...)
+            guard_not_invalidated(descr=...)
+            i34 = getarrayitem_raw(#, 1, descr=<ArrayU 1>)
+            guard_value(i34, 0, descr=...)
+            i35 = getarrayitem_raw(#, 0, descr=<ArrayU 1>)
+            i36 = int_add(i24, 1)
+            i37 = int_add(i29, i28)
+            i38 = int_ge(i36, i30)
+            guard_false(i38, descr=...)
+            guard_value(i35, 1, descr=...)
+            jump(p0, p25, i36, i37, i9, i28, i30, descr=...)
+        """)
+
     def test_array_getitem_basic(self):
         def main():
             import _numpypy.multiarray as np

From noreply at buildbot.pypy.org  Thu Dec  4 23:09:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  4 Dec 2014 23:09:13 +0100 (CET)
Subject: [pypy-commit] creflect default: tweaks
Message-ID: <20141204220913.0E0701C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r155:7c6a10d45a9d
Date: 2014-12-04 22:59 +0100
http://bitbucket.org/cffi/creflect/changeset/7c6a10d45a9d/

Log:	tweaks

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -563,11 +563,11 @@
             PyErr_Fetch(&exc, &val, &tb);
             ps = PyObject_Str(val);
             if (ps != NULL && PyString_Check(ps)) {
-                PyErr_Format(exc, "at pos %zd: %s",
-                             (size_t)(err - str), PyString_AS_STRING(ps));
                 Py_XDECREF(tb);
                 Py_XDECREF(val);
                 Py_XDECREF(exc);
+                PyErr_Format(exc, "at pos %zd: %s",
+                             (size_t)(err - str), PyString_AS_STRING(ps));
             }
             else {
                 PyErr_Restore(exc, val, tb);
@@ -576,8 +576,7 @@
         }
         return NULL;
     }
-    if (err != NULL)
-        abort();
+    assert(err == NULL);   /* else, zef_error() must have been called */
 
     Py_INCREF(result.type);
     return result.type;
diff --git a/zeffir/lib_obj.c b/zeffir/lib_obj.c
--- a/zeffir/lib_obj.c
+++ b/zeffir/lib_obj.c
@@ -43,6 +43,12 @@
     return x;
 }
 
+static int lib_setattr(ZefLibObject *lib, PyObject *name, PyObject *val)
+{
+    PyErr_SetString(PyExc_AttributeError, "Lib object is read-only");
+    return -1;
+}
+
 static PyObject *lib_dir(PyObject *lib, PyObject *noarg)
 {
     return PyDict_Keys(((ZefLibObject *)lib)->l_dict);
@@ -71,7 +77,7 @@
     0,                                          /* tp_call */
     0,                                          /* tp_str */
     (getattrofunc)lib_getattr,                  /* tp_getattro */
-    0,                                          /* tp_setattro */
+    (setattrofunc)lib_setattr,                  /* tp_setattro */
     0,                                          /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT,                         /* tp_flags */
     0,                                          /* tp_doc */
@@ -88,7 +94,7 @@
     0,                                          /* tp_dict */
     0,                                          /* tp_descr_get */
     0,                                          /* tp_descr_set */
-    offsetof(ZefLibObject, l_dict),              /* tp_dictoffset */
+    offsetof(ZefLibObject, l_dict),             /* tp_dictoffset */
 };
 
 static void lib_dlerror(ZefLibObject *lib)
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -14,6 +14,8 @@
     ffi, lib = support.compile_and_open('basic')
     assert lib.forty_two == 42
     assert type(lib.forty_two) is int
+    py.test.raises(AttributeError, setattr, lib, 'forty_two', 43)
+    py.test.raises(AttributeError, delattr, lib, 'forty_two')
 
 def test_dir():
     ffi, lib = support.compile_and_open('basic')
@@ -21,7 +23,10 @@
 
 def test_no_special_attribute():
     ffi, lib = support.compile_and_open('basic')
-    py.test.raises(AttributeError, getattr, lib, '__class__')
+    for name in dir(object()):   # '__class__', '__str__', etc.
+        py.test.raises(AttributeError, getattr, lib, name)
+        py.test.raises(AttributeError, setattr, lib, name, '?')
+        py.test.raises(AttributeError, delattr, lib, name)
 
 def test_types():
     ffi = support.new_ffi()

From noreply at buildbot.pypy.org  Thu Dec  4 23:57:11 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu,  4 Dec 2014 23:57:11 +0100 (CET)
Subject: [pypy-commit] pypy default: cleanups
Message-ID: <20141204225711.E12091C07BB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74819:a0e9ee8c4971
Date: 2014-12-04 17:57 -0500
http://bitbucket.org/pypy/pypy/changeset/a0e9ee8c4971/

Log:	cleanups

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -287,8 +287,9 @@
     def logical_xor(self, v1, v2):
         return bool(v1) ^ bool(v2)
 
+    @raw_unary_op
     def bool(self, v):
-        return bool(self.for_computation(self.unbox(v)))
+        return bool(v)
 
     @simple_binary_op
     def max(self, v1, v2):
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -18,6 +18,7 @@
 def done_if_false(dtype, val):
     return not dtype.itemtype.bool(val)
 
+
 def _get_dtype(space, w_npyobj):
     if isinstance(w_npyobj, boxes.W_GenericBox):
         return w_npyobj.get_dtype(space)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -16,14 +16,14 @@
             i33 = raw_load(i9, i29, descr=<ArrayU 1>)
             guard_false(i33, descr=...)
             guard_not_invalidated(descr=...)
-            i34 = getarrayitem_raw(#, 1, descr=<ArrayU 1>)
-            guard_value(i34, 0, descr=...)
-            i35 = getarrayitem_raw(#, 0, descr=<ArrayU 1>)
+            i34 = getarrayitem_raw(#, 1, descr=<ArrayU 1>)  # XXX what are these?
+            guard_value(i34, 0, descr=...)                  # XXX
+            i35 = getarrayitem_raw(#, 0, descr=<ArrayU 1>)  # XXX
             i36 = int_add(i24, 1)
             i37 = int_add(i29, i28)
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
-            guard_value(i35, 1, descr=...)
+            guard_value(i35, 1, descr=...)                  # XXX
             jump(p0, p25, i36, i37, i9, i28, i30, descr=...)
         """)
 
@@ -130,7 +130,7 @@
             setfield_gc(p97, i93, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
             i96 = int_lt(i95, 0)
             guard_false(i96, descr=...)
-            jump(p0, p1, p3, p6, p12, p14, p16, i94, f90, p26, i45, i91, i59, p97, p96, i67, i93, i76, descr=...)
+            jump(p0, p1, p3, p6, p12, p14, p16, i94, f90, p26, i45, i91, i59, p96, p97, i67, i93, i76, descr=...)
         """)
 
     def test_array_flatiter_getitem_single(self):

From noreply at buildbot.pypy.org  Fri Dec  5 02:38:09 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 02:38:09 +0100 (CET)
Subject: [pypy-commit] pypy default: cleanup some numpy test_pypy_c tests
Message-ID: <20141205013809.73BFD1C100F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74820:e99e8fe69789
Date: 2014-12-04 20:35 -0500
http://bitbucket.org/pypy/pypy/changeset/e99e8fe69789/

Log:	cleanup some numpy test_pypy_c tests

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -58,9 +58,8 @@
             """ + alignment_check + """
             f80 = raw_load(i67, i79, descr=<ArrayF 8>)
             i81 = int_add(i71, 1)
-            guard_not_invalidated(descr=...)
             --TICK--
-            jump(p0, p1, p3, p6, p7, p12, p14, p16, i81, f80, i59, p38, i55, p40, i37, i61, i67, descr=...)
+            jump(..., descr=...)
         """)
 
     def test_array_getitem_accumulate(self):
@@ -130,7 +129,7 @@
             setfield_gc(p97, i93, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
             i96 = int_lt(i95, 0)
             guard_false(i96, descr=...)
-            jump(p0, p1, p3, p6, p12, p14, p16, i94, f90, p26, i45, i91, i59, p96, p97, i67, i93, i76, descr=...)
+            jump(..., descr=...)
         """)
 
     def test_array_flatiter_getitem_single(self):
@@ -158,11 +157,8 @@
             setarrayitem_gc(p150, 1, 0, descr=<ArrayS .+>)
             setarrayitem_gc(p150, 0, 0, descr=<ArrayS .+>)
             setfield_gc(p156, i55, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
-            guard_not_invalidated(descr=...)
-            i154 = getfield_raw(#, descr=<FieldS pypysig_long_struct.c_value 0>)
-            i155 = int_lt(i154, 0)
-            guard_false(i155, descr=...)
-            jump(p0, p1, p3, p6, p7, p12, p14, p16, i151, f149, p26, i44, i50, i59, i55, i100, p150, p156, descr=...)
+            --TICK--
+            jump(..., descr=...)
         """)
 
     def test_array_flatiter_setitem_single(self):
@@ -194,5 +190,5 @@
             setfield_gc(p158, i53, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset .+>)
             i157 = int_lt(i154, 0)
             guard_false(i157, descr=...)
-            jump(p0, p1, p3, p6, p7, p12, p14, p16, i153, i42, i48, i57, i53, p47, i103, p152, p158, descr=...)
+            jump(..., descr=...)
         """)

From noreply at buildbot.pypy.org  Fri Dec  5 02:38:10 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 02:38:10 +0100 (CET)
Subject: [pypy-commit] pypy default: simplify array.any test_pypy_c to use
	reduce directly
Message-ID: <20141205013810.A81901C100F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74821:e301f280ff33
Date: 2014-12-04 20:24 -0500
http://bitbucket.org/pypy/pypy/changeset/e301f280ff33/

Log:	simplify array.any test_pypy_c to use reduce directly

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -3,28 +3,31 @@
 
 
 class TestMicroNumPy(BaseTestPyPyC):
-    def test_array_any(self):
+    def test_reduce_logical_and(self):
         def main():
             import _numpypy.multiarray as np
-            arr = np.array([False] * 1500)
-            return arr.any()
+            import _numpypy.umath as um
+            arr = np.array([1.0] * 1500)
+            return um.logical_and.reduce(arr)
         log = self.run(main, [])
-        assert log.result == False
+        assert log.result is True
         assert len(log.loops) == 1
         loop = log._filter(log.loops[0])
         assert loop.match("""
-            i33 = raw_load(i9, i29, descr=<ArrayU 1>)
-            guard_false(i33, descr=...)
+            f31 = raw_load(i9, i29, descr=<ArrayF 8>)
             guard_not_invalidated(descr=...)
-            i34 = getarrayitem_raw(#, 1, descr=<ArrayU 1>)  # XXX what are these?
-            guard_value(i34, 0, descr=...)                  # XXX
-            i35 = getarrayitem_raw(#, 0, descr=<ArrayU 1>)  # XXX
+            i32 = cast_float_to_int(f31)
+            i33 = int_and(i32, 255)
+            guard_true(i33, descr=...)
+            i34 = getarrayitem_raw(#, 2, descr=<ArrayU 1>)  # XXX what are these?
+            guard_value(i34, 1, descr=...)                  # XXX don't appear in
+            i35 = getarrayitem_raw(#, 1, descr=<ArrayU 1>)  # XXX equiv test_zjit
             i36 = int_add(i24, 1)
             i37 = int_add(i29, i28)
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
-            guard_value(i35, 1, descr=...)                  # XXX
-            jump(p0, p25, i36, i37, i9, i28, i30, descr=...)
+            guard_value(i35, 2, descr=...)                  # XXX
+            jump(..., descr=...)
         """)
 
     def test_array_getitem_basic(self):

From noreply at buildbot.pypy.org  Fri Dec  5 02:38:11 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 02:38:11 +0100 (CET)
Subject: [pypy-commit] pypy default: add test_pypy_c for logical_xor.reduce
Message-ID: <20141205013811.D716B1C100F@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74822:81a86598865d
Date: 2014-12-04 20:25 -0500
http://bitbucket.org/pypy/pypy/changeset/81a86598865d/

Log:	add test_pypy_c for logical_xor.reduce

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -3,6 +3,59 @@
 
 
 class TestMicroNumPy(BaseTestPyPyC):
+    def test_reduce_logical_xor(self):
+        def main():
+            import _numpypy.multiarray as np
+            import _numpypy.umath as um
+            arr = np.array([1.0] * 1500)
+            return um.logical_xor.reduce(arr)
+        log = self.run(main, [])
+        assert log.result is False
+        assert len(log.loops) == 1
+        loop = log._filter(log.loops[0])
+        assert loop.match("""
+            guard_class(p0, #, descr=...)
+            p4 = getfield_gc_pure(p0, descr=<FieldP pypy.module.micronumpy.iterators.ArrayIter.inst_array \d+>)
+            i5 = getfield_gc(p2, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset \d+>)
+            p6 = getfield_gc_pure(p4, descr=<FieldP pypy.module.micronumpy.concrete.BaseConcreteArray.inst_dtype \d+>)
+            p7 = getfield_gc_pure(p6, descr=<FieldP pypy.module.micronumpy.descriptor.W_Dtype.inst_itemtype \d+>)
+            guard_class(p7, ConstClass(Float64), descr=...)
+            i9 = getfield_gc_pure(p4, descr=<FieldU pypy.module.micronumpy.concrete.BaseConcreteArray.inst_storage \d+>)
+            f10 = raw_load(i9, i5, descr=<ArrayF \d+>)
+            i11 = getfield_gc_pure(p7, descr=<FieldU pypy.module.micronumpy.types.BaseType.inst_native \d+>)
+            guard_true(i11, descr=...)
+            guard_not_invalidated(descr=...)
+            i12 = cast_float_to_int(f10)
+            i14 = int_and(i12, 255)
+            guard_true(i14, descr=...)
+            i15 = getfield_gc_pure(p1, descr=<FieldU pypy.module.micronumpy.boxes.W_BoolBox.inst_value \d+>)
+            i16 = int_is_true(i15)
+            i18 = int_xor(i16, 1)
+            i19 = int_is_true(i18)
+            guard_true(i19, descr=...)
+            i20 = getfield_gc(p2, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index \d+>)
+            i21 = getfield_gc_pure(p0, descr=<FieldU pypy.module.micronumpy.iterators.ArrayIter.inst_track_index \d+>)
+            guard_true(i21, descr=...)
+            i23 = int_add(i20, 1)
+            p24 = getfield_gc_pure(p2, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices \d+>)
+            i25 = getfield_gc_pure(p0, descr=<FieldS pypy.module.micronumpy.iterators.ArrayIter.inst_contiguous \d+>)
+            i26 = int_is_true(i25)
+            guard_true(i26, descr=...)
+            i27 = getfield_gc_pure(p6, descr=<FieldS pypy.module.micronumpy.descriptor.W_Dtype.inst_elsize \d+>)
+            i28 = int_add(i5, i27)
+            i29 = getfield_gc_pure(p0, descr=<FieldS pypy.module.micronumpy.iterators.ArrayIter.inst_size \d+>)
+            i30 = int_ge(i23, i29)
+            guard_false(i30, descr=...)
+            p32 = new_with_vtable(#)
+            {{{
+            setfield_gc(p32, i23, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index \d+>)
+            setfield_gc(p32, p24, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst__indices \d+>)
+            setfield_gc(p32, i28, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_offset \d+>)
+            setfield_gc(p32, p0, descr=<FieldP pypy.module.micronumpy.iterators.IterState.inst_iterator \d+>)
+            }}}
+            jump(..., descr=...)
+        """)
+
     def test_reduce_logical_and(self):
         def main():
             import _numpypy.multiarray as np

From noreply at buildbot.pypy.org  Fri Dec  5 05:02:06 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 05:02:06 +0100 (CET)
Subject: [pypy-commit] pypy default: fix this test_pypy_c,
	these numbers can vary
Message-ID: <20141205040206.68D781C07BB@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74823:e27f4fbd7134
Date: 2014-12-04 21:57 -0500
http://bitbucket.org/pypy/pypy/changeset/e27f4fbd7134/

Log:	fix this test_pypy_c, these numbers can vary

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -72,14 +72,14 @@
             i32 = cast_float_to_int(f31)
             i33 = int_and(i32, 255)
             guard_true(i33, descr=...)
-            i34 = getarrayitem_raw(#, 2, descr=<ArrayU 1>)  # XXX what are these?
-            guard_value(i34, 1, descr=...)                  # XXX don't appear in
-            i35 = getarrayitem_raw(#, 1, descr=<ArrayU 1>)  # XXX equiv test_zjit
+            i34 = getarrayitem_raw(#, #, descr=<ArrayU 1>)  # XXX what are these?
+            guard_value(i34, #, descr=...)                  # XXX don't appear in
+            i35 = getarrayitem_raw(#, #, descr=<ArrayU 1>)  # XXX equiv test_zjit
             i36 = int_add(i24, 1)
             i37 = int_add(i29, i28)
             i38 = int_ge(i36, i30)
             guard_false(i38, descr=...)
-            guard_value(i35, 2, descr=...)                  # XXX
+            guard_value(i35, #, descr=...)                  # XXX
             jump(..., descr=...)
         """)
 

From noreply at buildbot.pypy.org  Fri Dec  5 06:50:14 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 06:50:14 +0100 (CET)
Subject: [pypy-commit] pypy default: test/fix ufunc reduce with comparison
 func when dtype specified
Message-ID: <20141205055014.464A61D28F0@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74824:4213885db36d
Date: 2014-12-05 00:23 -0500
http://bitbucket.org/pypy/pypy/changeset/4213885db36d/

Log:	test/fix ufunc reduce with comparison func when dtype specified

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -785,6 +785,7 @@
         assert exc.value[0] == "'axis' entry is out of bounds"
 
     def test_reduce_1d(self):
+        import numpy as np
         from numpypy import array, add, maximum, less, float16, complex64
 
         assert less.reduce([5, 4, 3, 2, 1])
@@ -799,6 +800,10 @@
         assert type(add.reduce(array([True, False] * 200, dtype='float16'))) is float16
         assert type(add.reduce(array([True, False] * 200, dtype='complex64'))) is complex64
 
+        for dtype in ['bool', 'int']:
+            assert np.equal.reduce([1, 2], dtype=dtype) == True
+            assert np.equal.reduce([1, 2, 0], dtype=dtype) == False
+
     def test_reduceND(self):
         from numpypy import add, arange
         a = arange(12).reshape(3, 4)
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -196,16 +196,15 @@
                 axis += shapelen
         assert axis >= 0
         dtype = descriptor.decode_w_dtype(space, dtype)
-        if dtype is None:
-            if self.comparison_func:
-                dtype = descriptor.get_dtype_cache(space).w_booldtype
-            else:
-                dtype = find_unaryop_result_dtype(
-                    space, obj.get_dtype(),
-                    promote_to_float=self.promote_to_float,
-                    promote_to_largest=self.promote_to_largest,
-                    promote_bools=self.promote_bools,
-                )
+        if self.comparison_func:
+            dtype = descriptor.get_dtype_cache(space).w_booldtype
+        elif dtype is None:
+            dtype = find_unaryop_result_dtype(
+                space, obj.get_dtype(),
+                promote_to_float=self.promote_to_float,
+                promote_to_largest=self.promote_to_largest,
+                promote_bools=self.promote_bools,
+            )
         if self.identity is None:
             for i in range(shapelen):
                 if space.is_none(w_axis) or i == axis:

From noreply at buildbot.pypy.org  Fri Dec  5 10:00:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 10:00:44 +0100 (CET)
Subject: [pypy-commit] creflect default: global vars, starting
Message-ID: <20141205090044.830AE1C03D5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r156:f5bc966d0d96
Date: 2014-12-05 10:01 +0100
http://bitbucket.org/cffi/creflect/changeset/f5bc966d0d96/

Log:	global vars, starting

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -413,7 +413,17 @@
 static void zef_define_var(_crx_builder_t *cb, const char *name,
                            _crx_type_t *type, int quals, const void *addr)
 {
-    abort();
+    if (PyErr_Occurred())
+        return;
+
+    PyObject *l_dict = ((zeffir_builder_t *)cb)->l_dict;
+
+    PyObject *x = make_global_var(type, addr);
+    if (x == NULL)
+        return;
+
+    PyDict_SetItemString(l_dict, name, x);
+    Py_DECREF(x);
 }
 
 static void zef_define_func(_crx_builder_t *cb, const char *name,
diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1,1 +1,56 @@
 
+struct CDataObject_s {
+    PyObject_HEAD
+    CTypeDescrObject *c_type;
+    char *c_data;
+    PyObject *c_weakreflist;
+};
+
+/*
+#define CData_Check(ob)       (Py_TYPE(ob) == &CData_Type ||            \
+                               Py_TYPE(ob) == &CDataOwning_Type ||      \
+                               Py_TYPE(ob) == &CDataOwningGC_Type)
+#define CDataOwn_Check(ob)    (Py_TYPE(ob) == &CDataOwning_Type ||      \
+                               Py_TYPE(ob) == &CDataOwningGC_Type)
+*/
+
+static void cdata_dealloc(CDataObject *cd)
+{
+    if (cd->c_weakreflist != NULL)
+        PyObject_ClearWeakRefs((PyObject *) cd);
+
+    Py_DECREF(cd->c_type);
+#ifndef CFFI_MEM_LEAK     /* never release anything, tests only */
+    Py_TYPE(cd)->tp_free((PyObject *)cd);
+#endif
+}
+
+static PyTypeObject CData_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.CData",
+    sizeof(CDataObject),
+    0,
+    (destructor)cdata_dealloc,                  /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    0,//(reprfunc)cdata_repr,                       /* tp_repr */
+    0,//&CData_as_number,                           /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,//&CData_as_mapping,                          /* tp_as_mapping */
+    0,//(hashfunc)cdata_hash,                       /* tp_hash */
+    0,//(ternaryfunc)cdata_call,                    /* tp_call */
+    0,                                          /* tp_str */
+    0,//(getattrofunc)cdata_getattro,               /* tp_getattro */
+    0,//(setattrofunc)cdata_setattro,               /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */
+    0,                                          /* tp_doc */
+    0,                                          /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,//cdata_richcompare,                          /* tp_richcompare */
+    offsetof(CDataObject, c_weakreflist),       /* tp_weaklistoffset */
+    0,//(getiterfunc)cdata_iter,                    /* tp_iter */
+    0,                                          /* tp_iternext */
+};
diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -9,7 +9,7 @@
 
 } ZefFuncSupportObject;
 
-static void zef_builtin_support_dealloc(ZefFuncSupportObject *zfs)
+static void zef_func_support_dealloc(ZefFuncSupportObject *zfs)
 {
     PyObject_Del(zfs);
 }
@@ -19,7 +19,7 @@
     "zeffir.FuncSupport",
     sizeof(ZefFuncSupportObject),
     0,
-    (destructor)zef_builtin_support_dealloc,    /* tp_dealloc */
+    (destructor)zef_func_support_dealloc,       /* tp_dealloc */
     0,                                          /* tp_print */
     0,                                          /* tp_getattr */
     0,                                          /* tp_setattr */
diff --git a/zeffir/cglob.c b/zeffir/cglob.c
new file mode 100644
--- /dev/null
+++ b/zeffir/cglob.c
@@ -0,0 +1,53 @@
+
+typedef struct {
+    PyObject_HEAD
+
+    CTypeDescrObject *zgs_type;
+    char             *zgs_data;
+
+} ZefGlobSupportObject;
+
+static void zef_glob_support_dealloc(ZefGlobSupportObject *zgs)
+{
+    Py_DECREF(zgs->zgs_type);
+    PyObject_Del(zgs);
+}
+
+static PyTypeObject ZefGlobSupport_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.GlobSupport",
+    sizeof(ZefGlobSupportObject),
+    0,
+    (destructor)zef_glob_support_dealloc,       /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    0,                                          /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
+};
+
+#define ZefGlobSupport_Check(ob)  (Py_TYPE(ob) == &ZefGlobSupport_Type)
+
+static PyObject *make_global_var(_crx_type_t *type, const void *addr)
+{
+    ZefGlobSupportObject *zgs =
+        PyObject_New(ZefGlobSupportObject, &ZefGlobSupport_Type);
+    if (zgs == NULL)
+        return NULL;
+
+    Py_INCREF(type);
+    zgs->zgs_type = type;
+    zgs->zgs_data = (char *)addr;
+    assert(addr != NULL);
+    return (PyObject *)zgs;
+}
diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -49,18 +49,7 @@
     char ct_name[1];        /* string, e.g. "int *" for pointers to ints */
 };
 
-static PyTypeObject CTypeDescr_Type;
-//static PyTypeObject CField_Type;
-//static PyTypeObject CData_Type;
-//static PyTypeObject CDataOwning_Type;
-//static PyTypeObject CDataOwningGC_Type;
-
 #define CTypeDescr_Check(ob)  (Py_TYPE(ob) == &CTypeDescr_Type)
-#define CData_Check(ob)       (Py_TYPE(ob) == &CData_Type ||            \
-                               Py_TYPE(ob) == &CDataOwning_Type ||      \
-                               Py_TYPE(ob) == &CDataOwningGC_Type)
-#define CDataOwn_Check(ob)    (Py_TYPE(ob) == &CDataOwning_Type ||      \
-                               Py_TYPE(ob) == &CDataOwningGC_Type)
 
 /************************************************************/
 
diff --git a/zeffir/lib_obj.c b/zeffir/lib_obj.c
--- a/zeffir/lib_obj.c
+++ b/zeffir/lib_obj.c
@@ -25,8 +25,10 @@
                                lib->l_dl_lib == NULL ? " (closed)" : "");
 }
 
-static PyObject *lib_getattr(ZefLibObject *lib, PyObject *name)
+static PyObject *lib_findattr(ZefLibObject *lib, PyObject *name)
 {
+    /* does not return a new reference! */
+
     if (lib->l_dict == NULL) {
         PyErr_Format(ZefError, "lib '%.200s' was closed", lib->l_libname);
         return NULL;
@@ -39,13 +41,49 @@
                      " global variable or constant '%.200s'",
                      lib->l_libname,
                      PyString_Check(name) ? PyString_AS_STRING(name) : "?");
+        return NULL;
     }
     return x;
 }
 
+static PyObject *lib_getattr(ZefLibObject *lib, PyObject *name)
+{
+    PyObject *x = lib_findattr(lib, name);
+    if (x == NULL)
+        return NULL;
+
+    if (ZefGlobSupport_Check(x)) {
+        // XXX!
+        return PyInt_FromLong(*(int *)(((ZefGlobSupportObject *)x)->zgs_data));
+    }
+    Py_INCREF(x);
+    return x;
+}
+
 static int lib_setattr(ZefLibObject *lib, PyObject *name, PyObject *val)
 {
-    PyErr_SetString(PyExc_AttributeError, "Lib object is read-only");
+    PyObject *x = lib_findattr(lib, name);
+    if (x == NULL)
+        return -1;
+
+    if (val == NULL) {
+        PyErr_SetString(PyExc_AttributeError,
+                        "cannot delete attributes from Lib object");
+        return -1;
+    }
+
+    if (ZefGlobSupport_Check(x)) {
+        // XXX!
+        long y = PyInt_AsLong(val);
+        if (PyErr_Occurred())
+            return -1;
+        *(int *)(((ZefGlobSupportObject *)x)->zgs_data) = y;
+        return 0;
+    }
+
+    PyErr_Format(PyExc_AttributeError,
+                 "cannot write to function or constant '%.200s'",
+                 PyString_Check(name) ? PyString_AS_STRING(name) : "?");
     return -1;
 }
 
diff --git a/zeffir/test/global.crx b/zeffir/test/global.crx
new file mode 100644
--- /dev/null
+++ b/zeffir/test/global.crx
@@ -0,0 +1,8 @@
+short myglob = 42;
+
+
+// CREFLECT: start
+
+short myglob;
+
+// CREFLECT: end
diff --git a/zeffir/test/test_global.py b/zeffir/test/test_global.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_global.py
@@ -0,0 +1,17 @@
+import py
+import support
+
+
+def test_simple_global():
+    ffi, lib = support.compile_and_open('global')
+    res = lib.myglob
+    assert type(res) is int
+    assert res == 42
+    #
+    lib.myglob = -43
+    res = lib.myglob
+    assert type(res) is int
+    assert res == -43
+    #
+    #py.test.raises(OverflowError, setattr, lib, 'myglob', 40000)
+    #py.test.raises(OverflowError, setattr, lib, 'myglob', -40000)
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -12,6 +12,7 @@
 #include "zeffir.h"
 #include "ctype.c"
 #include "cdata.c"
+#include "cglob.c"
 #include "lib_obj.c"
 #include "ffi_obj.c"
 #include "cfunc.c"
@@ -39,13 +40,21 @@
         return;
     if (PyType_Ready(&CTypeDescr_Type) < 0)
         return;
+    if (PyType_Ready(&CData_Type) < 0)
+        return;
     if (PyType_Ready(&ZefFuncSupport_Type) < 0)
         return;
+    if (PyType_Ready(&ZefGlobSupport_Type) < 0)
+        return;
 
     if (PyModule_AddObject(m, "FFI", (PyObject *)&ZefFFI_Type) < 0)
         return;
+    if (PyModule_AddObject(m, "Lib", (PyObject *)&ZefLib_Type) < 0)
+        return;
     if (PyModule_AddObject(m, "CType", (PyObject *)&CTypeDescr_Type) < 0)
         return;
+    if (PyModule_AddObject(m, "CData", (PyObject *)&CData_Type) < 0)
+        return;
 
     ZefError = PyErr_NewException("zeffir.error", NULL, NULL);
     if (ZefError == NULL)
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -1,9 +1,15 @@
 
 typedef struct _crx_type_s CTypeDescrObject;
+typedef struct CDataObject_s CDataObject;
 typedef struct ZefLibObject_s ZefLibObject;
 typedef struct ZefFFIObject_s ZefFFIObject;
 
 static PyTypeObject ZefFFI_Type;
+static PyTypeObject CTypeDescr_Type;
+//static PyTypeObject CField_Type;
+static PyTypeObject CData_Type;
+//static PyTypeObject CDataOwning_Type;
+//static PyTypeObject CDataOwningGC_Type;
 
 static PyObject *ZefError;
 

From noreply at buildbot.pypy.org  Fri Dec  5 10:04:03 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri,  5 Dec 2014 10:04:03 +0100 (CET)
Subject: [pypy-commit] stmgc c8-private-pages: merge (hopefully) fixed
	_stm_validate()
Message-ID: <20141205090403.0AFB71C03D5@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch: c8-private-pages
Changeset: r1514:970af41866c7
Date: 2014-12-05 10:05 +0100
http://bitbucket.org/pypy/stmgc/changeset/970af41866c7/

Log:	merge (hopefully) fixed _stm_validate()

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -278,87 +278,108 @@
         return;
     }
 
-    /* Find the set of segments we need to copy from and lock them: */
-    uint64_t segments_to_lock = 1UL << my_segnum;
-    cl = first_cl;
-    while ((next_cl = cl->next) != NULL) {
-        if (next_cl == INEV_RUNNING) {
+    bool needs_abort = false;
+    
+    while(1) {
+        /* retry IF: */
+        /* if at the time of "HERE" (s.b.) there happen to be
+           more commits (and bk copies) then it could be that
+           copy_bk_objs_in_page_from (s.b.) reads a bk copy that
+           is itself more recent than last_cl. This is fixed
+           by re-validating. */
+        first_cl = STM_PSEGMENT->last_commit_log_entry;
+        if (first_cl->next == NULL)
+            break;
+
+        /* Find the set of segments we need to copy from and lock them: */
+        uint64_t segments_to_lock = 1UL << my_segnum;
+        cl = first_cl;
+        while ((next_cl = cl->next) != NULL) {
+            if (next_cl == INEV_RUNNING) {
 #if STM_TESTS
-            if (free_if_abort != (void *)-1)
-                free(free_if_abort);
-            stm_abort_transaction();
+                if (free_if_abort != (void *)-1)
+                    free(free_if_abort);
+                stm_abort_transaction();
 #endif
-            /* only validate entries up to INEV */
-            break;
+                /* only validate entries up to INEV */
+                break;
+            }
+            assert(next_cl->rev_num > cl->rev_num);
+            cl = next_cl;
+
+            if (cl->written_count) {
+                segments_to_lock |= (1UL << cl->segment_num);
+            }
         }
-        assert(next_cl->rev_num > cl->rev_num);
-        cl = next_cl;
+        last_cl = cl;
+        /* HERE */
+        acquire_modification_lock_set(segments_to_lock);
 
-        if (cl->written_count) {
-            segments_to_lock |= (1UL << cl->segment_num);
-        }
-    }
-    last_cl = cl;
-    acquire_modification_lock_set(segments_to_lock);
 
+        /* import objects from first_cl to last_cl: */
+        if (first_cl != last_cl) {
+            uint64_t segment_really_copied_from = 0UL;
 
-    /* import objects from first_cl to last_cl: */
-    bool needs_abort = false;
-    if (first_cl != last_cl) {
-        uint64_t segment_really_copied_from = 0UL;
-
-        cl = first_cl;
-        while ((cl = cl->next) != NULL) {
-            if (!needs_abort) {
-                struct stm_undo_s *undo = cl->written;
-                struct stm_undo_s *end = cl->written + cl->written_count;
-                for (; undo < end; undo++) {
-                    if (_stm_was_read(undo->object)) {
-                        /* first reset all modified objects from the backup
-                           copies as soon as the first conflict is detected;
-                           then we will proceed below to update our segment from
-                           the old (but unmodified) version to the newer version.
-                        */
-                        reset_modified_from_backup_copies(my_segnum);
-                        needs_abort = true;
-                        break;
+            cl = first_cl;
+            while ((cl = cl->next) != NULL) {
+                if (!needs_abort) {
+                    struct stm_undo_s *undo = cl->written;
+                    struct stm_undo_s *end = cl->written + cl->written_count;
+                    for (; undo < end; undo++) {
+                        if (_stm_was_read(undo->object)) {
+                            /* first reset all modified objects from the backup
+                               copies as soon as the first conflict is detected;
+                               then we will proceed below to update our segment from
+                               the old (but unmodified) version to the newer version.
+                            */
+                            reset_modified_from_backup_copies(my_segnum);
+                            needs_abort = true;
+                            break;
+                        }
                     }
                 }
+
+                if (cl->written_count) {
+                    struct stm_undo_s *undo = cl->written;
+                    struct stm_undo_s *end = cl->written + cl->written_count;
+
+                    segment_really_copied_from |= (1UL << cl->segment_num);
+                    import_objects(cl->segment_num, -1, undo, end);
+
+                    /* here we can actually have our own modified version, so
+                       make sure to only copy things that are not modified in our
+                       segment... (if we do not abort) */
+                    copy_bk_objs_in_page_from
+                        (cl->segment_num, -1,     /* any page */
+                         !needs_abort);  /* if we abort, we still want to copy everything */
+                }
+
+                /* last fully validated entry */
+                STM_PSEGMENT->last_commit_log_entry = cl;
+                if (cl == last_cl)
+                    break;
             }
+            assert(cl == last_cl);
 
-            if (cl->written_count) {
-                struct stm_undo_s *undo = cl->written;
-                struct stm_undo_s *end = cl->written + cl->written_count;
+            /* XXX: this optimization fails in test_basic.py, bug3 */
+            /* OPT_ASSERT(segment_really_copied_from < (1 << NB_SEGMENTS)); */
+            /* int segnum; */
+            /* for (segnum = 0; segnum < NB_SEGMENTS; segnum++) { */
+            /*     if (segment_really_copied_from & (1UL << segnum)) { */
+            /*         /\* here we can actually have our own modified version, so */
+            /*            make sure to only copy things that are not modified in our */
+            /*            segment... (if we do not abort) *\/ */
+            /*         copy_bk_objs_in_page_from( */
+            /*             segnum, -1,     /\* any page *\/ */
+            /*             !needs_abort);  /\* if we abort, we still want to copy everything *\/ */
+            /*     } */
+            /* } */
+        }
 
-                segment_really_copied_from |= (1UL << cl->segment_num);
-                import_objects(cl->segment_num, -1, undo, end);
-
-                copy_bk_objs_in_page_from(
-                    segnum, -1,     /* any page */
-                    !needs_abort);  /* if we abort, we still want to copy everything */
-            }
-
-            /* last fully validated entry */
-            STM_PSEGMENT->last_commit_log_entry = cl;
-            if (cl == last_cl)
-                break;
-        }
-        assert(cl == last_cl);
-
-        /* OPT_ASSERT(segment_really_copied_from <= ((1UL << NB_SEGMENTS) - 1)); */
-        /* int segnum; */
-        /* for (segnum = 0; segnum < NB_SEGMENTS; segnum++) { */
-        /*     if (segment_really_copied_from & (1UL << segnum)) { */
-        /*         /\* here we can actually have our own modified version, so */
-        /*            make sure to only copy things that are not modified in our */
-        /*            segment... (if we do not abort) *\/ */
-        /*     } */
-        /* } */
+        /* done with modifications */
+        release_modification_lock_set(segments_to_lock);
     }
 
-    /* done with modifications */
-    release_modification_lock_set(segments_to_lock);
-
     if (needs_abort) {
         if (free_if_abort != (void *)-1)
             free(free_if_abort);
@@ -490,6 +511,7 @@
     assert(!(bk_obj->stm_flags & GCFLAG_WB_EXECUTED));
 
     dprintf(("write_slowpath(%p): sz=%lu, bk=%p\n", obj, obj_size, bk_obj));
+
  retry:
     /* privatize pages: */
     /* XXX don't always acquire all locks... */
@@ -540,6 +562,7 @@
     /* all pages are either private or we were the first to write to a shared
        page and therefore got it as our private one */
 
+
     /* phew, now add the obj to the write-set and register the
        backup copy. */
     /* XXX: we should not be here at all fiddling with page status
diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py
--- a/c8/test/test_basic.py
+++ b/c8/test/test_basic.py
@@ -811,6 +811,38 @@
 
         assert stm_get_char(lp_char_5, 384 - 1) == 'o'
 
+    def test_bug3(self):
+        lp_char_5 = stm_allocate_old(384)
+
+        for i in range(NB_SEGMENTS):
+            self.start_transaction()
+            stm_set_char(lp_char_5, '\0', HDR, False)
+            self.commit_transaction()
+            
+        #
+        self.switch(2)
+        self.start_transaction()
+        stm_set_char(lp_char_5, 'i', HDR, False)
+        self.commit_transaction()
+
+        self.start_transaction()
+        stm_set_char(lp_char_5, 'x', HDR, False)
+
+        #
+        self.switch(1)
+        self.start_transaction()
+        stm_set_char(lp_char_5, 'a', HDR, False)
+        self.commit_transaction()
+
+        #
+        self.switch(0)
+        self.start_transaction()
+        assert stm_get_char(lp_char_5, HDR) == 'a'
+
+        #
+        py.test.raises(Conflict, self.switch, 2)
+
+        
     def test_repeated_wb(self):
         lp_char_5 = stm_allocate_old(384)
 

From noreply at buildbot.pypy.org  Fri Dec  5 10:40:04 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 10:40:04 +0100 (CET)
Subject: [pypy-commit] creflect default: starting to port some more cdata
	logic
Message-ID: <20141205094004.D9C501D2E18@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r157:46722c958fdb
Date: 2014-12-05 10:40 +0100
http://bitbucket.org/cffi/creflect/changeset/46722c958fdb/

Log:	starting to port some more cdata logic

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -269,7 +269,10 @@
     if (ct == NULL)
         goto done;
 
+    Py_INCREF(totype);
+    ct->ct_itemdescr = totype;
     ct->ct_size = sizeof(void *);
+
     ct->ct_flags = CT_POINTER;
     if (totype->ct_flags & CT_VOID)
         ct->ct_flags |= CT_IS_VOID_PTR;
diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1,18 +1,53 @@
 
-struct CDataObject_s {
+typedef struct {
     PyObject_HEAD
     CTypeDescrObject *c_type;
     char *c_data;
     PyObject *c_weakreflist;
-};
+} CDataObject;
 
-/*
+typedef union {
+    unsigned char m_char;
+    unsigned short m_short;
+    unsigned int m_int;
+    unsigned long m_long;
+    unsigned long long m_longlong;
+    float m_float;
+    double m_double;
+    long double m_longdouble;
+} union_alignment;
+
+typedef struct {
+    CDataObject head;
+    union_alignment alignment;
+} CDataObject_casted_primitive;
+
+typedef struct {
+    CDataObject head;
+    union_alignment alignment;
+} CDataObject_own_nolength;
+
+typedef struct {
+    CDataObject head;
+    Py_ssize_t length;
+    union_alignment alignment;
+} CDataObject_own_length;
+
 #define CData_Check(ob)       (Py_TYPE(ob) == &CData_Type ||            \
                                Py_TYPE(ob) == &CDataOwning_Type ||      \
                                Py_TYPE(ob) == &CDataOwningGC_Type)
 #define CDataOwn_Check(ob)    (Py_TYPE(ob) == &CDataOwning_Type ||      \
                                Py_TYPE(ob) == &CDataOwningGC_Type)
-*/
+
+
+static Py_ssize_t get_array_length(CDataObject *cd)
+{
+    if (cd->c_type->ct_length < 0)
+        return ((CDataObject_own_length *)cd)->length;
+    else
+        return cd->c_type->ct_length;
+}
+
 
 static void cdata_dealloc(CDataObject *cd)
 {
@@ -25,6 +60,169 @@
 #endif
 }
 
+static void cdataowninggc_dealloc(CDataObject *cd)
+{
+    abort();
+#if 0
+    assert(!(cd->c_type->ct_flags & (CT_IS_PTR_TO_OWNED |
+                                     CT_PRIMITIVE_ANY |
+                                     CT_STRUCT | CT_UNION)));
+    PyObject_GC_UnTrack(cd);
+
+    if (cd->c_type->ct_flags & CT_IS_VOID_PTR) {        /* a handle */
+        PyObject *x = (PyObject *)(cd->c_data + 42);
+        Py_DECREF(x);
+    }
+    else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) {   /* a callback */
+        ffi_closure *closure = (ffi_closure *)cd->c_data;
+        PyObject *args = (PyObject *)(closure->user_data);
+        Py_XDECREF(args);
+        cffi_closure_free(closure);
+    }
+    cdata_dealloc(cd);
+#endif
+}
+
+static int cdataowninggc_traverse(CDataObject *cd, visitproc visit, void *arg)
+{
+    abort();
+#if 0
+    if (cd->c_type->ct_flags & CT_IS_VOID_PTR) {        /* a handle */
+        PyObject *x = (PyObject *)(cd->c_data + 42);
+        Py_VISIT(x);
+    }
+    else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) {   /* a callback */
+        ffi_closure *closure = (ffi_closure *)cd->c_data;
+        PyObject *args = (PyObject *)(closure->user_data);
+        Py_VISIT(args);
+    }
+    return 0;
+#endif
+}
+
+static int cdataowninggc_clear(CDataObject *cd)
+{
+    abort();
+#if 0
+    if (cd->c_type->ct_flags & CT_IS_VOID_PTR) {        /* a handle */
+        PyObject *x = (PyObject *)(cd->c_data + 42);
+        Py_INCREF(Py_None);
+        cd->c_data = ((char *)Py_None) - 42;
+        Py_DECREF(x);
+    }
+    else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) {   /* a callback */
+        ffi_closure *closure = (ffi_closure *)cd->c_data;
+        PyObject *args = (PyObject *)(closure->user_data);
+        closure->user_data = NULL;
+        Py_XDECREF(args);
+    }
+    return 0;
+#endif
+}
+
+static PyObject *cdata_repr(CDataObject *cd)
+{
+    char *extra;
+    PyObject *result, *s;
+
+    if (cd->c_type->ct_flags & CT_PRIMITIVE_ANY) {
+        abort();//XXX
+#if 0
+        if (cd->c_type->ct_flags & CT_IS_ENUM) {
+            s = convert_cdata_to_enum_string(cd, 1);
+        }
+        else if (cd->c_type->ct_flags & CT_IS_LONGDOUBLE) {
+            long double lvalue;
+            char buffer[128];   /* big enough */
+            /*READ(cd->c_data, sizeof(long double)*/
+            lvalue = read_raw_longdouble_data(cd->c_data);
+            sprintf(buffer, "%LE", lvalue);
+            s = PyText_FromString(buffer);
+        }
+        else {
+            PyObject *o = convert_to_object(cd->c_data, cd->c_type);
+            if (o == NULL)
+                return NULL;
+            s = PyObject_Repr(o);
+            Py_DECREF(o);
+        }
+#endif
+    }
+    else if ((cd->c_type->ct_flags & CT_ARRAY) && cd->c_type->ct_length < 0) {
+        s = PyString_FromFormat("sliced length %zd", get_array_length(cd));
+    }
+    else {
+        if (cd->c_data != NULL) {
+            s = PyString_FromFormat("%p", cd->c_data);
+        }
+        else
+            s = PyString_FromString("NULL");
+    }
+    if (s == NULL)
+        return NULL;
+    /* it's slightly confusing to get "<cdata 'struct foo' 0x...>" because the
+       struct foo is not owned.  Trying to make it clearer, write in this
+       case "<cdata 'struct foo &' 0x...>". */
+    if (cd->c_type->ct_flags & (CT_STRUCT|CT_UNION))
+        extra = " &";
+    else
+        extra = "";
+    result = PyString_FromFormat("<cdata '%s%s' %s>",
+                                 cd->c_type->ct_name, extra,
+                                 PyString_AsString(s));
+    Py_DECREF(s);
+    return result;
+}
+
+static PyObject *_cdata_repr2(CDataObject *cd, char *text, PyObject *x)
+{
+    PyObject *res, *s = PyObject_Repr(x);
+    if (s == NULL)
+        return NULL;
+    res = PyString_FromFormat("<cdata '%s' %s %s>",
+                              cd->c_type->ct_name, text, PyString_AsString(s));
+    Py_DECREF(s);
+    return res;
+}
+
+static PyObject *cdataowning_repr(CDataObject *cd)
+{
+    Py_ssize_t size;
+    if (cd->c_type->ct_flags & CT_POINTER) {
+        if (cd->c_type->ct_flags & CT_IS_VOID_PTR)
+            goto handle_repr;
+        size = cd->c_type->ct_itemdescr->ct_size;
+    }
+    else if (cd->c_type->ct_flags & CT_ARRAY)
+        size = get_array_length(cd) * cd->c_type->ct_itemdescr->ct_size;
+#if 0  // XXX
+    else if (cd->c_type->ct_flags & CT_FUNCTIONPTR)
+        goto callback_repr;
+#endif
+    else
+        size = cd->c_type->ct_size;
+
+    return PyString_FromFormat("<cdata '%s' owning %zd bytes>",
+                               cd->c_type->ct_name, size);
+
+#if 0
+ callback_repr:
+    {
+        PyObject *args = (PyObject *)((ffi_closure *)cd->c_data)->user_data;
+        if (args == NULL)
+            return cdata_repr(cd);
+        else
+            return _cdata_repr2(cd, "calling", PyTuple_GET_ITEM(args, 1));
+    }
+#endif
+
+ handle_repr:
+    {
+        PyObject *x = (PyObject *)(cd->c_data + 42);
+        return _cdata_repr2(cd, "handle to", x);
+    }
+}
+
 static PyTypeObject CData_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "zeffir.CData",
@@ -35,7 +233,7 @@
     0,                                          /* tp_getattr */
     0,                                          /* tp_setattr */
     0,                                          /* tp_compare */
-    0,//(reprfunc)cdata_repr,                       /* tp_repr */
+    (reprfunc)cdata_repr,                       /* tp_repr */
     0,//&CData_as_number,                           /* tp_as_number */
     0,                                          /* tp_as_sequence */
     0,//&CData_as_mapping,                          /* tp_as_mapping */
@@ -54,3 +252,86 @@
     0,//(getiterfunc)cdata_iter,                    /* tp_iter */
     0,                                          /* tp_iternext */
 };
+
+static PyTypeObject CDataOwning_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.CDataOwn",
+    sizeof(CDataObject),
+    0,
+    (destructor)cdata_dealloc,                  /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    (reprfunc)cdataowning_repr,                 /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,//&CDataOwn_as_mapping,                       /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    0,                                          /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */
+    0,                                          /* tp_doc */
+    0,                                          /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    0,                                          /* tp_members */
+    0,                                          /* tp_getset */
+    &CData_Type,                                /* tp_base */
+};
+
+static PyTypeObject CDataOwningGC_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.CDataOwnGC",
+    sizeof(CDataObject),
+    0,
+    (destructor)cdataowninggc_dealloc,          /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    0,                                          /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    0,                                          /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES  /* tp_flags */
+                       | Py_TPFLAGS_HAVE_GC,
+    0,                                          /* tp_doc */
+    (traverseproc)cdataowninggc_traverse,       /* tp_traverse */
+    (inquiry)cdataowninggc_clear,               /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    0,                                          /* tp_members */
+    0,                                          /* tp_getset */
+    &CDataOwning_Type,                          /* tp_base */
+};
+
+static CDataObject *allocate_owning_object(Py_ssize_t size,
+                                           CTypeDescrObject *ct)
+{
+    CDataObject *cd;
+    cd = (CDataObject *)PyObject_Malloc(size);
+    if (PyObject_Init((PyObject *)cd, &CDataOwning_Type) == NULL)
+        return NULL;
+
+    Py_INCREF(ct);
+    cd->c_type = ct;
+    cd->c_weakreflist = NULL;
+    return cd;
+}
diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -16,7 +16,6 @@
 #define CT_CAST_ANYTHING         2048    /* 'char *' and 'void *' only */
 #define CT_PRIMITIVE_FITS_LONG   4096
 #define CT_IS_ENUM               8192
-#define CT_IS_PTR_TO_OWNED      16384
 #define CT_IS_LONGDOUBLE        65536
 #define CT_IS_BOOL             131072
 #define CT_IS_VOID_PTR         524288
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -35,7 +35,7 @@
 static PyObject *ffiobj_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
     char *keywords[] = {NULL};
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "", keywords))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, ":FFI", keywords))
         return NULL;
 
     PyObject *dict = PyDict_New();
@@ -62,7 +62,7 @@
     PyObject *path;
     ZefLibObject *lib;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|z", keywords,
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|z:load_library", keywords,
                                      &libname, &relative_to))
         return NULL;
 
@@ -100,7 +100,7 @@
 {
     ZefLibObject *lib;
 
-    if (!PyArg_ParseTuple(args, "O!", &lib, &ZefLib_Type))
+    if (!PyArg_ParseTuple(args, "O!:close_library", &lib, &ZefLib_Type))
         return NULL;
 
     if (lib_close(lib) < 0)
@@ -110,12 +110,18 @@
     return Py_None;
 }
 
-static CTypeDescrObject *_ffi_type(ZefFFIObject *ffi, PyObject *arg)
+#define ACCEPT_STRING   1
+#define ACCEPT_CTYPE    2
+#define ACCEPT_CDATA    4
+#define ACCEPT_ALL      (ACCEPT_STRING | ACCEPT_CTYPE | ACCEPT_CDATA)
+
+static CTypeDescrObject *_ffi_type(ZefFFIObject *ffi, PyObject *arg,
+                                   int accept)
 {
     /* Returns the CTypeDescrObject from the user-supplied 'arg'.
        Does not return a new reference!
     */
-    if (PyString_Check(arg)) {
+    if ((accept & ACCEPT_STRING) && PyString_Check(arg)) {
         PyObject *x = PyDict_GetItem(ffi->types_dict, arg);
         if (x != NULL && CTypeDescr_Check(x))
             return (CTypeDescrObject *)x;
@@ -130,24 +136,34 @@
         Py_DECREF(x);
         return ct;
     }
-    else if (CTypeDescr_Check(arg)) {
+    else if ((accept & ACCEPT_CTYPE) && CTypeDescr_Check(arg)) {
         return (CTypeDescrObject *)arg;
     }
+    else if ((accept & ACCEPT_CDATA) && CData_Check(arg)) {
+        return ((CDataObject *)arg)->c_type;
+    }
     else {
-        PyErr_SetString(PyExc_TypeError, "expected a string or a ctype object");
+        const char *m1 = (accept & ACCEPT_STRING) ? "string" : "";
+        const char *m2 = (accept & ACCEPT_CTYPE) ? "ctype object" : "";
+        const char *m3 = (accept & ACCEPT_CDATA) ? "cdata object" : "";
+        const char *s12 = (*m1 && (*m2 || *m3)) ? " or " : "";
+        const char *s23 = (*m2 && *m3) ? " or " : "";
+        PyErr_Format(PyExc_TypeError, "expected a %s%s%s%s%s, got '%.200s'",
+                     m1, s12, m2, s23, m3,
+                     Py_TYPE(arg)->tp_name);
         return NULL;
     }
 }
 
 static PyObject *ffi_sizeof(ZefFFIObject *self, PyObject *arg)
 {
-    CTypeDescrObject *ct = _ffi_type(self, arg);
+    CTypeDescrObject *ct = _ffi_type(self, arg, ACCEPT_ALL);
     if (ct == NULL)
         return NULL;
 
     if (ct->ct_size < 0) {
-        PyErr_Format(ZefError, "don't know the size of ctype '%s': "
-                     "incomplete type", ct->ct_name);
+        PyErr_Format(ZefError, "don't know the size of ctype '%s'",
+                     ct->ct_name);
         return NULL;
     }
     return PyInt_FromSsize_t(ct->ct_size);
@@ -155,20 +171,97 @@
 
 static PyObject *ffi_typeof(ZefFFIObject *self, PyObject *arg)
 {
-    if (!PyString_Check(arg)) {
-        PyErr_SetString(PyExc_TypeError, "expected a string");
+    PyObject *x = (PyObject *)_ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CDATA);
+    Py_XINCREF(x);
+    return x;
+}
+
+static PyObject *ffi_new(ZefFFIObject *self, PyObject *args)
+{
+    CTypeDescrObject *ct, *ctitem;
+    CDataObject *cd;
+    PyObject *arg, *init = Py_None;
+    Py_ssize_t dataoffset, datasize, explicitlength;
+    if (!PyArg_ParseTuple(args, "O|O:new", &arg, &init))
+        return NULL;
+
+    ct = _ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CTYPE);
+    if (ct == NULL)
+        return NULL;
+
+    explicitlength = -1;
+    if (ct->ct_flags & CT_POINTER) {
+        dataoffset = offsetof(CDataObject_own_nolength, alignment);
+        ctitem = ct->ct_itemdescr;
+        datasize = ctitem->ct_size;
+        if (datasize < 0) {
+            PyErr_Format(PyExc_TypeError,
+                         "cannot instantiate ctype '%s' of unknown size",
+                         ctitem->ct_name);
+            return NULL;
+        }
+        if (ctitem->ct_flags & CT_PRIMITIVE_CHAR)
+            datasize *= 2;   /* forcefully add another character: a null */
+
+        /* XXX
+        if ((ctitem->ct_flags & CT_WITH_VAR_ARRAY) && init != Py_None) {
+            Py_ssize_t optvarsize = datasize;
+            if (convert_struct_from_object(NULL,ctitem, init, &optvarsize) < 0)
+                return NULL;
+            datasize = optvarsize;
+        }*/
+    }
+    /* XXX
+    else if (ct->ct_flags & CT_ARRAY) {
+        dataoffset = offsetof(CDataObject_own_nolength, alignment);
+        datasize = ct->ct_size;
+        if (datasize < 0) {
+            explicitlength = get_new_array_length(&init);
+            if (explicitlength < 0)
+                return NULL;
+            ctitem = ct->ct_itemdescr;
+            dataoffset = offsetof(CDataObject_own_length, alignment);
+            datasize = explicitlength * ctitem->ct_size;
+            if (explicitlength > 0 &&
+                    (datasize / explicitlength) != ctitem->ct_size) {
+                PyErr_SetString(PyExc_OverflowError,
+                                "array size would overflow a Py_ssize_t");
+                return NULL;
+            }
+        }
+    }*/
+    else {
+        PyErr_Format(PyExc_TypeError,
+                     "expected a pointer or array ctype, got '%s'",
+                     ct->ct_name);
         return NULL;
     }
 
-    PyObject *x = (PyObject *)_ffi_type(self, arg);
-    Py_XINCREF(x);
-    return x;
+    cd = allocate_owning_object(dataoffset + datasize, ct);
+    if (cd == NULL)
+        return NULL;
+
+    cd->c_data = ((char *)cd) + dataoffset;
+    if (explicitlength >= 0)
+        ((CDataObject_own_length*)cd)->length = explicitlength;
+
+    memset(cd->c_data, 0, datasize);
+    if (init != Py_None) {
+        /* XXX
+        if (convert_from_object(cd->c_data,
+              (ct->ct_flags & CT_POINTER) ? ct->ct_itemdescr : ct, init) < 0) {
+            Py_DECREF(cd);
+            return NULL;
+        }*/
+    }
+    return (PyObject *)cd;
 }
 
 static PyMethodDef ffi_methods[] = {
     {"close_library", ffi_close_library,         METH_VARARGS | METH_STATIC},
     {"load_library",  (PyCFunction)ffi_load_library,
                                                  METH_VARARGS | METH_KEYWORDS},
+    {"new",           (PyCFunction)ffi_new,      METH_VARARGS},
     {"sizeof",        (PyCFunction)ffi_sizeof,   METH_O},
     {"typeof",        (PyCFunction)ffi_typeof,   METH_O},
     {NULL}
diff --git a/zeffir/test/test_cdata.py b/zeffir/test/test_cdata.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_cdata.py
@@ -0,0 +1,8 @@
+import py
+import support
+
+
+def test_simple_new():
+    ffi = support.new_ffi()
+    p = ffi.new("short *")
+    assert repr(p) == "<cdata 'short *' owning 2 bytes>"
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -42,6 +42,10 @@
         return;
     if (PyType_Ready(&CData_Type) < 0)
         return;
+    if (PyType_Ready(&CDataOwning_Type) < 0)
+        return;
+    if (PyType_Ready(&CDataOwningGC_Type) < 0)
+        return;
     if (PyType_Ready(&ZefFuncSupport_Type) < 0)
         return;
     if (PyType_Ready(&ZefGlobSupport_Type) < 0)
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -1,6 +1,5 @@
 
 typedef struct _crx_type_s CTypeDescrObject;
-typedef struct CDataObject_s CDataObject;
 typedef struct ZefLibObject_s ZefLibObject;
 typedef struct ZefFFIObject_s ZefFFIObject;
 
@@ -8,8 +7,8 @@
 static PyTypeObject CTypeDescr_Type;
 //static PyTypeObject CField_Type;
 static PyTypeObject CData_Type;
-//static PyTypeObject CDataOwning_Type;
-//static PyTypeObject CDataOwningGC_Type;
+static PyTypeObject CDataOwning_Type;
+static PyTypeObject CDataOwningGC_Type;
 
 static PyObject *ZefError;
 

From noreply at buildbot.pypy.org  Fri Dec  5 12:19:03 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  5 Dec 2014 12:19:03 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: refactor,
 properly allocate via signature dtypes. Still no casting
Message-ID: <20141205111903.0DE0A1D257D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r74825:d5aace1c3418
Date: 2014-12-05 01:54 +0200
http://bitbucket.org/pypy/pypy/changeset/d5aace1c3418/

Log:	refactor, properly allocate via signature dtypes. Still no casting

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -561,7 +561,8 @@
                  self.name, self.nin, len(args_w))
         for i in range(self.nin):
             inargs[i] = convert_to_array(space, args_w[i])
-            assert isinstance(inargs[i], W_NDimArray)
+        for i in inargs:
+            assert isinstance(i, W_NDimArray)
         outargs = [None] * self.nout
         for i in range(len(args_w)-self.nin):
             out = args_w[i+self.nin]
@@ -574,16 +575,23 @@
                 outargs[i] = out
         if sig is None:
             sig = space.wrap(self.signature)
-        index, dtypes = self.type_resolver(space, inargs, outargs, sig)
-        outargs = self.alloc_outargs(space, index, inargs, outargs, dtypes)
-        for i in range(len(outargs)):
-            assert isinstance(outargs[i], W_NDimArray)
-        outargs0 = outargs[0]
-        assert isinstance(outargs0, W_NDimArray)
-        res_dtype = outargs0.get_dtype()
+        _dtypes = self.dtypes
+        if self.match_dtypes:
+            _dtypes = [i.get_dtype() for i in inargs if isinstance(i, W_NDimArray)]
+            for i in outargs:
+                if isinstance(i, W_NDimArray):
+                    _dtypes.append(i.get_dtype())
+                else:
+                    _dtypes.append(_dtypes[0])
+        index, dtypes = self.type_resolver(space, inargs, outargs, sig, _dtypes)
         func = self.funcs[index]
         if not self.core_enabled:
             # func is going to do all the work, it must accept W_NDimArray args
+            inargs0 = inargs[0]
+            assert isinstance(inargs0, W_NDimArray)
+            outarg_shapes = [inargs0.get_shape()] * self.nargs
+            inargs, outargs = self.alloc_args(space, inargs, outargs,
+                                              dtypes, outarg_shapes)
             if self.stack_inputs:
                 arglist = space.newlist(list(inargs + outargs))
                 space.call_args(func, Arguments.frompacked(space, arglist))
@@ -594,121 +602,14 @@
             if len(outargs) < 2:
                 return outargs[0]
             return space.newtuple(outargs)
-        # Figure out the number of iteration dimensions, which
-        # is the broadcast result of all the input non-core
-        # dimensions
-        broadcast_ndim = 0
-        for i in range(len(inargs)):
-            curarg = inargs[i]
-            assert isinstance(curarg, W_NDimArray)
-            n = curarg.ndims() - self.core_num_dims[i]
-            broadcast_ndim = max(broadcast_ndim, n)
-        # Figure out the number of iterator creation dimensions,
-        # which is the broadcast dimensions + all the core dimensions of
-        # the outputs, so that the iterator can allocate those output
-        # dimensions following the rules of order='F', for example.
-        iter_ndim = broadcast_ndim
-        for i in range(self.nin, self.nargs):
-            iter_ndim += self.core_num_dims[i];
-        # Validate the core dimensions of all the operands,
-        inner_dimensions = [1] * (self.core_num_dim_ix + 2)
-        idim = 0
-        core_start_dim = 0
-        for i in range(self.nin):
-            curarg = inargs[i]
-            assert isinstance(curarg, W_NDimArray)
-            dim_offset = self.core_offsets[i]
-            num_dims = self.core_num_dims[i]
-            core_start_dim = curarg.ndims() - num_dims
-            if core_start_dim >=0:
-                idim = 0
-            else:
-                idim = -core_start_dim
-            while(idim < num_dims):
-                core_dim_index = self.core_dim_ixs[dim_offset + idim]
-                op_dim_size = curarg.get_shape()[core_start_dim + idim]
-                if inner_dimensions[core_dim_index + 1] == 1:
-                    inner_dimensions[core_dim_index + 1] = op_dim_size
-                elif op_dim_size != 1 and inner_dimensions[1 + core_dim_index] != op_dim_size:
-                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
-                        "mismatch in its core dimension %d, with gufunc "
-                        "signature %s (size %d is different from %d)",
-                         self.name, i, idim, self.signature, op_dim_size,
-                         inner_dimensions[1 + core_dim_index])
-                idim += 1
-        for i in range(len(outargs)):
-            curarg = outargs[i]
-            assert isinstance(curarg, W_NDimArray)
-            dim_offset = self.core_offsets[i]
-            num_dims = self.core_num_dims[i]
-            core_start_dim = curarg.ndims() - num_dims
-            if core_start_dim < 0:
-                raise oefmt(space.w_ValueError, "%s: Output operand %d does "
-                    "not have enough dimensions (has %d, gufunc with "
-                    "signature %s requires %d)", self.name, i,
-                    curarg.ndims(), self.signature, num_dims)
-            if core_start_dim >=0:
-                idim = 0
-            else:
-                idim = -core_start_dim
-            while(idim < num_dims):
-                core_dim_index = self.core_dim_ixs[dim_offset + idim]
-                op_dim_size = curarg.get_shape()[core_start_dim + idim]
-                if inner_dimensions[core_dim_index + 1] == 1:
-                    inner_dimensions[core_dim_index + 1] = op_dim_size
-                elif inner_dimensions[1 + core_dim_index] != op_dim_size:
-                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
-                        "mismatch in its core dimension %d, with gufunc "
-                        "signature %s (size %d is different from %d)",
-                         self.name, i, idim, self.signature, op_dim_size,
-                         inner_dimensions[1 + core_dim_index])
-                idim += 1
-        iter_shape = [-1] * (broadcast_ndim + (len(outargs) * iter_ndim))
-        j = broadcast_ndim
-        core_dim_ixs_size = 0
-        firstdim = broadcast_ndim
-        for cnd in self.core_num_dims:
-            firstdim  += cnd
-        op_axes_arrays = [[-1,] * (self.nin + len(outargs)),] * firstdim
-        for i in range(self.nin):
-            # Note that n may be negative if broadcasting
-            # extends into the core dimensions.
-            curarg = inargs[i]
-            assert isinstance(curarg, W_NDimArray)
-            n = curarg.ndims() - self.core_num_dims[i]
-            for idim in range(broadcast_ndim):
-                if idim >= broadcast_ndim -n:
-                    op_axes_arrays[idim][i] = idim - (broadcast_ndim -n)
-            core_dim_ixs_size += self.core_num_dims[i];
-        for i in range(len(outargs)):
-            curarg = outargs[i]
-            assert isinstance(curarg, W_NDimArray)
-            iout = i + self.nin
-            n = curarg.ndims() - self.core_num_dims[iout]
-            for idim in range(broadcast_ndim):
-                if idim >= broadcast_ndim -n:
-                    op_axes_arrays[idim][iout] = idim - (broadcast_ndim -n)
-            dim_offset = self.core_offsets[iout]
-            num_dims = self.core_num_dims[iout]
-            for idim in range(num_dims):
-                cdi = self.core_dim_ixs[dim_offset + idim]
-                iter_shape[j] = inner_dimensions[1 + cdi]
-                op_axes_arrays[j][iout] = n + idim
-                j += 1
-            core_dim_ixs_size += self.core_num_dims[iout];
-        # TODO once we support obejct dtypes,
-        # FAIL with NotImplemented if the other object has
-        # the __r<op>__ method and has a higher priority than
-        # the current op (signalling it can handle ndarray's).
-
-        # TODO parse and handle subok
-        # TODO handle flags, op_flags
+        iter_shape, outarg_shapes, matched_dims = self.verify_args(space, inargs, outargs)
+        inargs, outargs = self.alloc_args(space, inargs, outargs, dtypes,
+                                          outarg_shapes)
         w_flags = space.w_None # NOT 'external_loop', we do coalescing by core_num_dims
         w_op_flags = space.newtuple([space.wrap(r) for r in ['readonly'] * len(inargs)] + \
                                     [space.wrap(r) for r in ['readwrite'] * len(outargs)])
         w_op_dtypes = space.w_None
         w_casting = space.w_None
-        w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) 
         w_op_axes = space.w_None
 
         if self.stack_inputs:
@@ -722,16 +623,17 @@
                 # Unlike numpy, we will not broadcast dims before
                 # the core_ndims rather we use nditer iteration
                 # so dims[0] == 1
-                dims = [1] + inner_dimensions[1:]
+                dims = [1] + matched_dims
                 steps = []
                 allargs = inargs + outargs
-                assert core_start_dim >= 0
                 for i in range(len(allargs)):
                     steps.append(0)
                 for i in range(len(allargs)):
                     _arg = allargs[i]
                     assert isinstance(_arg, W_NDimArray)
-                    steps += _arg.implementation.strides[core_start_dim:]
+                    start_dim = len(_arg.get_shape()) - len(iter_shape)
+                    assert start_dim >= 0
+                    steps += _arg.implementation.strides[start_dim:]
                 #print 'set_dims_and_steps with dims, steps',dims,steps
                 func.set_dims_and_steps(space, dims, steps)
             else:
@@ -739,6 +641,7 @@
                 # from frompyfunc
                 pass
             # mimic NpyIter_AdvancedNew with a nditer
+            w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) 
             nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags,
                           w_op_flags, w_op_dtypes, w_casting, w_op_axes,
                           w_itershape)
@@ -770,7 +673,10 @@
                 return space.newtuple([convert_to_array(space, o) for o in outargs])
             return outargs[0]
         inargs0 = inargs[0]
+        outargs0 = outargs[0]
         assert isinstance(inargs0, W_NDimArray)
+        assert isinstance(outargs0, W_NDimArray)
+        res_dtype = outargs0.get_dtype()
         new_shape = inargs0.get_shape()
         if len(outargs) < 2:
             return loop.call_many_to_one(space, new_shape, func,
@@ -808,8 +714,8 @@
             kwargs_w.pop(kw)
         return w_subok, w_out, sig, casting, extobj
 
-    def type_resolver(self, space, inargs, outargs, type_tup):
-        # Find a match for the inargs.dtype in self.dtypes, like
+    def type_resolver(self, space, inargs, outargs, type_tup, _dtypes):
+        # Find a match for the inargs.dtype in _dtypes, like
         # linear_search_type_resolver in numpy ufunc_type_resolutions.c
         # type_tup can be '', a tuple of dtypes, or a string
         # of the form d,t -> D where the letters are dtype specs
@@ -831,7 +737,7 @@
                     " after the -> sign, not '%s'", self.name, self.nin, 
                     self.nout, type_tup)
         else:
-            # XXX does not pass translation
+            # XXX why does the next line not pass translation?
             # dtypes = [i.get_dtype() for i in inargs]
             for i in inargs:
                 if isinstance(i, W_NDimArray):
@@ -843,39 +749,164 @@
                     dtypes.append(i.get_dtype())
                 else:
                     dtypes.append(None)
-        #Find the first matchup of dtypes with self.dtypes
-        for i in range(0, len(self.dtypes), self.nargs):
+        #Find the first matchup of dtypes with _dtypes
+        for i in range(0, len(_dtypes), self.nargs):
             allok = True
             for j in range(self.nargs):
-                if dtypes[j] is not None and dtypes[j] != self.dtypes[i+j]:
+                if dtypes[j] is not None and dtypes[j] != _dtypes[i+j]:
                     allok = False
             if allok:    
                 break
         else:
-            if len(self.funcs) < 2:
-                return 0, dtypes
-            raise oefmt(space.w_TypeError,
-                         'input dtype did not match any known dtypes',
-                       )
+            if len(self.funcs) > 1:
+                raise oefmt(space.w_TypeError,
+                            'input dtype did not match any known dtypes',
+                           )
+            i = 0
         # Fill in empty dtypes
         for j in range(self.nargs):
             if dtypes[j] is None:
-                dtypes[j] = self.dtypes[i+j]
+                dtypes[j] = _dtypes[i+j]
         return i / self.nargs, dtypes
 
-    def alloc_outargs(self, space, index, inargs, outargs, dtypes):
-        # Any None outarg should be allocated here
+    def alloc_args(self, space, inargs, outargs, dtypes, outarg_shapes):
+        # Any None outarg are allocated, and inargs, outargs may need casting
         inargs0 = inargs[0]
         assert isinstance(inargs0, W_NDimArray)
-        temp_shape = inargs0.get_shape() # XXX wrong!!!
-        dtype = inargs0.get_dtype() # XXX wrong!!!
         order = inargs0.get_order()
         for i in range(len(outargs)):
             if outargs[i] is None:
-                outargs[i] = W_NDimArray.from_shape(space, temp_shape, dtype, order)
+                j = self.nin + i
+                outargs[i] = W_NDimArray.from_shape(space, outarg_shapes[i], dtypes[j], order)
+        for i in outargs:
+            assert isinstance(i, W_NDimArray)
+        return inargs, outargs
+
+    def verify_args(self, space, inargs, outargs):
+        # Figure out the number of iteration dimensions, which
+        # is the broadcast result of all the input non-core
+        # dimensions
+        iter_shape = []
+        max_matched_dims = 0
+        for i in self.core_dim_ixs:
+            if i > max_matched_dims:
+                max_matched_dims = i
+        matched_dims = [-1] * (1 + max_matched_dims)
+        for i in range(self.nin):
+            curarg = inargs[i]
+            assert isinstance(curarg, W_NDimArray)
+            dim_offset = self.core_offsets[i] # index into XXX_ixs
+            num_dims = self.core_num_dims[i]
+            # Make sure the last num_dims shape of curarg match the signature
+            n = len(curarg.get_shape()) - num_dims
+            if n < 0:
+                raise oefmt(space.w_ValueError, "%s: Input operand %d does "
+                    "not have enough dimensions (has %d, gufunc with "
+                    "signature %s requires %d)", self.name, i,
+                    num_dims+n, self.signature, num_dims)
+            dims_to_match = curarg.get_shape()[n:]
+            dims_to_broadcast = curarg.get_shape()[:n]
+            offset = len(dims_to_broadcast) - len(iter_shape) 
+            if offset >= 0:
+                # Prepend extra dimensions to iter_shape
+                iter_shape = dims_to_broadcast[:offset] + iter_shape
+                offset = 0
+            # Make sure iter_shape[offset:] matches dims_to_broadcast
+            offset = abs(offset) # for translation
+            for j in range(offset, len(iter_shape)):
+                x = iter_shape[j + offset]
+                y = dims_to_broadcast[j]
+                if (x > y and x % y) or y %x:
+                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
+                        "mismatch in its broadcast dimension %d "
+                        "(size %d is different from %d)",
+                         self.name, i, j, x, y)
+                iter_shape[offset + j] = max(x, y)
+            # Find or verify signature ixs
+            for j in range(num_dims):
+                core_dim_index = self.core_dim_ixs[dim_offset + j]
+                if core_dim_index > len(dims_to_match):
+                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
+                        "mismatch in its core dimension %d, with gufunc "
+                        "signature %s (index is larger than input shape)",
+                         self.name, i, j, self.signature, core_dim_index)
+                if matched_dims[core_dim_index] < 0:
+                    matched_dims[core_dim_index] = dims_to_match[j]
+                elif matched_dims[core_dim_index] != dims_to_match[j]:
+                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
+                        "mismatch in its core dimension %d, with gufunc "
+                        "signature %s (expected %d, got %d)",
+                         self.name, i, j, 
+                         self.signature, matched_dims[core_dim_index],
+                         dims_to_match[j])
+        outarg_shapes = []
         for i in range(len(outargs)):
-            assert isinstance(outargs[i], W_NDimArray)
-        return outargs
+            curarg = outargs[i]
+            dim_offset = self.core_offsets[i]
+            num_dims = self.core_num_dims[i]
+            if not isinstance(curarg, W_NDimArray):
+                arg_shape = iter_shape
+                for j in range(num_dims):
+                    core_dim_index = self.core_dim_ixs[dim_offset + j]
+                    if matched_dims[core_dim_index] < 0:
+                        raise oefmt(space.w_ValueError, "%s: Output operand %d "
+                            "is empty but unique core dimension %d in signature "
+                            "%s of gufunc was not specified",
+                             self.name, i, core_dim_index, self.signature)
+                    arg_shape.append(matched_dims[core_dim_index])
+                outarg_shapes.append(arg_shape) 
+                continue
+            n = len(curarg.get_shape()) - num_dims
+            if n < 0:
+                raise oefmt(space.w_ValueError, "%s: Output operand %d does "
+                    "not have enough dimensions (has %d, gufunc with "
+                    "signature %s requires %d)", self.name, i,
+                    num_dims+n, self.signature, num_dims)
+            dims_to_match = curarg.get_shape()[n:]
+            dims_to_broadcast = curarg.get_shape()[:n]
+            offset = len(dims_to_broadcast) - len(iter_shape)
+            if offset >= 0:
+                # Prepend extra dimensions to iter_shape, matched_dims
+                iter_shape = dims_to_broadcast[:offset] + iter_shape
+                outarg_shapes = [dims_to_broadcast[:offset] + asp for asp in outarg_shapes]
+                offset = 0
+            # Make sure iter_shape[offset:] matches dims_to_broadcast
+            offset = abs(offset) # for translation
+            for j in range(offset, len(iter_shape)):
+                x = iter_shape[j + offset]
+                y = dims_to_broadcast[j]
+                if (x > y and x % y) or y %x:
+                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
+                        "mismatch in its broadcast dimension %d "
+                        "(size %d is different from %d)",
+                         self.name, i, j, x, y)
+                iter_shape[offset + j] = max(x, y)
+            # Find or verify signature ixs
+            for j in range(num_dims):
+                core_dim_index = self.core_dim_ixs[dim_offset + j]
+                if core_dim_index > len(dims_to_match):
+                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
+                        "mismatch in its core dimension %d, with gufunc "
+                        "signature %s (index is larger than input shape)",
+                         self.name, i, j, self.signature, core_dim_index)
+                if matched_dims[core_dim_index] < 0:
+                    matched_dims[core_dim_index] = dims_to_match[core_dim_index]
+                elif matched_dims[core_dim_index] != dims_to_match[core_dim_index]:
+                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
+                        "mismatch in its core dimension %d, with gufunc "
+                        "signature %s (expected %d, got %d)",
+                         self.name, i, core_dim_index + num_dims, 
+                         self.signature, matched_dims[core_dim_index],
+                         dims_to_match[core_dim_index])
+            outarg_shapes.append(iter_shape + dims_to_match)
+        # TODO once we support obejct dtypes,
+        # FAIL with NotImplemented if the other object has
+        # the __r<op>__ method and has a higher priority than
+        # the current op (signalling it can handle ndarray's).
+
+        # TODO parse and handle subok
+        # TODO handle flags, op_flags
+        return iter_shape, outarg_shapes, matched_dims
 
 W_Ufunc.typedef = TypeDef("numpy.ufunc",
     __call__ = interp2app(W_Ufunc.descr_call),

From noreply at buildbot.pypy.org  Fri Dec  5 12:19:04 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  5 Dec 2014 12:19:04 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: simplify,
	test for not-implemented need to cast args
Message-ID: <20141205111904.468851D257D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r74826:384b1e427559
Date: 2014-12-05 08:28 +0200
http://bitbucket.org/pypy/pypy/changeset/384b1e427559/

Log:	simplify, test for not-implemented need to cast args

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -590,8 +590,10 @@
             inargs0 = inargs[0]
             assert isinstance(inargs0, W_NDimArray)
             outarg_shapes = [inargs0.get_shape()] * self.nargs
-            inargs, outargs = self.alloc_args(space, inargs, outargs,
+            inargs, outargs, need_to_cast = self.alloc_args(space, inargs, outargs,
                                               dtypes, outarg_shapes)
+            if any(need_to_cast):
+                raise oefmt(space.w_NotImplementedError, "casting not supported yet")
             if self.stack_inputs:
                 arglist = space.newlist(list(inargs + outargs))
                 space.call_args(func, Arguments.frompacked(space, arglist))
@@ -602,9 +604,11 @@
             if len(outargs) < 2:
                 return outargs[0]
             return space.newtuple(outargs)
-        iter_shape, outarg_shapes, matched_dims = self.verify_args(space, inargs, outargs)
-        inargs, outargs = self.alloc_args(space, inargs, outargs, dtypes,
-                                          outarg_shapes)
+        iter_shape, arg_shapes, matched_dims = self.verify_args(space, inargs, outargs)
+        inargs, outargs, need_to_cast = self.alloc_args(space, inargs, outargs, dtypes,
+                                          arg_shapes)
+        if any(need_to_cast):
+            raise oefmt(space.w_NotImplementedError, "casting not supported yet")
         w_flags = space.w_None # NOT 'external_loop', we do coalescing by core_num_dims
         w_op_flags = space.newtuple([space.wrap(r) for r in ['readonly'] * len(inargs)] + \
                                     [space.wrap(r) for r in ['readwrite'] * len(outargs)])
@@ -769,79 +773,49 @@
                 dtypes[j] = _dtypes[i+j]
         return i / self.nargs, dtypes
 
-    def alloc_args(self, space, inargs, outargs, dtypes, outarg_shapes):
+    def alloc_args(self, space, inargs, outargs, dtypes, arg_shapes):
         # Any None outarg are allocated, and inargs, outargs may need casting
         inargs0 = inargs[0]
         assert isinstance(inargs0, W_NDimArray)
         order = inargs0.get_order()
+        need_to_cast = []
+        for i in range(self.nin):
+            curarg = inargs[i]
+            assert isinstance(curarg, W_NDimArray)
+            if len(arg_shapes[i]) != curarg.ndims():
+                # XXX reshape (after merge with default)
+                pass
+            need_to_cast.append(curarg.get_dtype() == dtypes[i])
         for i in range(len(outargs)):
+            j = self.nin + i
             if outargs[i] is None:
-                j = self.nin + i
-                outargs[i] = W_NDimArray.from_shape(space, outarg_shapes[i], dtypes[j], order)
-        for i in outargs:
-            assert isinstance(i, W_NDimArray)
-        return inargs, outargs
+                outargs[i] = W_NDimArray.from_shape(space, outarg_shapes[j], dtypes[j], order)
+            elif len(arg_shapes[i]) != curarg.ndims():
+                # XXX reshape (after merge with default)
+                pass
+            need_to_cast.append(curarg.get_dtype() == dtypes[j])
+        return inargs, outargs, need_to_cast
 
     def verify_args(self, space, inargs, outargs):
         # Figure out the number of iteration dimensions, which
         # is the broadcast result of all the input non-core
         # dimensions
         iter_shape = []
+        arg_shapes = []
         max_matched_dims = 0
         for i in self.core_dim_ixs:
             if i > max_matched_dims:
                 max_matched_dims = i
         matched_dims = [-1] * (1 + max_matched_dims)
-        for i in range(self.nin):
-            curarg = inargs[i]
-            assert isinstance(curarg, W_NDimArray)
-            dim_offset = self.core_offsets[i] # index into XXX_ixs
-            num_dims = self.core_num_dims[i]
-            # Make sure the last num_dims shape of curarg match the signature
-            n = len(curarg.get_shape()) - num_dims
-            if n < 0:
-                raise oefmt(space.w_ValueError, "%s: Input operand %d does "
-                    "not have enough dimensions (has %d, gufunc with "
-                    "signature %s requires %d)", self.name, i,
-                    num_dims+n, self.signature, num_dims)
-            dims_to_match = curarg.get_shape()[n:]
-            dims_to_broadcast = curarg.get_shape()[:n]
-            offset = len(dims_to_broadcast) - len(iter_shape) 
-            if offset >= 0:
-                # Prepend extra dimensions to iter_shape
-                iter_shape = dims_to_broadcast[:offset] + iter_shape
-                offset = 0
-            # Make sure iter_shape[offset:] matches dims_to_broadcast
-            offset = abs(offset) # for translation
-            for j in range(offset, len(iter_shape)):
-                x = iter_shape[j + offset]
-                y = dims_to_broadcast[j]
-                if (x > y and x % y) or y %x:
-                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
-                        "mismatch in its broadcast dimension %d "
-                        "(size %d is different from %d)",
-                         self.name, i, j, x, y)
-                iter_shape[offset + j] = max(x, y)
-            # Find or verify signature ixs
-            for j in range(num_dims):
-                core_dim_index = self.core_dim_ixs[dim_offset + j]
-                if core_dim_index > len(dims_to_match):
-                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
-                        "mismatch in its core dimension %d, with gufunc "
-                        "signature %s (index is larger than input shape)",
-                         self.name, i, j, self.signature, core_dim_index)
-                if matched_dims[core_dim_index] < 0:
-                    matched_dims[core_dim_index] = dims_to_match[j]
-                elif matched_dims[core_dim_index] != dims_to_match[j]:
-                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
-                        "mismatch in its core dimension %d, with gufunc "
-                        "signature %s (expected %d, got %d)",
-                         self.name, i, j, 
-                         self.signature, matched_dims[core_dim_index],
-                         dims_to_match[j])
-        outarg_shapes = []
-        for i in range(len(outargs)):
-            curarg = outargs[i]
+        for i in range(len(inargs) + len(outargs)):
+            if i < len(inargs):
+                _i = i
+                name = 'Input'
+                curarg = inargs[i]
+            else:
+                _i = i + self.nin
+                name = 'Output'
+                curarg = outargs[_i]
             dim_offset = self.core_offsets[i]
             num_dims = self.core_num_dims[i]
             if not isinstance(curarg, W_NDimArray):
@@ -849,18 +823,18 @@
                 for j in range(num_dims):
                     core_dim_index = self.core_dim_ixs[dim_offset + j]
                     if matched_dims[core_dim_index] < 0:
-                        raise oefmt(space.w_ValueError, "%s: Output operand %d "
+                        raise oefmt(space.w_ValueError, "%s: %s operand %d "
                             "is empty but unique core dimension %d in signature "
                             "%s of gufunc was not specified",
-                             self.name, i, core_dim_index, self.signature)
+                             self.name, name, _i, core_dim_index, self.signature)
                     arg_shape.append(matched_dims[core_dim_index])
-                outarg_shapes.append(arg_shape) 
+                arg_shapes.append(arg_shape) 
                 continue
             n = len(curarg.get_shape()) - num_dims
             if n < 0:
-                raise oefmt(space.w_ValueError, "%s: Output operand %d does "
+                raise oefmt(space.w_ValueError, "%s: %s operand %d does "
                     "not have enough dimensions (has %d, gufunc with "
-                    "signature %s requires %d)", self.name, i,
+                    "signature %s requires %d)", self.name, name, _i,
                     num_dims+n, self.signature, num_dims)
             dims_to_match = curarg.get_shape()[n:]
             dims_to_broadcast = curarg.get_shape()[:n]
@@ -876,37 +850,37 @@
                 x = iter_shape[j + offset]
                 y = dims_to_broadcast[j]
                 if (x > y and x % y) or y %x:
-                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
+                    raise oefmt(space.w_ValueError, "%s: %s operand %d has a "
                         "mismatch in its broadcast dimension %d "
                         "(size %d is different from %d)",
-                         self.name, i, j, x, y)
+                         self.name, name, _i, j, x, y)
                 iter_shape[offset + j] = max(x, y)
             # Find or verify signature ixs
             for j in range(num_dims):
                 core_dim_index = self.core_dim_ixs[dim_offset + j]
                 if core_dim_index > len(dims_to_match):
-                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
+                    raise oefmt(space.w_ValueError, "%s: %s operand %d has a "
                         "mismatch in its core dimension %d, with gufunc "
                         "signature %s (index is larger than input shape)",
-                         self.name, i, j, self.signature, core_dim_index)
+                         self.name, name, _i, j, self.signature, core_dim_index)
                 if matched_dims[core_dim_index] < 0:
                     matched_dims[core_dim_index] = dims_to_match[core_dim_index]
                 elif matched_dims[core_dim_index] != dims_to_match[core_dim_index]:
-                    raise oefmt(space.w_ValueError, "%s: Operand %d has a "
+                    raise oefmt(space.w_ValueError, "%s: %s operand %d has a "
                         "mismatch in its core dimension %d, with gufunc "
                         "signature %s (expected %d, got %d)",
-                         self.name, i, core_dim_index + num_dims, 
+                         self.name, name, _i, core_dim_index + num_dims, 
                          self.signature, matched_dims[core_dim_index],
                          dims_to_match[core_dim_index])
-            outarg_shapes.append(iter_shape + dims_to_match)
+            arg_shapes.append(iter_shape + dims_to_match)
         # TODO once we support obejct dtypes,
-        # FAIL with NotImplemented if the other object has
+        # FAIL with NotImplementedError if the other object has
         # the __r<op>__ method and has a higher priority than
         # the current op (signalling it can handle ndarray's).
 
         # TODO parse and handle subok
         # TODO handle flags, op_flags
-        return iter_shape, outarg_shapes, matched_dims
+        return iter_shape, arg_shapes, matched_dims
 
 W_Ufunc.typedef = TypeDef("numpy.ufunc",
     __call__ = interp2app(W_Ufunc.descr_call),

From noreply at buildbot.pypy.org  Fri Dec  5 12:19:08 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  5 Dec 2014 12:19:08 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: merge default into branch
Message-ID: <20141205111908.8E9321D257D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r74827:8f7cc875fd26
Date: 2014-12-05 08:50 +0200
http://bitbucket.org/pypy/pypy/changeset/8f7cc875fd26/

Log:	merge default into branch

diff too long, truncating to 2000 out of 8352 lines

diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -205,23 +205,28 @@
 The above is true both in CPython and in PyPy.  Differences
 can occur about whether a built-in function or method will
 call an overridden method of *another* object than ``self``.
-In PyPy, they are generally always called, whereas not in
-CPython.  For example, in PyPy, ``dict1.update(dict2)``
-considers that ``dict2`` is just a general mapping object, and
-will thus call overridden ``keys()``  and ``__getitem__()``
-methods on it.  So the following code prints ``42`` on PyPy
-but ``foo`` on CPython::
+In PyPy, they are often called in cases where CPython would not.
+Two examples::
 
-    >>>> class D(dict):
-    ....     def __getitem__(self, key):
-    ....         return 42
-    ....
-    >>>>
-    >>>> d1 = {}
-    >>>> d2 = D(a='foo')
-    >>>> d1.update(d2)
-    >>>> print d1['a']
-    42
+    class D(dict):
+        def __getitem__(self, key):
+            return "%r from D" % (key,)
+
+    class A(object):
+        pass
+
+    a = A()
+    a.__dict__ = D()
+    a.foo = "a's own foo"
+    print a.foo
+    # CPython => a's own foo
+    # PyPy => 'foo' from D
+
+    glob = D(foo="base item")
+    loc = {}
+    exec "print foo" in glob, loc
+    # CPython => base item
+    # PyPy => 'foo' from D
 
 
 Mutating classes of objects which are already used as dictionary keys
@@ -292,6 +297,9 @@
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
+Notably missing from the list above are ``str`` and ``unicode``.  If your
+code relies on comparing strings with ``is``, then it might break in PyPy.
+
 
 Miscellaneous
 -------------
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,15 +1,13 @@
-"""
-Python control flow graph generation and bytecode assembly.
-"""
+"""Python control flow graph generation and bytecode assembly."""
 
-from pypy.interpreter.astcompiler import ast, symtable
-from pypy.interpreter import pycode
+from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import we_are_translated
+
+from pypy.interpreter.astcompiler import ast, misc, symtable
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pycode import PyCode
 from pypy.tool import stdlib_opcode as ops
 
-from pypy.interpreter.error import OperationError
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import rfloat
-
 
 class Instruction(object):
     """Represents a single opcode."""
@@ -21,14 +19,12 @@
         self.has_jump = False
 
     def size(self):
-        """Return the size of bytes of this instruction when it is encoded."""
+        """Return the size of bytes of this instruction when it is
+        encoded.
+        """
         if self.opcode >= ops.HAVE_ARGUMENT:
-            if self.arg > 0xFFFF:
-                return 6
-            else:
-                return 3
-        else:
-            return 1
+            return (6 if self.arg > 0xFFFF else 3)
+        return 1
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -54,9 +50,9 @@
 class Block(object):
     """A basic control flow block.
 
-    It has one entry point and several possible exit points.  Its instructions
-    may be jumps to other blocks, or if control flow reaches the end of the
-    block, it continues to next_block.
+    It has one entry point and several possible exit points.  Its
+    instructions may be jumps to other blocks, or if control flow
+    reaches the end of the block, it continues to next_block.
     """
 
     def __init__(self):
@@ -71,10 +67,10 @@
             stack.append(nextblock)
 
     def post_order(self):
-        """Return this block and its children in post order.
-        This means that the graph of blocks is first cleaned up to
-        ignore back-edges, thus turning it into a DAG.  Then the DAG
-        is linearized.  For example:
+        """Return this block and its children in post order.  This means
+        that the graph of blocks is first cleaned up to ignore
+        back-edges, thus turning it into a DAG.  Then the DAG is
+        linearized.  For example:
 
                    A --> B -\           =>     [A, D, B, C]
                      \-> D ---> C
@@ -105,7 +101,9 @@
         return resultblocks
 
     def code_size(self):
-        """Return the encoded size of all the instructions in this block."""
+        """Return the encoded size of all the instructions in this
+        block.
+        """
         i = 0
         for instr in self.instructions:
             i += instr.size()
@@ -141,6 +139,7 @@
             i += 1
     return result
 
+
 def _list_to_dict(l, offset=0):
     result = {}
     index = offset
@@ -300,11 +299,11 @@
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
         last_extended_arg_count = 0
-        # The reason for this loop is extended jumps.  EXTENDED_ARG extends the
-        # bytecode size, so it might invalidate the offsets we've already given.
-        # Thus we have to loop until the number of extended args is stable.  Any
-        # extended jump at all is extremely rare, so performance is not too
-        # concerning.
+        # The reason for this loop is extended jumps.  EXTENDED_ARG
+        # extends the bytecode size, so it might invalidate the offsets
+        # we've already given.  Thus we have to loop until the number of
+        # extended args is stable.  Any extended jump at all is
+        # extremely rare, so performance is not too concerning.
         while True:
             extended_arg_count = 0
             offset = 0
@@ -330,7 +329,8 @@
                                     instr.opcode = ops.JUMP_ABSOLUTE
                                     absolute = True
                                 elif target_op == ops.RETURN_VALUE:
-                                    # Replace JUMP_* to a RETURN into just a RETURN
+                                    # Replace JUMP_* to a RETURN into
+                                    # just a RETURN
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
@@ -345,7 +345,8 @@
                         instr.arg = jump_arg
                         if jump_arg > 0xFFFF:
                             extended_arg_count += 1
-            if extended_arg_count == last_extended_arg_count and not force_redo:
+            if (extended_arg_count == last_extended_arg_count and
+                not force_redo):
                 break
             else:
                 last_extended_arg_count = extended_arg_count
@@ -360,12 +361,14 @@
         while True:
             try:
                 w_key = space.next(w_iter)
-            except OperationError, e:
+            except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise
                 break
             w_index = space.getitem(w_consts, w_key)
-            consts_w[space.int_w(w_index)] = space.getitem(w_key, first)
+            w_constant = space.getitem(w_key, first)
+            w_constant = misc.intern_if_common_string(space, w_constant)
+            consts_w[space.int_w(w_index)] = w_constant
         return consts_w
 
     def _get_code_flags(self):
@@ -433,15 +436,16 @@
                         continue
                     addr = offset - current_off
                     # Python assumes that lineno always increases with
-                    # increasing bytecode address (lnotab is unsigned char).
-                    # Depending on when SET_LINENO instructions are emitted this
-                    # is not always true.  Consider the code:
+                    # increasing bytecode address (lnotab is unsigned
+                    # char).  Depending on when SET_LINENO instructions
+                    # are emitted this is not always true.  Consider the
+                    # code:
                     #     a = (1,
                     #          b)
-                    # In the bytecode stream, the assignment to "a" occurs after
-                    # the loading of "b".  This works with the C Python compiler
-                    # because it only generates a SET_LINENO instruction for the
-                    # assignment.
+                    # In the bytecode stream, the assignment to "a"
+                    # occurs after the loading of "b".  This works with
+                    # the C Python compiler because it only generates a
+                    # SET_LINENO instruction for the assignment.
                     if line or addr:
                         while addr > 255:
                             push(chr(255))
@@ -484,22 +488,22 @@
         free_names = _list_from_dict(self.free_vars, len(cell_names))
         flags = self._get_code_flags() | self.compile_info.flags
         bytecode = ''.join([block.get_code() for block in blocks])
-        return pycode.PyCode(self.space,
-                             self.argcount,
-                             len(self.var_names),
-                             stack_depth,
-                             flags,
-                             bytecode,
-                             list(consts_w),
-                             names,
-                             var_names,
-                             self.compile_info.filename,
-                             self.name,
-                             self.first_lineno,
-                             lnotab,
-                             free_names,
-                             cell_names,
-                             self.compile_info.hidden_applevel)
+        return PyCode(self.space,
+                      self.argcount,
+                      len(self.var_names),
+                      stack_depth,
+                      flags,
+                      bytecode,
+                      list(consts_w),
+                      names,
+                      var_names,
+                      self.compile_info.filename,
+                      self.name,
+                      self.first_lineno,
+                      lnotab,
+                      free_names,
+                      cell_names,
+                      self.compile_info.hidden_applevel)
 
 
 def _list_from_dict(d, offset=0):
@@ -510,134 +514,134 @@
 
 
 _static_opcode_stack_effects = {
-    ops.NOP : 0,
-    ops.STOP_CODE : 0,
+    ops.NOP: 0,
+    ops.STOP_CODE: 0,
 
-    ops.POP_TOP : -1,
-    ops.ROT_TWO : 0,
-    ops.ROT_THREE : 0,
-    ops.ROT_FOUR : 0,
-    ops.DUP_TOP : 1,
+    ops.POP_TOP: -1,
+    ops.ROT_TWO: 0,
+    ops.ROT_THREE: 0,
+    ops.ROT_FOUR: 0,
+    ops.DUP_TOP: 1,
 
-    ops.UNARY_POSITIVE : 0,
-    ops.UNARY_NEGATIVE : 0,
-    ops.UNARY_NOT : 0,
-    ops.UNARY_CONVERT : 0,
-    ops.UNARY_INVERT : 0,
+    ops.UNARY_POSITIVE: 0,
+    ops.UNARY_NEGATIVE: 0,
+    ops.UNARY_NOT: 0,
+    ops.UNARY_CONVERT: 0,
+    ops.UNARY_INVERT: 0,
 
-    ops.LIST_APPEND : -1,
-    ops.SET_ADD : -1,
-    ops.MAP_ADD : -2,
-    ops.STORE_MAP : -2,
+    ops.LIST_APPEND: -1,
+    ops.SET_ADD: -1,
+    ops.MAP_ADD: -2,
+    ops.STORE_MAP: -2,
 
-    ops.BINARY_POWER : -1,
-    ops.BINARY_MULTIPLY : -1,
-    ops.BINARY_DIVIDE : -1,
-    ops.BINARY_MODULO : -1,
-    ops.BINARY_ADD : -1,
-    ops.BINARY_SUBTRACT : -1,
-    ops.BINARY_SUBSCR : -1,
-    ops.BINARY_FLOOR_DIVIDE : -1,
-    ops.BINARY_TRUE_DIVIDE : -1,
-    ops.BINARY_LSHIFT : -1,
-    ops.BINARY_RSHIFT : -1,
-    ops.BINARY_AND : -1,
-    ops.BINARY_OR : -1,
-    ops.BINARY_XOR : -1,
+    ops.BINARY_POWER: -1,
+    ops.BINARY_MULTIPLY: -1,
+    ops.BINARY_DIVIDE: -1,
+    ops.BINARY_MODULO: -1,
+    ops.BINARY_ADD: -1,
+    ops.BINARY_SUBTRACT: -1,
+    ops.BINARY_SUBSCR: -1,
+    ops.BINARY_FLOOR_DIVIDE: -1,
+    ops.BINARY_TRUE_DIVIDE: -1,
+    ops.BINARY_LSHIFT: -1,
+    ops.BINARY_RSHIFT: -1,
+    ops.BINARY_AND: -1,
+    ops.BINARY_OR: -1,
+    ops.BINARY_XOR: -1,
 
-    ops.INPLACE_FLOOR_DIVIDE : -1,
-    ops.INPLACE_TRUE_DIVIDE : -1,
-    ops.INPLACE_ADD : -1,
-    ops.INPLACE_SUBTRACT : -1,
-    ops.INPLACE_MULTIPLY : -1,
-    ops.INPLACE_DIVIDE : -1,
-    ops.INPLACE_MODULO : -1,
-    ops.INPLACE_POWER : -1,
-    ops.INPLACE_LSHIFT : -1,
-    ops.INPLACE_RSHIFT : -1,
-    ops.INPLACE_AND : -1,
-    ops.INPLACE_OR : -1,
-    ops.INPLACE_XOR : -1,
+    ops.INPLACE_FLOOR_DIVIDE: -1,
+    ops.INPLACE_TRUE_DIVIDE: -1,
+    ops.INPLACE_ADD: -1,
+    ops.INPLACE_SUBTRACT: -1,
+    ops.INPLACE_MULTIPLY: -1,
+    ops.INPLACE_DIVIDE: -1,
+    ops.INPLACE_MODULO: -1,
+    ops.INPLACE_POWER: -1,
+    ops.INPLACE_LSHIFT: -1,
+    ops.INPLACE_RSHIFT: -1,
+    ops.INPLACE_AND: -1,
+    ops.INPLACE_OR: -1,
+    ops.INPLACE_XOR: -1,
 
-    ops.SLICE+0 : 1,
-    ops.SLICE+1 : 0,
-    ops.SLICE+2 : 0,
-    ops.SLICE+3 : -1,
-    ops.STORE_SLICE+0 : -2,
-    ops.STORE_SLICE+1 : -3,
-    ops.STORE_SLICE+2 : -3,
-    ops.STORE_SLICE+3 : -4,
-    ops.DELETE_SLICE+0 : -1,
-    ops.DELETE_SLICE+1 : -2,
-    ops.DELETE_SLICE+2 : -2,
-    ops.DELETE_SLICE+3 : -3,
+    ops.SLICE+0: 1,
+    ops.SLICE+1: 0,
+    ops.SLICE+2: 0,
+    ops.SLICE+3: -1,
+    ops.STORE_SLICE+0: -2,
+    ops.STORE_SLICE+1: -3,
+    ops.STORE_SLICE+2: -3,
+    ops.STORE_SLICE+3: -4,
+    ops.DELETE_SLICE+0: -1,
+    ops.DELETE_SLICE+1: -2,
+    ops.DELETE_SLICE+2: -2,
+    ops.DELETE_SLICE+3: -3,
 
-    ops.STORE_SUBSCR : -2,
-    ops.DELETE_SUBSCR : -2,
+    ops.STORE_SUBSCR: -2,
+    ops.DELETE_SUBSCR: -2,
 
-    ops.GET_ITER : 0,
-    ops.FOR_ITER : 1,
-    ops.BREAK_LOOP : 0,
-    ops.CONTINUE_LOOP : 0,
-    ops.SETUP_LOOP : 0,
+    ops.GET_ITER: 0,
+    ops.FOR_ITER: 1,
+    ops.BREAK_LOOP: 0,
+    ops.CONTINUE_LOOP: 0,
+    ops.SETUP_LOOP: 0,
 
-    ops.PRINT_EXPR : -1,
-    ops.PRINT_ITEM : -1,
-    ops.PRINT_NEWLINE : 0,
-    ops.PRINT_ITEM_TO : -2,
-    ops.PRINT_NEWLINE_TO : -1,
+    ops.PRINT_EXPR: -1,
+    ops.PRINT_ITEM: -1,
+    ops.PRINT_NEWLINE: 0,
+    ops.PRINT_ITEM_TO: -2,
+    ops.PRINT_NEWLINE_TO: -1,
 
-    ops.WITH_CLEANUP : -1,
-    ops.POP_BLOCK : 0,
-    ops.END_FINALLY : -1,
-    ops.SETUP_WITH : 1,
-    ops.SETUP_FINALLY : 0,
-    ops.SETUP_EXCEPT : 0,
+    ops.WITH_CLEANUP: -1,
+    ops.POP_BLOCK: 0,
+    ops.END_FINALLY: -1,
+    ops.SETUP_WITH: 1,
+    ops.SETUP_FINALLY: 0,
+    ops.SETUP_EXCEPT: 0,
 
-    ops.LOAD_LOCALS : 1,
-    ops.RETURN_VALUE : -1,
-    ops.EXEC_STMT : -3,
-    ops.YIELD_VALUE : 0,
-    ops.BUILD_CLASS : -2,
-    ops.BUILD_MAP : 1,
-    ops.BUILD_SET : 1,
-    ops.COMPARE_OP : -1,
+    ops.LOAD_LOCALS: 1,
+    ops.RETURN_VALUE: -1,
+    ops.EXEC_STMT: -3,
+    ops.YIELD_VALUE: 0,
+    ops.BUILD_CLASS: -2,
+    ops.BUILD_MAP: 1,
+    ops.BUILD_SET: 1,
+    ops.COMPARE_OP: -1,
 
-    ops.LOOKUP_METHOD : 1,
+    ops.LOOKUP_METHOD: 1,
 
-    ops.LOAD_NAME : 1,
-    ops.STORE_NAME : -1,
-    ops.DELETE_NAME : 0,
+    ops.LOAD_NAME: 1,
+    ops.STORE_NAME: -1,
+    ops.DELETE_NAME: 0,
 
-    ops.LOAD_FAST : 1,
-    ops.STORE_FAST : -1,
-    ops.DELETE_FAST : 0,
+    ops.LOAD_FAST: 1,
+    ops.STORE_FAST: -1,
+    ops.DELETE_FAST: 0,
 
-    ops.LOAD_ATTR : 0,
-    ops.STORE_ATTR : -2,
-    ops.DELETE_ATTR : -1,
+    ops.LOAD_ATTR: 0,
+    ops.STORE_ATTR: -2,
+    ops.DELETE_ATTR: -1,
 
-    ops.LOAD_GLOBAL : 1,
-    ops.STORE_GLOBAL : -1,
-    ops.DELETE_GLOBAL : 0,
+    ops.LOAD_GLOBAL: 1,
+    ops.STORE_GLOBAL: -1,
+    ops.DELETE_GLOBAL: 0,
 
-    ops.LOAD_CLOSURE : 1,
-    ops.LOAD_DEREF : 1,
-    ops.STORE_DEREF : -1,
+    ops.LOAD_CLOSURE: 1,
+    ops.LOAD_DEREF: 1,
+    ops.STORE_DEREF: -1,
 
-    ops.LOAD_CONST : 1,
+    ops.LOAD_CONST: 1,
 
-    ops.IMPORT_STAR : -1,
-    ops.IMPORT_NAME : -1,
-    ops.IMPORT_FROM : 1,
+    ops.IMPORT_STAR: -1,
+    ops.IMPORT_NAME: -1,
+    ops.IMPORT_FROM: 1,
 
-    ops.JUMP_FORWARD : 0,
-    ops.JUMP_ABSOLUTE : 0,
-    ops.JUMP_IF_TRUE_OR_POP : 0,
-    ops.JUMP_IF_FALSE_OR_POP : 0,
-    ops.POP_JUMP_IF_TRUE : -1,
-    ops.POP_JUMP_IF_FALSE : -1,
-    ops.JUMP_IF_NOT_DEBUG : 0,
+    ops.JUMP_FORWARD: 0,
+    ops.JUMP_ABSOLUTE: 0,
+    ops.JUMP_IF_TRUE_OR_POP: 0,
+    ops.JUMP_IF_FALSE_OR_POP: 0,
+    ops.POP_JUMP_IF_TRUE: -1,
+    ops.POP_JUMP_IF_FALSE: -1,
+    ops.JUMP_IF_NOT_DEBUG: 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -106,3 +106,13 @@
     except IndexError:
         return name
     return "_%s%s" % (klass[i:], name)
+
+
+def intern_if_common_string(space, w_const):
+    # only intern identifier-like strings
+    if not space.is_w(space.type(w_const), space.w_str):
+        return w_const
+    for c in space.str_w(w_const):
+        if not (c.isalnum() or c == '_'):
+            return w_const
+    return space.new_interned_w_str(w_const)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -272,6 +272,11 @@
                 if w_const is None:
                     return tup
                 consts_w[i] = w_const
+            # intern the string constants packed into the tuple here,
+            # because assemble.py will see the result as just a tuple constant
+            for i in range(len(consts_w)):
+                consts_w[i] = misc.intern_if_common_string(
+                    self.space, consts_w[i])
         else:
             consts_w = []
         w_consts = self.space.newtuple(consts_w)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -14,7 +14,7 @@
     UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
-from pypy.interpreter.miscutils import ThreadLocals
+from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
 
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
@@ -384,7 +384,7 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        self.interned_strings = {}
+        self.interned_strings = make_weak_value_dictionary(self, str, W_Root)
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
@@ -777,25 +777,30 @@
             return self.w_False
 
     def new_interned_w_str(self, w_s):
+        assert isinstance(w_s, W_Root)   # and is not None
         s = self.str_w(w_s)
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        self.interned_strings[s] = w_s
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = w_s
+            self.interned_strings.set(s, w_s1)
+        return w_s1
 
     def new_interned_str(self, s):
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        w_s = self.interned_strings[s] = self.wrap(s)
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = self.wrap(s)
+            self.interned_strings.set(s, w_s1)
+        return w_s1
+
+    def is_interned_str(self, s):
+        # interface for marshal_impl
+        if not we_are_translated():
+            assert type(s) is str
+        return self.interned_strings.get(s) is not None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -31,3 +31,19 @@
 
     def getallvalues(self):
         return {0: self._value}
+
+
+def make_weak_value_dictionary(space, keytype, valuetype):
+    "NOT_RPYTHON"
+    if space.config.translation.rweakref:
+        from rpython.rlib.rweakref import RWeakValueDictionary
+        return RWeakValueDictionary(keytype, valuetype)
+    else:
+        class FakeWeakValueDict(object):
+            def __init__(self):
+                self._dict = {}
+            def get(self, key):
+                return self._dict.get(key, None)
+            def set(self, key, value):
+                self._dict[key] = value
+        return FakeWeakValueDict()
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -131,7 +131,6 @@
         # class bodies only have CO_NEWLOCALS.
         # CO_NEWLOCALS: make a locals dict unless optimized is also set
         # CO_OPTIMIZED: no locals dict needed at all
-        # NB: this method is overridden in nestedscope.py
         flags = code.co_flags
         if not (flags & pycode.CO_OPTIMIZED):
             if flags & pycode.CO_NEWLOCALS:
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -970,7 +970,12 @@
             sys.stdout = out
         output = s.getvalue()
         assert "CALL_METHOD" in output
-            
+
+    def test_interned_strings(self):
+        source = """x = ('foo_bar42', 5); y = 'foo_bar42'; z = x[0]"""
+        exec source
+        assert y is z
+
 
 class AppTestExceptions:
     def test_indentation_error(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -378,3 +378,41 @@
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+
+    def test_interned_strings_are_weak(self):
+        import weakref, gc, random
+        space = self.space
+        assert space.config.translation.rweakref
+        w1 = space.new_interned_str("abcdef")
+        w2 = space.new_interned_str("abcdef")
+        assert w2 is w1
+        #
+        # check that 'w1' goes away if we don't hold a reference to it
+        rw1 = weakref.ref(w1)
+        del w1, w2
+        i = 10
+        while rw1() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
+        #
+        s = "foobar%r" % random.random()
+        w0 = space.wrap(s)
+        w1 = space.new_interned_w_str(w0)
+        assert w1 is w0
+        w2 = space.new_interned_w_str(w0)
+        assert w2 is w0
+        w3 = space.wrap(s)
+        assert w3 is not w0
+        w4 = space.new_interned_w_str(w3)
+        assert w4 is w0
+        #
+        # check that 'w0' goes away if we don't hold a reference to it
+        # (even if we hold a reference to 'w3')
+        rw0 = weakref.ref(w0)
+        del w0, w1, w2, w4
+        i = 10
+        while rw0() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -158,21 +158,14 @@
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
-    _attrs_            = ['value_fits_long', 'vmin', 'vrangemax']
-    _immutable_fields_ = ['value_fits_long', 'vmin', 'vrangemax']
+    _attrs_            = ['value_fits_long', 'value_smaller_than_long']
+    _immutable_fields_ = ['value_fits_long', 'value_smaller_than_long']
     is_primitive_integer = True
 
     def __init__(self, *args):
         W_CTypePrimitive.__init__(self, *args)
         self.value_fits_long = self.size <= rffi.sizeof(lltype.Signed)
-        if self.size < rffi.sizeof(lltype.Signed):
-            assert self.value_fits_long
-            sh = self.size * 8
-            self.vmin = r_uint(-1) << (sh - 1)
-            self.vrangemax = (r_uint(1) << sh) - 1
-        else:
-            self.vmin = r_uint(0)
-            self.vrangemax = r_uint(-1)
+        self.value_smaller_than_long = self.size < rffi.sizeof(lltype.Signed)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -192,8 +185,17 @@
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
             value = misc.as_long(self.space, w_ob)
-            if self.size < rffi.sizeof(lltype.Signed):
-                if r_uint(value) - self.vmin > self.vrangemax:
+            if self.value_smaller_than_long:
+                size = self.size
+                if size == 1:
+                    signextended = misc.signext(value, 1)
+                elif size == 2:
+                    signextended = misc.signext(value, 2)
+                elif size == 4:
+                    signextended = misc.signext(value, 4)
+                else:
+                    raise AssertionError("unsupported size")
+                if value != signextended:
                     self._overflow(w_ob)
             misc.write_raw_signed_data(cdata, value, self.size)
         else:
@@ -221,7 +223,7 @@
             length = w_cdata.get_array_length()
             populate_list_from_raw_array(res, buf, length)
             return res
-        elif self.value_fits_long:
+        elif self.value_smaller_than_long:
             res = [0] * w_cdata.get_array_length()
             misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
             return res
@@ -235,8 +237,8 @@
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
-                overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                overflowed = misc.pack_list_to_raw_array_bounds_signed(
+                    int_list, cdata, self.size)
                 if overflowed != 0:
                     self._overflow(self.space.wrap(overflowed))
             return True
@@ -314,8 +316,8 @@
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
-            overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
+                int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.wrap(overflowed))
             return True
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -216,6 +216,19 @@
 neg_msg = "can't convert negative number to unsigned"
 ovf_msg = "long too big to convert"
 
+ at specialize.arg(1)
+def signext(value, size):
+    # 'value' is sign-extended from 'size' bytes to a full integer.
+    # 'size' should be a constant smaller than a full integer size.
+    if size == rffi.sizeof(rffi.SIGNEDCHAR):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SIGNEDCHAR, value))
+    elif size == rffi.sizeof(rffi.SHORT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SHORT, value))
+    elif size == rffi.sizeof(rffi.INT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+    else:
+        raise AssertionError("unsupported size")
+
 # ____________________________________________________________
 
 class _NotStandardObject(Exception):
@@ -334,13 +347,26 @@
 
 # ____________________________________________________________
 
-def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+def pack_list_to_raw_array_bounds_signed(int_list, target, size):
     for TP, TPP in _prim_signed_types:
         if size == rffi.sizeof(TP):
             ptr = rffi.cast(TPP, target)
             for i in range(len(int_list)):
                 x = int_list[i]
-                if r_uint(x) - vmin > vrangemax:
+                y = rffi.cast(TP, x)
+                if x != rffi.cast(lltype.Signed, y):
+                    return x      # overflow
+                ptr[i] = y
+            return 0
+    raise NotImplementedError("bad integer size")
+
+def pack_list_to_raw_array_bounds_unsigned(int_list, target, size, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) > vrangemax:
                     return x      # overflow
                 ptr[i] = rffi.cast(TP, x)
             return 0
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -304,7 +304,7 @@
             py.test.skip("works with internals of _file impl on py.py")
         state = [0]
         def read(fd, n=None):
-            if fd != 42:
+            if fd != 424242:
                 return cls.old_read(fd, n)
             if state[0] == 0:
                 state[0] += 1
@@ -315,7 +315,7 @@
             return ''
         os.read = read
         stdin = W_File(cls.space)
-        stdin.file_fdopen(42, 'rb', 1)
+        stdin.file_fdopen(424242, 'rb', 1)
         stdin.name = '<stdin>'
         cls.w_stream = stdin
 
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -24,12 +24,19 @@
 
 separate_module_source = """
 #include <openssl/crypto.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 
 static unsigned int _ssl_locks_count = 0;
 static struct RPyOpaque_ThreadLock *_ssl_locks;
 
 static unsigned long _ssl_thread_id_function(void) {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (unsigned long)GetCurrentThreadId();
+#else
+    return (unsigned long)pthread_self();
+#endif
 }
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
diff --git a/pypy/module/cpyext/src/pythread.c b/pypy/module/cpyext/src/pythread.c
--- a/pypy/module/cpyext/src/pythread.c
+++ b/pypy/module/cpyext/src/pythread.c
@@ -1,11 +1,18 @@
 #include <Python.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 #include "pythread.h"
 #include "src/thread.h"
 
 long
 PyThread_get_thread_ident(void)
 {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (long)GetCurrentThreadId();
+#else
+    return (long)pthread_self();
+#endif
 }
 
 PyThread_type_lock
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -144,7 +144,6 @@
     atom_int(tc, int)           puts code and int
     atom_int64(tc, int64)       puts code and int64
     atom_str(tc, str)           puts code, len and string
-    atom_strlist(tc, strlist)   puts code, len and list of strings
 
     building blocks for compound types:
 
@@ -198,15 +197,6 @@
         self.atom_int(typecode, len(x))
         self.put(x)
 
-    def atom_strlist(self, typecode, tc2, x):
-        self.atom_int(typecode, len(x))
-        atom_str = self.atom_str
-        for item in x:
-            # type(str) seems to be forbidden
-            #if type(item) is not str:
-            #    self.raise_exc('object with wrong type in strlist')
-            atom_str(tc2, item)
-
     def start(self, typecode):
         # type(char) not supported
         self.put(typecode)
@@ -379,16 +369,6 @@
         self.start(typecode)
         return self.get_lng()
 
-    def atom_strlist(self, typecode, tc2):
-        self.start(typecode)
-        lng = self.get_lng()
-        res = [None] * lng
-        idx = 0
-        while idx < lng:
-            res[idx] = self.atom_str(tc2)
-            idx += 1
-        return res
-
     def start(self, typecode):
         tc = self.get1()
         if tc != typecode:
@@ -436,7 +416,6 @@
 
     def get_w_obj(self, allow_null=False):
         space = self.space
-        w_ret = space.w_None # something not None
         tc = self.get1()
         w_ret = self._dispatch[ord(tc)](space, self, tc)
         if w_ret is None and not allow_null:
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from rpython.tool.pairtype import extendabletype
-
+from pypy.module.micronumpy import support
 
 def wrap_impl(space, w_cls, w_instance, impl):
     if w_cls is None or space.is_w(w_cls, space.gettypefor(W_NDimArray)):
@@ -45,11 +45,32 @@
         return W_NDimArray(impl)
 
     @staticmethod
-    def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False,
-                               w_subtype=None, w_base=None, writable=True):
+    def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1,
+                               order='C', owning=False, w_subtype=None,
+                               w_base=None, writable=True, strides=None):
         from pypy.module.micronumpy import concrete
-        from pypy.module.micronumpy.strides import calc_strides
-        strides, backstrides = calc_strides(shape, dtype, order)
+        from pypy.module.micronumpy.strides import (calc_strides,
+                                                    calc_backstrides)
+        isize = dtype.elsize
+        if storage_bytes > 0 :
+            totalsize = support.product(shape) * isize
+            if totalsize > storage_bytes:
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "buffer is too small for requested array"))
+        else:
+            storage_bytes = support.product(shape) * isize
+        if strides is None:
+            strides, backstrides = calc_strides(shape, dtype, order)
+        else:
+            if len(strides) != len(shape):
+                raise oefmt(space.w_ValueError,
+                    'strides, if given, must be the same length as shape')
+            for i in range(len(strides)):
+                if strides[i] < 0 or strides[i]*shape[i] > storage_bytes:
+                    raise oefmt(space.w_ValueError,
+                        'strides is incompatible with shape of requested '
+                        'array and size of buffer')
+            backstrides = calc_backstrides(strides, shape)
         if w_base is not None:
             if owning:
                 raise OperationError(space.w_ValueError,
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -168,7 +168,7 @@
         if len(args_w) >= 1:
             for w_arg in args_w:
                 try:
-                    idx = support.index_w(space, w_arg)
+                    support.index_w(space, w_arg)
                 except OperationError:
                     raise oefmt(space.w_TypeError, "an integer is required")
             raise oefmt(space.w_ValueError, "axes don't match array")
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -11,7 +11,7 @@
 from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
     RecordChunk, calc_strides, calc_new_strides, shape_agreement,
-    calculate_broadcast_strides)
+    calculate_broadcast_strides, calc_backstrides)
 
 
 class BaseConcreteArray(object):
@@ -47,6 +47,7 @@
     def setitem(self, index, value):
         self.dtype.itemtype.store(self, index, 0, value)
 
+    @jit.unroll_safe
     def setslice(self, space, arr):
         if len(arr.get_shape()) > 0 and len(self.get_shape()) == 0:
             raise oefmt(space.w_ValueError,
@@ -78,10 +79,7 @@
                                                self.get_strides(), self.order)
         if new_strides is not None:
             # We can create a view, strides somehow match up.
-            ndims = len(new_shape)
-            new_backstrides = [0] * ndims
-            for nd in range(ndims):
-                new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd]
+            new_backstrides = calc_backstrides(new_strides, new_shape)
             assert isinstance(orig_array, W_NDimArray) or orig_array is None
             return SliceArray(self.start, new_strides, new_backstrides,
                               new_shape, self, orig_array)
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -302,5 +302,5 @@
         return a
     else:
         writable = not buf.readonly
-    return W_NDimArray.from_shape_and_storage(space, [n], storage, dtype=dtype,
-                                              w_base=w_buffer, writable=writable)
+    return W_NDimArray.from_shape_and_storage(space, [n], storage, storage_bytes=s, 
+                                dtype=dtype, w_base=w_buffer, writable=writable)
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -288,7 +288,6 @@
     def descr_hash(self, space):
         return space.wrap(self._compute_hash(space, 0x345678))
 
-
     def descr_str(self, space):
         if self.fields:
             return space.str(self.descr_get_descr(space))
diff --git a/pypy/module/micronumpy/flagsobj.py b/pypy/module/micronumpy/flagsobj.py
--- a/pypy/module/micronumpy/flagsobj.py
+++ b/pypy/module/micronumpy/flagsobj.py
@@ -1,3 +1,5 @@
+from rpython.rlib import jit
+
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import interp2app
@@ -13,6 +15,7 @@
     arr.flags &= ~flags
 
 
+ at jit.unroll_safe
 def _update_contiguous_flags(arr):
     shape = arr.shape
     strides = arr.strides
diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py
--- a/pypy/module/micronumpy/flatiter.py
+++ b/pypy/module/micronumpy/flatiter.py
@@ -22,6 +22,9 @@
     def get_shape(self):
         return self.shape
 
+    def get_size(self):
+        return self.base().get_size()
+
     def create_iter(self, shape=None, backward_broadcast=False):
         assert isinstance(self.base(), W_NDimArray)
         return self.base().create_iter()
@@ -41,8 +44,8 @@
         return space.wrap(self.state.index)
 
     def descr_coords(self, space):
-        self.state = self.iter.update(self.state)
-        return space.newtuple([space.wrap(c) for c in self.state.indices])
+        coords = self.iter.indices(self.state)
+        return space.newtuple([space.wrap(c) for c in coords])
 
     def descr_iter(self):
         return self
@@ -54,7 +57,7 @@
         if self.iter.done(self.state):
             raise OperationError(space.w_StopIteration, space.w_None)
         w_res = self.iter.getitem(self.state)
-        self.state = self.iter.next(self.state)
+        self.iter.next(self.state, mutate=True)
         return w_res
 
     def descr_getitem(self, space, w_idx):
@@ -71,7 +74,7 @@
                                          base.get_order(), w_instance=base)
             return loop.flatiter_getitem(res, self.iter, state, step)
         finally:
-            self.state = self.iter.reset(self.state)
+            self.iter.reset(self.state, mutate=True)
 
     def descr_setitem(self, space, w_idx, w_value):
         if not (space.isinstance_w(w_idx, space.w_int) or
@@ -91,7 +94,7 @@
             arr = convert_to_array(space, w_value)
             loop.flatiter_setitem(space, dtype, arr, self.iter, state, step, length)
         finally:
-            self.state = self.iter.reset(self.state)
+            self.iter.reset(self.state, mutate=True)
 
 
 W_FlatIterator.typedef = TypeDef("numpy.flatiter",
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -41,16 +41,6 @@
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.flagsobj import _update_contiguous_flags
 
-class OpFlag(object):
-    def __init__(self):
-        self.rw = ''
-        self.broadcast = True
-        self.force_contig = False
-        self.force_align = False
-        self.native_byte_order = False
-        self.tmp_copy = ''
-        self.allocate = False
-
 
 class PureShapeIter(object):
     def __init__(self, shape, idx_w):
@@ -87,25 +77,24 @@
 
 
 class IterState(object):
-    _immutable_fields_ = ['iterator', 'index', 'indices', 'offset']
+    _immutable_fields_ = ['iterator', '_indices']
 
     def __init__(self, iterator, index, indices, offset):
         self.iterator = iterator
         self.index = index
-        self.indices = indices
+        self._indices = indices
         self.offset = offset
 
 
 class ArrayIter(object):
     _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
                           'strides[*]', 'backstrides[*]', 'factors[*]',
-                          'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]',
-                          'track_index', 'operand_type', 'slice_operand_type']
+                          'track_index']
 
     track_index = True
 
-    def __init__(self, array, size, shape, strides, backstrides, op_flags=OpFlag()):
-        from pypy.module.micronumpy import concrete
+    @jit.unroll_safe
+    def __init__(self, array, size, shape, strides, backstrides):
         assert len(shape) == len(strides) == len(backstrides)
         _update_contiguous_flags(array)
         self.contiguous = (array.flags & NPY.ARRAY_C_CONTIGUOUS and
@@ -117,10 +106,6 @@
         self.shape_m1 = [s - 1 for s in shape]
         self.strides = strides
         self.backstrides = backstrides
-        self.slice_shape = []
-        self.slice_stride = []
-        self.slice_backstride = []
-        self.slice_operand_type = concrete.SliceArray
 
         ndim = len(shape)
         factors = [0] * ndim
@@ -130,32 +115,35 @@
             else:
                 factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i]
         self.factors = factors
-        if op_flags.rw == 'r':
-            self.operand_type = concrete.ConcreteNonWritableArrayWithBase
-        else:
-            self.operand_type = concrete.ConcreteArrayWithBase
 
     @jit.unroll_safe
-    def reset(self, state=None):
+    def reset(self, state=None, mutate=False):
+        index = 0
         if state is None:
             indices = [0] * len(self.shape_m1)
         else:
             assert state.iterator is self
-            indices = state.indices
+            indices = state._indices
             for i in xrange(self.ndim_m1, -1, -1):
                 indices[i] = 0
-        return IterState(self, 0, indices, self.array.start)
+        offset = self.array.start
+        if not mutate:
+            return IterState(self, index, indices, offset)
+        state.index = index
+        state.offset = offset
 
     @jit.unroll_safe
-    def next(self, state):
+    def next(self, state, mutate=False):
         assert state.iterator is self
         index = state.index
         if self.track_index:
             index += 1
-        indices = state.indices
+        indices = state._indices
         offset = state.offset
         if self.contiguous:
             offset += self.array.dtype.elsize
+        elif self.ndim_m1 == 0:
+            offset += self.strides[0]
         else:
             for i in xrange(self.ndim_m1, -1, -1):
                 idx = indices[i]
@@ -166,13 +154,18 @@
                 else:
                     indices[i] = 0
                     offset -= self.backstrides[i]
-        return IterState(self, index, indices, offset)
+        if not mutate:
+            return IterState(self, index, indices, offset)
+        state.index = index
+        state.offset = offset
 
     @jit.unroll_safe
     def goto(self, index):
         offset = self.array.start
         if self.contiguous:
             offset += index * self.array.dtype.elsize
+        elif self.ndim_m1 == 0:
+            offset += index * self.strides[0]
         else:
             current = index
             for i in xrange(len(self.shape_m1)):
@@ -181,20 +174,20 @@
         return IterState(self, index, None, offset)
 
     @jit.unroll_safe
-    def update(self, state):
+    def indices(self, state):
         assert state.iterator is self
         assert self.track_index
-        if not self.contiguous:
-            return state
+        indices = state._indices
+        if not (self.contiguous or self.ndim_m1 == 0):
+            return indices
         current = state.index
-        indices = state.indices
         for i in xrange(len(self.shape_m1)):
             if self.factors[i] != 0:
                 indices[i] = current / self.factors[i]
                 current %= self.factors[i]
             else:
                 indices[i] = 0
-        return IterState(self, state.index, indices, state.offset)
+        return indices
 
     def done(self, state):
         assert state.iterator is self
@@ -213,12 +206,6 @@
         assert state.iterator is self
         self.array.setitem(state.offset, elem)
 
-    def getoperand(self, st, base):
-        impl = self.operand_type
-        res = impl([], self.array.dtype, self.array.order, [], [],
-                   self.array.storage, base)
-        res.start = st.offset
-        return res
 
 def AxisIter(array, shape, axis, cumulative):
     strides = array.get_strides()
@@ -242,42 +229,3 @@
         size /= shape[axis]
     shape[axis] = backstrides[axis] = 0
     return ArrayIter(array, size, shape, array.strides, backstrides)
-
-class SliceIter(ArrayIter):
-    '''
-    used with external loops, getitem and setitem return a SliceArray
-    view into the original array
-    '''
-    _immutable_fields_ = ['base']
-
-    def __init__(self, array, size, shape, strides, backstrides, slice_shape,
-                 slice_stride, slice_backstride, op_flags, base):
-        from pypy.module.micronumpy import concrete
-        ArrayIter.__init__(self, array, size, shape, strides, backstrides, op_flags)
-        self.slice_shape = slice_shape
-        self.slice_stride = slice_stride
-        self.slice_backstride = slice_backstride
-        self.base = base
-        if op_flags.rw == 'r':
-            self.slice_operand_type = concrete.NonWritableSliceArray
-        else:
-            self.slice_operand_type = concrete.SliceArray
-
-    def getitem(self, state):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def getitem_bool(self, state):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def setitem(self, state, elem):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def getoperand(self, state, base):
-        assert state.iterator is self
-        impl = self.slice_operand_type
-        arr = impl(state.offset, self.slice_stride, self.slice_backstride,
-                   self.slice_shape, self.array, self.base)
-        return arr
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -43,23 +43,38 @@
 
     # TODO handle __array_priorities__ and maybe flip the order
 
+    if w_lhs.get_size() == 1:
+        w_left = w_lhs.get_scalar_value().convert_to(space, calc_dtype)
+        left_iter = left_state = None
+    else:
+        w_left = None
+        left_iter, left_state = w_lhs.create_iter(shape)
+        left_iter.track_index = False
+
+    if w_rhs.get_size() == 1:
+        w_right = w_rhs.get_scalar_value().convert_to(space, calc_dtype)
+        right_iter = right_state = None
+    else:
+        w_right = None
+        right_iter, right_state = w_rhs.create_iter(shape)
+        right_iter.track_index = False
+
     if out is None:
         out = W_NDimArray.from_shape(space, shape, res_dtype,
                                      w_instance=lhs_for_subtype)
-    left_iter, left_state = w_lhs.create_iter(shape)
-    right_iter, right_state = w_rhs.create_iter(shape)
     out_iter, out_state = out.create_iter(shape)
-    left_iter.track_index = right_iter.track_index = False
     shapelen = len(shape)
     while not out_iter.done(out_state):
         call2_driver.jit_merge_point(shapelen=shapelen, func=func,
                                      calc_dtype=calc_dtype, res_dtype=res_dtype)
-        w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
-        w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
+        if left_iter:
+            w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
+            left_state = left_iter.next(left_state)
+        if right_iter:
+            w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
+            right_state = right_iter.next(right_state)
         out_iter.setitem(out_state, func(calc_dtype, w_left, w_right).convert_to(
             space, res_dtype))
-        left_state = left_iter.next(left_state)
-        right_state = right_iter.next(right_state)
         out_state = out_iter.next(out_state)
     return out
 
@@ -69,11 +84,12 @@
     reds='auto')
 
 def call1(space, shape, func, calc_dtype, res_dtype, w_obj, out):
+    obj_iter, obj_state = w_obj.create_iter(shape)
+    obj_iter.track_index = False
+
     if out is None:
         out = W_NDimArray.from_shape(space, shape, res_dtype, w_instance=w_obj)
-    obj_iter, obj_state = w_obj.create_iter(shape)
     out_iter, out_state = out.create_iter(shape)
-    obj_iter.track_index = False
     shapelen = len(shape)
     while not out_iter.done(out_state):
         call1_driver.jit_merge_point(shapelen=shapelen, func=func,
@@ -172,10 +188,23 @@
                                 reds = 'auto')
 
 def setslice(space, shape, target, source):
+    if not shape:
+        dtype = target.dtype
+        val = source.getitem(source.start)
+        if dtype.is_str_or_unicode():
+            val = dtype.coerce(space, val)
+        else:
+            val = val.convert_to(space, dtype)
+        target.setitem(target.start, val)
+        return target
+    return _setslice(space, shape, target, source)
+
+def _setslice(space, shape, target, source):
     # note that unlike everything else, target and source here are
     # array implementations, not arrays
     target_iter, target_state = target.create_iter(shape)
     source_iter, source_state = source.create_iter(shape)
+    source_iter.track_index = False
     dtype = target.dtype
     shapelen = len(shape)
     while not target_iter.done(target_state):
@@ -294,10 +323,9 @@
             state = x_state
     return out
 
-axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce',
-                                    greens=['shapelen',
-                                            'func', 'dtype'],
-                                    reds='auto')
+axis_reduce_driver = jit.JitDriver(name='numpy_axis_reduce',
+                                   greens=['shapelen', 'func', 'dtype'],
+                                   reds='auto')
 
 def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity, cumulative,
                    temp):
@@ -310,21 +338,24 @@
         temp_iter = out_iter  # hack
         temp_state = out_state
     arr_iter, arr_state = arr.create_iter()
+    arr_iter.track_index = False
     if identity is not None:
         identity = identity.convert_to(space, dtype)
     shapelen = len(shape)
     while not out_iter.done(out_state):
-        axis_reduce__driver.jit_merge_point(shapelen=shapelen, func=func,
-                                            dtype=dtype)
-        assert not arr_iter.done(arr_state)
+        axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func,
+                                           dtype=dtype)
         w_val = arr_iter.getitem(arr_state).convert_to(space, dtype)
-        out_state = out_iter.update(out_state)
-        if out_state.indices[axis] == 0:
+        arr_state = arr_iter.next(arr_state)
+
+        out_indices = out_iter.indices(out_state)
+        if out_indices[axis] == 0:
             if identity is not None:
                 w_val = func(dtype, identity, w_val)
         else:
             cur = temp_iter.getitem(temp_state)
             w_val = func(dtype, cur, w_val)
+
         out_iter.setitem(out_state, w_val)
         out_state = out_iter.next(out_state)
         if cumulative:
@@ -332,7 +363,6 @@
             temp_state = temp_iter.next(temp_state)
         else:
             temp_state = out_state
-        arr_state = arr_iter.next(arr_state)
     return out
 
 
@@ -451,9 +481,9 @@
     while not arr_iter.done(arr_state):
         nonzero_driver.jit_merge_point(shapelen=shapelen, dims=dims, dtype=dtype)
         if arr_iter.getitem_bool(arr_state):
-            arr_state = arr_iter.update(arr_state)
+            arr_indices = arr_iter.indices(arr_state)
             for d in dims:
-                res_iter.setitem(res_state, box(arr_state.indices[d]))
+                res_iter.setitem(res_state, box(arr_indices[d]))
                 res_state = res_iter.next(res_state)
         arr_state = arr_iter.next(arr_state)
     return res
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -516,9 +516,10 @@
                 "__array__(dtype) not implemented"))
         if type(self) is W_NDimArray:
             return self
+        sz = support.product(self.get_shape()) * self.get_dtype().elsize
         return W_NDimArray.from_shape_and_storage(
             space, self.get_shape(), self.implementation.storage,
-            self.get_dtype(), w_base=self)
+            self.get_dtype(), storage_bytes=sz, w_base=self)
 
     def descr_array_iface(self, space):
         addr = self.implementation.get_storage_as_int(space)
@@ -1180,8 +1181,8 @@
                         "improper dtype '%R'", dtype)
         self.implementation = W_NDimArray.from_shape_and_storage(
             space, [space.int_w(i) for i in space.listview(shape)],
-            rffi.str2charp(space.str_w(storage), track_allocation=False),
-            dtype, owning=True).implementation
+            rffi.str2charp(space.str_w(storage), track_allocation=False), 
+            dtype, storage_bytes=space.len_w(storage), owning=True).implementation
 
     def descr___array_finalize__(self, space, w_obj):
         pass
@@ -1205,8 +1206,10 @@
 
     if not space.is_none(w_buffer):
         if (not space.is_none(w_strides)):
-            raise OperationError(space.w_NotImplementedError,
-                                 space.wrap("unsupported param"))
+            strides = [space.int_w(w_i) for w_i in
+                       space.unpackiterable(w_strides)]
+        else:
+            strides = None
 
         try:
             buf = space.writebuf_w(w_buffer)
@@ -1220,16 +1223,14 @@
         if not shape:
             raise OperationError(space.w_TypeError, space.wrap(
                 "numpy scalars from buffers not supported yet"))
-        totalsize = support.product(shape) * dtype.elsize
-        if totalsize + offset > buf.getlength():
-            raise OperationError(space.w_TypeError, space.wrap(
-                "buffer is too small for requested array"))
         storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr)
         storage = rffi.ptradd(storage, offset)
-        return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
+        return W_NDimArray.from_shape_and_storage(space, shape, storage,
+                                                  dtype, w_base=w_buffer,
+                                                  storage_bytes=buf.getlength()-offset,
                                                   w_subtype=w_subtype,
-                                                  w_base=w_buffer,
-                                                  writable=not buf.readonly)
+                                                  writable=not buf.readonly,
+                                                  strides=strides)
 
     order = order_converter(space, w_order, NPY.CORDER)
     if order == NPY.CORDER:
@@ -1247,8 +1248,9 @@
     return w_ret
 
 
- at unwrap_spec(addr=int)
-def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype, w_subtype=None):
+ at unwrap_spec(addr=int, buf_len=int)
+def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype,
+                buf_len=-1, w_subtype=None, w_strides=None):
     """
     Create an array from an existing buffer, given its address as int.
     PyPy-only implementation detail.
@@ -1257,14 +1259,22 @@
     dtype = space.interp_w(descriptor.W_Dtype, space.call_function(
         space.gettypefor(descriptor.W_Dtype), w_dtype))
     shape = shape_converter(space, w_shape, dtype)
+    if not space.is_none(w_strides):
+        strides = [space.int_w(w_i) for w_i in
+                   space.unpackiterable(w_strides)]
+    else:
+        strides = None
     if w_subtype:
         if not space.isinstance_w(w_subtype, space.w_type):
             raise OperationError(space.w_ValueError, space.wrap(
                 "subtype must be a subtype of ndarray, not a class instance"))
         return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
-                                                  'C', False, w_subtype)
+                                                  buf_len, 'C', False, w_subtype,
+                                                  strides=strides)
     else:
-        return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype)
+        return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
+                                                  storage_bytes=buf_len,
+                                                  strides=strides)
 
 app_take = applevel(r"""
     def take(a, indices, axis, out, mode):
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -1,3 +1,4 @@
+from rpython.rlib import jit
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -5,7 +6,7 @@
 from pypy.module.micronumpy import support, concrete
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.descriptor import decode_w_dtype
-from pypy.module.micronumpy.iterators import ArrayIter, SliceIter, OpFlag
+from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
                                             shape_agreement, shape_agreement_multiple)
 
@@ -35,6 +36,16 @@
     return ret
 
 
+class OpFlag(object):
+    def __init__(self):
+        self.rw = ''
+        self.broadcast = True
+        self.force_contig = False
+        self.force_align = False
+        self.native_byte_order = False
+        self.tmp_copy = ''
+        self.allocate = False
+
 def parse_op_flag(space, lst):
     op_flag = OpFlag()
     for w_item in lst:
@@ -141,11 +152,73 @@
         raise NotImplementedError('not implemented yet')
 
 
-def get_iter(space, order, arr, shape, dtype, op_flags):
+class OperandIter(ArrayIter):
+    _immutable_fields_ = ['slice_shape', 'slice_stride', 'slice_backstride',
+                          'operand_type', 'base']
+
+    def getitem(self, state):
+        # cannot be called - must return a boxed value
+        assert False
+
+    def getitem_bool(self, state):
+        # cannot be called - must return a boxed value
+        assert False
+
+    def setitem(self, state, elem):
+        # cannot be called - must return a boxed value
+        assert False
+
+
+class ConcreteIter(OperandIter):
+    def __init__(self, array, size, shape, strides, backstrides,
+                 op_flags, base):
+        OperandIter.__init__(self, array, size, shape, strides, backstrides)
+        self.slice_shape = 1
+        self.slice_stride = -1
+        if strides:
+            self.slice_stride = strides[-1]
+        self.slice_backstride = 1
+        if op_flags.rw == 'r':
+            self.operand_type = concrete.ConcreteNonWritableArrayWithBase
+        else:
+            self.operand_type = concrete.ConcreteArrayWithBase
+        self.base = base
+
+    def getoperand(self, state):
+        assert state.iterator is self
+        impl = self.operand_type
+        res = impl([], self.array.dtype, self.array.order, [], [],
+                   self.array.storage, self.base)
+        res.start = state.offset
+        return res
+
+
+class SliceIter(OperandIter):
+    def __init__(self, array, size, shape, strides, backstrides, slice_shape,
+                 slice_stride, slice_backstride, op_flags, base):
+        OperandIter.__init__(self, array, size, shape, strides, backstrides)
+        self.slice_shape = slice_shape
+        self.slice_stride = slice_stride
+        self.slice_backstride = slice_backstride
+        if op_flags.rw == 'r':
+            self.operand_type = concrete.NonWritableSliceArray
+        else:
+            self.operand_type = concrete.SliceArray
+        self.base = base
+
+    def getoperand(self, state):
+        assert state.iterator is self
+        impl = self.operand_type
+        arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
+                   [self.slice_shape], self.array, self.base)
+        return arr
+
+
+def get_iter(space, order, arr, shape, dtype, op_flags, base):
     imp = arr.implementation
     backward = is_backward(imp, order)
     if arr.is_scalar():
-        return ArrayIter(imp, 1, [], [], [], op_flags=op_flags)
+        return ConcreteIter(imp, 1, [], [], [], op_flags, base)
     if (imp.strides[0] < imp.strides[-1] and not backward) or \
        (imp.strides[0] > imp.strides[-1] and backward):
         # flip the strides. Is this always true for multidimension?
@@ -160,7 +233,7 @@
         backstrides = imp.backstrides
     r = calculate_broadcast_strides(strides, backstrides, imp.shape,
                                     shape, backward)
-    return ArrayIter(imp, imp.get_size(), shape, r[0], r[1], op_flags=op_flags)
+    return ConcreteIter(imp, imp.get_size(), shape, r[0], r[1], op_flags, base)
 
 def calculate_ndim(op_in, oa_ndim):
     if oa_ndim >=0:
@@ -269,8 +342,8 @@
         self.index = [0] * len(shape)
         self.backward = backward
 
+    @jit.unroll_safe
     def next(self):
-        # TODO It's probably possible to refactor all the "next" method from each iterator
         for i in range(len(self.shape) - 1, -1, -1):
             if self.index[i] < self.shape[i] - 1:
                 self.index[i] += 1
@@ -404,7 +477,7 @@
         self.iters = []
         for i in range(len(self.seq)):
             it = get_iter(space, self.order, self.seq[i], self.shape,
-                          self.dtypes[i], self.op_flags[i])
+                          self.dtypes[i], self.op_flags[i], self)
             it.contiguous = False
             self.iters.append((it, it.reset()))
 
@@ -443,7 +516,7 @@
         return space.wrap(self)
 
     def getitem(self, it, st):
-        res = it.getoperand(st, self)
+        res = it.getoperand(st)
         return W_NDimArray(res)
 
     def descr_getitem(self, space, w_idx):
@@ -461,6 +534,7 @@
     def descr_len(self, space):
         space.wrap(len(self.iters))
 
+    @jit.unroll_safe
     def descr_next(self, space):
         for it, st in self.iters:
             if not it.done(st):
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -270,7 +270,7 @@
                 shape = shape_agreement(space, shape, arr)
     return shape
 
-
+ at jit.unroll_safe
 def _shape_agreement(shape1, shape2):
     """ Checks agreement about two shapes with respect to broadcasting. Returns
     the resulting shape.
@@ -362,6 +362,13 @@
         backstrides.reverse()
     return strides, backstrides
 
+ at jit.unroll_safe
+def calc_backstrides(strides, shape):
+    ndims = len(shape)
+    new_backstrides = [0] * ndims
+    for nd in range(ndims):
+        new_backstrides[nd] = (shape[nd] - 1) * strides[nd]
+    return new_backstrides
 
 # Recalculating strides. Find the steps that the iteration does for each
 # dimension, given the stride and shape. Then try to create a new stride that
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -392,6 +392,7 @@
         t5 = dtype([('x', '<f4'), ('y', '<i4')])
         t6 = dtype([('y', '<i4'), ('x', '<f4')])
         assert hash(t5) != hash(t6)
+
     def test_pickle(self):
         import numpy as np
         from numpypy import array, dtype
diff --git a/pypy/module/micronumpy/test/test_iterators.py b/pypy/module/micronumpy/test/test_iterators.py
--- a/pypy/module/micronumpy/test/test_iterators.py
+++ b/pypy/module/micronumpy/test/test_iterators.py
@@ -31,16 +31,14 @@
         s = i.next(s)
         assert s.offset == 3
         assert not i.done(s)
-        assert s.indices == [0,0]
-        s = i.update(s)
-        assert s.indices == [0,3]
+        assert s._indices == [0,0]
+        assert i.indices(s) == [0,3]
         #cause a dimension overflow
         s = i.next(s)
         s = i.next(s)
         assert s.offset == 5
-        assert s.indices == [0,3]
-        s = i.update(s)
-        assert s.indices == [1,0]
+        assert s._indices == [0,3]
+        assert i.indices(s) == [1,0]
 
         #Now what happens if the array is transposed? strides[-1] != 1
         # therefore layout is non-contiguous
@@ -56,12 +54,12 @@
         s = i.next(s)
         assert s.offset == 9
         assert not i.done(s)
-        assert s.indices == [0,3]
+        assert s._indices == [0,3]
         #cause a dimension overflow
         s = i.next(s)
         s = i.next(s)
         assert s.offset == 1
-        assert s.indices == [1,0]
+        assert s._indices == [1,0]
 
     def test_iterator_goto(self):
         shape = [3, 5]
@@ -74,9 +72,9 @@
         assert not i.contiguous
         s = i.reset()
         assert s.index == 0
-        assert s.indices == [0, 0]
+        assert s._indices == [0, 0]
         assert s.offset == a.start
         s = i.goto(11)
         assert s.index == 11
-        assert s.indices is None
+        assert s._indices is None
         assert s.offset == a.start + 5
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -186,7 +186,8 @@
         #
         dtypes = get_dtype_cache(self.space)
         w_array = W_NDimArray.from_shape_and_storage(self.space, [2, 2],
-                                                storage, dtypes.w_int8dtype)
+                                                storage, dtypes.w_int8dtype,
+                                                storage_bytes=4)
         def get(i, j):
             return w_array.getitem(self.space, [i, j]).value
         assert get(0, 0) == 0
@@ -3416,6 +3417,21 @@
         assert str(array(1.5)) == '1.5'
         assert str(array(1.5).real) == '1.5'
 
+    def test_ndarray_buffer_strides(self):
+        from numpy import ndarray, array
+        base = array([1, 2, 3, 4], dtype=int)
+        a = ndarray((4,), buffer=base, dtype=int)
+        assert a[1] == 2
+        a = ndarray((4,), buffer=base, dtype=int, strides=[base.strides[0]])
+        assert a[1] == 2
+        a = ndarray((2,), buffer=base, dtype=int, strides=[2 * base.strides[0]])
+        assert a[1] == 3
+        exc = raises(ValueError, ndarray, (4,), buffer=base, dtype=int, strides=[2 * base.strides[0]])
+        assert exc.value[0] == 'strides is incompatible with shape of requested array and size of buffer'
+        exc = raises(ValueError, ndarray, (2, 1), buffer=base, dtype=int, strides=[base.strides[0]])
+        assert exc.value[0] == 'strides, if given, must be the same length as shape'
+
+
 
 class AppTestRepr(BaseNumpyAppTest):
     def setup_class(cls):
@@ -3883,6 +3899,7 @@
         assert np.greater(a, a) is NotImplemented
         assert np.less_equal(a, a) is NotImplemented
 
+
 class AppTestPyPy(BaseNumpyAppTest):
     def setup_class(cls):
         if option.runappdirect and '__pypy__' not in sys.builtin_module_names:
@@ -3906,13 +3923,14 @@
         from numpypy import array, ndarray
         x = array([1, 2, 3, 4])
         addr, _ = x.__array_interface__['data']
-        y = ndarray._from_shape_and_storage([2, 2], addr, x.dtype)
+        sz = x.size * x.dtype.itemsize
+        y = ndarray._from_shape_and_storage([2, 2], addr, x.dtype, sz)
         assert y[0, 1] == 2
         y[0, 1] = 42
         assert x[1] == 42
         class C(ndarray):
             pass
-        z = ndarray._from_shape_and_storage([4, 1], addr, x.dtype, C)
+        z = ndarray._from_shape_and_storage([4, 1], addr, x.dtype, sz, C)
         assert isinstance(z, C)
         assert z.shape == (4, 1)
         assert z[1, 0] == 42
@@ -3926,3 +3944,17 @@
         assert x.__pypy_data__ is obj
         del x.__pypy_data__
         assert x.__pypy_data__ is None
+
+    def test_from_shape_and_storage_strides(self):
+        from numpy import ndarray, array
+        base = array([1, 2, 3, 4], dtype=int)
+        addr, _ = base.__array_interface__['data']
+        sz = base.size * base.dtype.itemsize
+        a = ndarray._from_shape_and_storage((4,), addr, int, sz)
+        assert a[1] == 2
+        a = ndarray._from_shape_and_storage((4,), addr, int, sz,
+                                           strides=[base.strides[0]])
+        assert a[1] == 2
+        a = ndarray._from_shape_and_storage((2,), addr, int, sz,
+                                           strides=[2 * base.strides[0]])
+        assert a[1] == 3
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -947,6 +947,7 @@
         assert exc.value[0] == "'axis' entry is out of bounds"
 
     def test_reduce_1d(self):
+        import numpy as np
         from numpy import array, add, maximum, less, float16, complex64
 
         assert less.reduce([5, 4, 3, 2, 1])
@@ -961,6 +962,10 @@
         assert type(add.reduce(array([True, False] * 200, dtype='float16'))) is float16
         assert type(add.reduce(array([True, False] * 200, dtype='complex64'))) is complex64
 
+        for dtype in ['bool', 'int']:
+            assert np.equal.reduce([1, 2], dtype=dtype) == True
+            assert np.equal.reduce([1, 2, 0], dtype=dtype) == False
+
     def test_reduceND(self):
         from numpy import add, arange
         a = arange(12).reshape(3, 4)
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -103,17 +103,13 @@
         self.check_trace_count(1)
         self.check_simple_loop({
             'float_add': 1,

From noreply at buildbot.pypy.org  Fri Dec  5 12:19:09 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  5 Dec 2014 12:19:09 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: fix after merge, wip
Message-ID: <20141205111909.B62811D257D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r74828:1453b5101ed2
Date: 2014-12-05 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/1453b5101ed2/

Log:	fix after merge, wip

diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -173,11 +173,9 @@
     def __init__(self, array, size, shape, strides, backstrides,
                  op_flags, base):
         OperandIter.__init__(self, array, size, shape, strides, backstrides)
-        self.slice_shape = 1
-        self.slice_stride = -1
-        if strides:
-            self.slice_stride = strides[-1]
-        self.slice_backstride = 1
+        self.slice_shape =[] 
+        self.slice_stride = []
+        self.slice_backstride = []
         if op_flags.rw == 'r':
             self.operand_type = concrete.ConcreteNonWritableArrayWithBase
         else:
@@ -209,8 +207,8 @@
     def getoperand(self, state):
         assert state.iterator is self
         impl = self.operand_type
-        arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
-                   [self.slice_shape], self.array, self.base)
+        arr = impl(state.offset, self.slice_stride, self.slice_backstride,
+                   self.slice_shape, self.array, self.base)
         return arr
 
 
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -120,11 +120,13 @@
                       c128_dtype, c128_dtype],
                      '')
         f32_array = W_NDimArray(VoidBoxStorage(0, f32_dtype))
-        index, dtypes = ufunc.type_resolver(space, [f32_array], [None], 'd->D')
+        index, dtypes = ufunc.type_resolver(space, [f32_array], [None],
+                                            'd->D', ufunc.dtypes)
         #needs to cast input type, create output type
         assert index == 1
         assert dtypes == [f64_dtype, c128_dtype]
-        index, dtypes = ufunc.type_resolver(space, [f32_array], [None], '')
+        index, dtypes = ufunc.type_resolver(space, [f32_array], [None],
+                                             '', ufunc.dtypes)
         assert index == 0
         assert dtypes == [f32_dtype, c64_dtype]
 
@@ -229,7 +231,7 @@
                           )
         ai = arange(18, dtype=int).reshape(2,3,3)
         exc = raises(ValueError, ufunc, ai[:,:,0])
-        assert "Operand 0 has a mismatch in its core dimension 1" in exc.value.message
+        assert "perand 0 has a mismatch in its core dimension 1" in exc.value.message
         ai3 = ufunc(ai[0,:,:])
         ai2 = ufunc(ai)
         assert (ai2 == ai * 2).all()
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -590,11 +590,12 @@
             # func is going to do all the work, it must accept W_NDimArray args
             inargs0 = inargs[0]
             assert isinstance(inargs0, W_NDimArray)
-            outarg_shapes = [inargs0.get_shape()] * self.nargs
+            arg_shapes = [inargs0.get_shape()] * self.nargs
             inargs, outargs, need_to_cast = self.alloc_args(space, inargs, outargs,
-                                              dtypes, outarg_shapes)
-            if any(need_to_cast):
-                raise oefmt(space.w_NotImplementedError, "casting not supported yet")
+                                              dtypes, arg_shapes)
+            for tf in need_to_cast:
+                if tf:
+                    raise oefmt(space.w_NotImplementedError, "casting not supported yet")
             if self.stack_inputs:
                 arglist = space.newlist(list(inargs + outargs))
                 space.call_args(func, Arguments.frompacked(space, arglist))
@@ -608,8 +609,9 @@
         iter_shape, arg_shapes, matched_dims = self.verify_args(space, inargs, outargs)
         inargs, outargs, need_to_cast = self.alloc_args(space, inargs, outargs, dtypes,
                                           arg_shapes)
-        if any(need_to_cast):
-            raise oefmt(space.w_NotImplementedError, "casting not supported yet")
+        for tf in need_to_cast:
+            if tf:
+                raise oefmt(space.w_NotImplementedError, "casting not supported yet")
         w_flags = space.w_None # NOT 'external_loop', we do coalescing by core_num_dims
         w_op_flags = space.newtuple([space.wrap(r) for r in ['readonly'] * len(inargs)] + \
                                     [space.wrap(r) for r in ['readwrite'] * len(outargs)])
@@ -786,15 +788,18 @@
             if len(arg_shapes[i]) != curarg.ndims():
                 # XXX reshape (after merge with default)
                 pass
-            need_to_cast.append(curarg.get_dtype() == dtypes[i])
+            need_to_cast.append(curarg.get_dtype() != dtypes[i])
         for i in range(len(outargs)):
             j = self.nin + i
-            if outargs[i] is None:
-                outargs[i] = W_NDimArray.from_shape(space, outarg_shapes[j], dtypes[j], order)
+            curarg = outargs[i]
+            assert isinstance(curarg, W_NDimArray)
+            if not isinstance(curarg, W_NDimArray):
+                outargs[i] = W_NDimArray.from_shape(space, arg_shapes[j], dtypes[j], order)
+                curarg = outargs[i]
             elif len(arg_shapes[i]) != curarg.ndims():
                 # XXX reshape (after merge with default)
                 pass
-            need_to_cast.append(curarg.get_dtype() == dtypes[j])
+            need_to_cast.append(curarg.get_dtype() != dtypes[j])
         return inargs, outargs, need_to_cast
 
     def verify_args(self, space, inargs, outargs):
@@ -814,13 +819,13 @@
                 name = 'Input'
                 curarg = inargs[i]
             else:
-                _i = i + self.nin
+                _i = i - self.nin
                 name = 'Output'
                 curarg = outargs[_i]
             dim_offset = self.core_offsets[i]
             num_dims = self.core_num_dims[i]
             if not isinstance(curarg, W_NDimArray):
-                arg_shape = iter_shape
+                arg_shape = iter_shape[:]
                 for j in range(num_dims):
                     core_dim_index = self.core_dim_ixs[dim_offset + j]
                     if matched_dims[core_dim_index] < 0:
@@ -839,11 +844,11 @@
                     num_dims+n, self.signature, num_dims)
             dims_to_match = curarg.get_shape()[n:]
             dims_to_broadcast = curarg.get_shape()[:n]
-            offset = len(dims_to_broadcast) - len(iter_shape)
-            if offset >= 0:
+            offset = n - len(iter_shape)
+            if offset > 0:
                 # Prepend extra dimensions to iter_shape, matched_dims
                 iter_shape = dims_to_broadcast[:offset] + iter_shape
-                outarg_shapes = [dims_to_broadcast[:offset] + asp for asp in outarg_shapes]
+                arg_shapes = [dims_to_broadcast[:offset] + asp for asp in arg_shapes]
                 offset = 0
             # Make sure iter_shape[offset:] matches dims_to_broadcast
             offset = abs(offset) # for translation
@@ -856,7 +861,8 @@
                         "(size %d is different from %d)",
                          self.name, name, _i, j, x, y)
                 iter_shape[offset + j] = max(x, y)
-            # Find or verify signature ixs
+            #print 'Find or verify signature ixs',self.core_dim_ixs,
+            #print 'starting',dim_offset,'n',num_dims,'matching',dims_to_match
             for j in range(num_dims):
                 core_dim_index = self.core_dim_ixs[dim_offset + j]
                 if core_dim_index > len(dims_to_match):
@@ -865,12 +871,12 @@
                         "signature %s (index is larger than input shape)",
                          self.name, name, _i, j, self.signature, core_dim_index)
                 if matched_dims[core_dim_index] < 0:
-                    matched_dims[core_dim_index] = dims_to_match[core_dim_index]
-                elif matched_dims[core_dim_index] != dims_to_match[core_dim_index]:
+                    matched_dims[core_dim_index] = dims_to_match[j]
+                elif matched_dims[core_dim_index] != dims_to_match[j]:
                     raise oefmt(space.w_ValueError, "%s: %s operand %d has a "
                         "mismatch in its core dimension %d, with gufunc "
                         "signature %s (expected %d, got %d)",
-                         self.name, name, _i, core_dim_index + num_dims, 
+                         self.name, name, _i, j, 
                          self.signature, matched_dims[core_dim_index],
                          dims_to_match[core_dim_index])
             arg_shapes.append(iter_shape + dims_to_match)
@@ -881,6 +887,7 @@
 
         # TODO parse and handle subok
         # TODO handle flags, op_flags
+        #print 'iter_shape',iter_shape,'arg_shapes',arg_shapes,'matched_dims',matched_dims
         return iter_shape, arg_shapes, matched_dims
 
 W_Ufunc.typedef = TypeDef("numpy.ufunc",

From noreply at buildbot.pypy.org  Fri Dec  5 12:19:10 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  5 Dec 2014 12:19:10 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: translation fix
Message-ID: <20141205111910.C9BFA1D257D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r74829:9d7d30ef07d4
Date: 2014-12-05 12:53 +0200
http://bitbucket.org/pypy/pypy/changeset/9d7d30ef07d4/

Log:	translation fix

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -825,16 +825,17 @@
             dim_offset = self.core_offsets[i]
             num_dims = self.core_num_dims[i]
             if not isinstance(curarg, W_NDimArray):
-                arg_shape = iter_shape[:]
+                target_dims = []
                 for j in range(num_dims):
                     core_dim_index = self.core_dim_ixs[dim_offset + j]
-                    if matched_dims[core_dim_index] < 0:
+                    v = matched_dims[core_dim_index]
+                    if v < 0:
                         raise oefmt(space.w_ValueError, "%s: %s operand %d "
                             "is empty but unique core dimension %d in signature "
                             "%s of gufunc was not specified",
                              self.name, name, _i, core_dim_index, self.signature)
-                    arg_shape.append(matched_dims[core_dim_index])
-                arg_shapes.append(arg_shape) 
+                    target_dims.append(v)
+                arg_shapes.append(iter_shape + target_dims) 
                 continue
             n = len(curarg.get_shape()) - num_dims
             if n < 0:

From noreply at buildbot.pypy.org  Fri Dec  5 12:19:11 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  5 Dec 2014 12:19:11 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: reshape for broadcasting
Message-ID: <20141205111911.E927A1D257D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r74830:d5e8a34c0382
Date: 2014-12-05 13:18 +0200
http://bitbucket.org/pypy/pypy/changeset/d5e8a34c0382/

Log:	reshape for broadcasting

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -13,7 +13,7 @@
 from pypy.module.micronumpy.ctors import numpify
 from pypy.module.micronumpy.nditer import W_NDIter, coalesce_iter
 from pypy.module.micronumpy.strides import shape_agreement
-from pypy.module.micronumpy.support import _parse_signature
+from pypy.module.micronumpy.support import _parse_signature, product
 from rpython.rlib.rawstorage import (raw_storage_setitem, free_raw_storage,
              alloc_raw_storage)
 from rpython.rtyper.lltypesystem import rffi, lltype
@@ -786,19 +786,27 @@
             curarg = inargs[i]
             assert isinstance(curarg, W_NDimArray)
             if len(arg_shapes[i]) != curarg.ndims():
-                # XXX reshape (after merge with default)
+                # reshape
+                sz = product(curarg.get_shape()) * curarg.get_dtype().elsize
+                inargs[i] = W_NDimArray.from_shape_and_storage(
+                    space, arg_shapes[i], curarg.implementation.storage,
+                    curarg.get_dtype(), storage_bytes=sz, w_base=curarg)
                 pass
             need_to_cast.append(curarg.get_dtype() != dtypes[i])
         for i in range(len(outargs)):
             j = self.nin + i
             curarg = outargs[i]
-            assert isinstance(curarg, W_NDimArray)
             if not isinstance(curarg, W_NDimArray):
                 outargs[i] = W_NDimArray.from_shape(space, arg_shapes[j], dtypes[j], order)
                 curarg = outargs[i]
             elif len(arg_shapes[i]) != curarg.ndims():
-                # XXX reshape (after merge with default)
-                pass
+                # reshape
+                sz = product(curarg.get_shape()) * curarg.get_dtype().elsize
+                outargs[i] = W_NDimArray.from_shape_and_storage(
+                    space, arg_shapes[i], curarg.implementation.storage,
+                    curarg.get_dtype(), storage_bytes=sz, w_base=curarg)
+                curarg = outargs[i]
+            assert isinstance(curarg, W_NDimArray)
             need_to_cast.append(curarg.get_dtype() != dtypes[j])
         return inargs, outargs, need_to_cast
 

From noreply at buildbot.pypy.org  Fri Dec  5 12:24:41 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 12:24:41 +0100 (CET)
Subject: [pypy-commit] creflect default: copy loads of code from
	_cffi_backend.c
Message-ID: <20141205112441.A9E661D29F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r158:64bc5e56d01a
Date: 2014-12-05 12:24 +0100
http://bitbucket.org/cffi/creflect/changeset/64bc5e56d01a/

Log:	copy loads of code from _cffi_backend.c

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -288,9 +288,12 @@
     return ct;
 }
 
-static _crx_type_t *zef_get_array_type(_crx_builder_t *cb, _crx_type_t *ctitem,
-                                       size_t len)
+static _crx_type_t *_zef_array_type(_crx_builder_t *cb, _crx_type_t *ctitem,
+                                    size_t len, CTypeDescrObject *ctptr)
 {
+    /* this function is called with different sets of non-NULL arguments
+       from various functions (just afterwards)
+    */
     if (PyErr_Occurred())
         return NULL;
 
@@ -315,9 +318,11 @@
     if (name_obj == NULL)
         return NULL;
 
-    ct = get_cached_type(cb, name_obj);
-    if (ct && (ct->ct_flags & CT_ARRAY))
-        goto done;
+    if (cb != NULL) {
+        ct = get_cached_type(cb, name_obj);
+        if (ct && (ct->ct_flags & CT_ARRAY))
+            goto done;
+    }
 
     if (length < 0) {
         arraysize = -1;
@@ -331,6 +336,12 @@
         }
     }
 
+    if (ctptr == NULL) {
+        ctptr = zef_get_pointer_type(cb, ctitem, 0);
+        if (PyErr_Occurred())
+            goto done;
+    }
+
     ct = ctypedescr_new(name_obj, ctitem->ct_name_position);
     if (ct == NULL)
         goto done;
@@ -338,18 +349,43 @@
     ct->ct_size = arraysize;
     ct->ct_length = length;
     ct->ct_flags = CT_ARRAY;
+    Py_INCREF(ctptr);
+    ct->ct_stuff = (PyObject *)ctptr;
 
-    put_cached_type(cb, name_obj, ct);
+    if (cb != NULL) {
+        if (length < 0 && ctptr->ct_stuff == NULL) {
+            Py_INCREF(ct);
+            ctptr->ct_stuff = (PyObject *)ct;
+        }
+        put_cached_type(cb, name_obj, ct);
+    }
+    else {
+        assert(length < 0);
+        assert(ctptr->ct_stuff == NULL);
+        ctptr->ct_stuff = (PyObject *)ct;
+    }
 
  done:
     Py_DECREF(name_obj);
     return ct;
 }
 
+static int fill_array_type(CTypeDescrObject *ctptr)
+{
+    _zef_array_type(NULL, ctptr->ct_itemdescr, (size_t)-1, ctptr);
+    return PyErr_Occurred() ? -1 : 0;
+}
+
+static _crx_type_t *zef_get_array_type(_crx_builder_t *cb, _crx_type_t *ctitem,
+                                       size_t len)
+{
+    return _zef_array_type(cb, ctitem, len, NULL);
+}
+
 static _crx_type_t *zef_get_incomplete_array_type(_crx_builder_t *cb,
                                                   _crx_type_t *ctitem)
 {
-    return zef_get_array_type(cb, ctitem, (size_t)-1);
+    return _zef_array_type(cb, ctitem, (size_t)-1, NULL);
 }
 
 static _crx_type_t *zef_get_struct_type(_crx_builder_t *cb, const char *name)
diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -6,6 +6,17 @@
     PyObject *c_weakreflist;
 } CDataObject;
 
+struct cfieldobject_s {
+    PyObject_HEAD
+    CTypeDescrObject *cf_type;
+    Py_ssize_t cf_offset;
+    short cf_bitshift;   /* >= 0: bitshift; or BS_REGULAR or BS_EMPTY_ARRAY */
+    short cf_bitsize;
+    struct cfieldobject_s *cf_next;
+};
+#define BS_REGULAR     (-1)      /* a regular field, not with bitshift */
+#define BS_EMPTY_ARRAY (-2)      /* a field which is an array 'type[0]' */
+
 typedef union {
     unsigned char m_char;
     unsigned short m_short;
@@ -39,7 +50,763 @@
 #define CDataOwn_Check(ob)    (Py_TYPE(ob) == &CDataOwning_Type ||      \
                                Py_TYPE(ob) == &CDataOwningGC_Type)
 
+/************************************************************/
 
+static int
+CDataObject_Or_PyFloat_Check(PyObject *ob)
+{
+    return (PyFloat_Check(ob) ||
+            (CData_Check(ob) &&
+             (((CDataObject *)ob)->c_type->ct_flags & CT_PRIMITIVE_FLOAT)));
+}
+
+static PY_LONG_LONG
+_my_PyLong_AsLongLong(PyObject *ob)
+{
+    /* (possibly) convert and cast a Python object to a long long.
+       Like PyLong_AsLongLong(), this version accepts a Python int too, and
+       does convertions from other types of objects.  The difference is that
+       this version refuses floats. */
+#if PY_MAJOR_VERSION < 3
+    if (PyInt_Check(ob)) {
+        return PyInt_AS_LONG(ob);
+    }
+    else 
+#endif
+    if (PyLong_Check(ob)) {
+        return PyLong_AsLongLong(ob);
+    }
+    else {
+        PyObject *io;
+        PY_LONG_LONG res;
+        PyNumberMethods *nb = ob->ob_type->tp_as_number;
+
+        if (CDataObject_Or_PyFloat_Check(ob) ||
+                nb == NULL || nb->nb_int == NULL) {
+            PyErr_SetString(PyExc_TypeError, "an integer is required");
+            return -1;
+        }
+        io = (*nb->nb_int) (ob);
+        if (io == NULL)
+            return -1;
+
+        if (PyIntOrLong_Check(io)) {
+            res = _my_PyLong_AsLongLong(io);
+        }
+        else {
+            PyErr_SetString(PyExc_TypeError, "integer conversion failed");
+            res = -1;
+        }
+        Py_DECREF(io);
+        return res;
+    }
+}
+
+static unsigned PY_LONG_LONG
+_my_PyLong_AsUnsignedLongLong(PyObject *ob, int strict)
+{
+    /* (possibly) convert and cast a Python object to an unsigned long long.
+       Like PyLong_AsLongLong(), this version accepts a Python int too, and
+       does convertions from other types of objects.  If 'strict', complains
+       with OverflowError and refuses floats.  If '!strict', rounds floats
+       and masks the result. */
+#if PY_MAJOR_VERSION < 3
+    if (PyInt_Check(ob)) {
+        long value1 = PyInt_AS_LONG(ob);
+        if (strict && value1 < 0)
+            goto negative;
+        return (unsigned PY_LONG_LONG)(PY_LONG_LONG)value1;
+    }
+    else
+#endif
+    if (PyLong_Check(ob)) {
+        if (strict) {
+            if (_PyLong_Sign(ob) < 0)
+                goto negative;
+            return PyLong_AsUnsignedLongLong(ob);
+        }
+        else {
+            return PyLong_AsUnsignedLongLongMask(ob);
+        }
+    }
+    else {
+        PyObject *io;
+        unsigned PY_LONG_LONG res;
+        PyNumberMethods *nb = ob->ob_type->tp_as_number;
+
+        if ((strict && CDataObject_Or_PyFloat_Check(ob)) ||
+                nb == NULL || nb->nb_int == NULL) {
+            PyErr_SetString(PyExc_TypeError, "an integer is required");
+            return (unsigned PY_LONG_LONG)-1;
+        }
+        io = (*nb->nb_int) (ob);
+        if (io == NULL)
+            return (unsigned PY_LONG_LONG)-1;
+
+        if (PyIntOrLong_Check(io)) {
+            res = _my_PyLong_AsUnsignedLongLong(io, strict);
+        }
+        else {
+            PyErr_SetString(PyExc_TypeError, "integer conversion failed");
+            res = (unsigned PY_LONG_LONG)-1;
+        }
+        Py_DECREF(io);
+        return res;
+    }
+
+ negative:
+    PyErr_SetString(PyExc_OverflowError,
+                    "can't convert negative number to unsigned");
+    return (unsigned PY_LONG_LONG)-1;
+}
+
+#define _read_raw_data(type)                    \
+    do {                                        \
+        if (size == sizeof(type)) {             \
+            type r;                             \
+            memcpy(&r, target, sizeof(type));   \
+            return r;                           \
+        }                                       \
+    } while(0)
+
+static PY_LONG_LONG
+read_raw_signed_data(char *target, int size)
+{
+    _read_raw_data(signed char);
+    _read_raw_data(short);
+    _read_raw_data(int);
+    _read_raw_data(long);
+    _read_raw_data(PY_LONG_LONG);
+    Py_FatalError("read_raw_signed_data: bad integer size");
+    return 0;
+}
+
+static unsigned PY_LONG_LONG
+read_raw_unsigned_data(char *target, int size)
+{
+    _read_raw_data(unsigned char);
+    _read_raw_data(unsigned short);
+    _read_raw_data(unsigned int);
+    _read_raw_data(unsigned long);
+    _read_raw_data(unsigned PY_LONG_LONG);
+    Py_FatalError("read_raw_unsigned_data: bad integer size");
+    return 0;
+}
+
+#define _write_raw_data(type)                           \
+    do {                                                \
+        if (size == sizeof(type)) {                     \
+            type r = (type)source;                      \
+            memcpy(target, &r, sizeof(type));           \
+            return;                                     \
+        }                                               \
+    } while(0)
+
+static void
+write_raw_integer_data(char *target, unsigned PY_LONG_LONG source, int size)
+{
+    _write_raw_data(unsigned char);
+    _write_raw_data(unsigned short);
+    _write_raw_data(unsigned int);
+    _write_raw_data(unsigned long);
+    _write_raw_data(unsigned PY_LONG_LONG);
+    Py_FatalError("write_raw_integer_data: bad integer size");
+}
+
+static double
+read_raw_float_data(char *target, int size)
+{
+    _read_raw_data(float);
+    _read_raw_data(double);
+    Py_FatalError("read_raw_float_data: bad float size");
+    return 0;
+}
+
+static long double
+read_raw_longdouble_data(char *target)
+{
+    int size = sizeof(long double);
+    _read_raw_data(long double);
+    Py_FatalError("read_raw_longdouble_data: bad long double size");
+    return 0;
+}
+
+static void
+write_raw_float_data(char *target, double source, int size)
+{
+    _write_raw_data(float);
+    _write_raw_data(double);
+    Py_FatalError("write_raw_float_data: bad float size");
+}
+
+static void
+write_raw_longdouble_data(char *target, long double source)
+{
+    int size = sizeof(long double);
+    _write_raw_data(long double);
+}
+
+static PyObject *
+new_simple_cdata(char *data, CTypeDescrObject *ct)
+{
+    CDataObject *cd = PyObject_New(CDataObject, &CData_Type);
+    if (cd == NULL)
+        return NULL;
+    Py_INCREF(ct);
+    cd->c_data = data;
+    cd->c_type = ct;
+    cd->c_weakreflist = NULL;
+    return (PyObject *)cd;
+}
+
+static CDataObject *_new_casted_primitive(CTypeDescrObject *ct)
+{
+    int dataoffset = offsetof(CDataObject_casted_primitive, alignment);
+    CDataObject *cd = (CDataObject *)PyObject_Malloc(dataoffset + ct->ct_size);
+    if (PyObject_Init((PyObject *)cd, &CData_Type) == NULL)
+        return NULL;
+    Py_INCREF(ct);
+    cd->c_type = ct;
+    cd->c_data = ((char*)cd) + dataoffset;
+    cd->c_weakreflist = NULL;
+    return cd;
+}
+
+//845
+static PyObject *convert_to_object(char *data, CTypeDescrObject *ct)
+{
+    if (!(ct->ct_flags & CT_PRIMITIVE_ANY)) {
+        /* non-primitive types (check done just for performance) */
+        if (ct->ct_flags & CT_POINTER) {
+            char *ptrdata = *(char **)data;
+            /*READ(data, sizeof(char *))*/
+            return new_simple_cdata(ptrdata, ct);
+        }
+        else if (ct->ct_flags & CT_ARRAY) {
+            if (ct->ct_length < 0) {
+                /* we can't return a <cdata 'int[]'> here, because we don't
+                   know the length to give it.  As a compromize, returns
+                   <cdata 'int *'> in this case. */
+                ct = (CTypeDescrObject *)ct->ct_stuff;
+            }
+            return new_simple_cdata(data, ct);
+        }
+        else if (ct->ct_size < 0) {
+            PyErr_Format(PyExc_TypeError, "cdata '%s' is opaque",
+                         ct->ct_name);
+            return NULL;
+        }
+        else if (ct->ct_flags & (CT_STRUCT|CT_UNION)) {
+            return new_simple_cdata(data, ct);
+        }
+    }
+    else if (ct->ct_flags & CT_PRIMITIVE_SIGNED) {
+        PY_LONG_LONG value;
+        /*READ(data, ct->ct_size)*/
+        value = read_raw_signed_data(data, ct->ct_size);
+        if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG)
+            return PyInt_FromLong((long)value);
+        else
+            return PyLong_FromLongLong(value);
+    }
+    else if (ct->ct_flags & CT_PRIMITIVE_UNSIGNED) {
+        unsigned PY_LONG_LONG value;
+        /*READ(data, ct->ct_size)*/
+        value = read_raw_unsigned_data(data, ct->ct_size);
+
+        if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG)
+            return PyInt_FromLong((long)value);
+        else
+            return PyLong_FromUnsignedLongLong(value);
+    }
+    else if (ct->ct_flags & CT_PRIMITIVE_FLOAT) {
+        /*READ(data, ct->ct_size)*/
+        if (!(ct->ct_flags & CT_IS_LONGDOUBLE)) {
+            double value = read_raw_float_data(data, ct->ct_size);
+            return PyFloat_FromDouble(value);
+        }
+        else {
+            long double value = read_raw_longdouble_data(data);
+            CDataObject *cd = _new_casted_primitive(ct);
+            if (cd != NULL)
+                write_raw_longdouble_data(cd->c_data, value);
+            return (PyObject *)cd;
+        }
+    }
+    else if (ct->ct_flags & CT_PRIMITIVE_CHAR) {
+        /*READ(data, ct->ct_size)*/
+        return PyBytes_FromStringAndSize(data, 1);
+    }
+
+    PyErr_Format(PyExc_SystemError,
+                 "convert_to_object: '%s'", ct->ct_name);
+    return NULL;
+}
+
+#if 0
+static PyObject *
+convert_to_object_bitfield(char *data, CFieldObject *cf)
+{
+    CTypeDescrObject *ct = cf->cf_type;
+    /*READ(data, ct->ct_size)*/
+
+    if (ct->ct_flags & CT_PRIMITIVE_SIGNED) {
+        unsigned PY_LONG_LONG value, valuemask, shiftforsign;
+        PY_LONG_LONG result;
+
+        value = (unsigned PY_LONG_LONG)read_raw_signed_data(data, ct->ct_size);
+        valuemask = (1ULL << cf->cf_bitsize) - 1ULL;
+        shiftforsign = 1ULL << (cf->cf_bitsize - 1);
+        value = ((value >> cf->cf_bitshift) + shiftforsign) & valuemask;
+        result = ((PY_LONG_LONG)value) - (PY_LONG_LONG)shiftforsign;
+
+        if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG)
+            return PyInt_FromLong((long)result);
+        else
+            return PyLong_FromLongLong(result);
+    }
+    else {
+        unsigned PY_LONG_LONG value, valuemask;
+
+        value = read_raw_unsigned_data(data, ct->ct_size);
+        valuemask = (1ULL << cf->cf_bitsize) - 1ULL;
+        value = (value >> cf->cf_bitshift) & valuemask;
+
+        if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG)
+            return PyInt_FromLong((long)value);
+        else
+            return PyLong_FromUnsignedLongLong(value);
+    }
+}
+#endif
+
+static int _convert_overflow(PyObject *init, const char *ct_name)
+{
+    PyObject *s;
+    if (PyErr_Occurred())   /* already an exception pending */
+        return -1;
+    s = PyObject_Str(init);
+    if (s == NULL)
+        return -1;
+    PyErr_Format(PyExc_OverflowError, "integer %s does not fit '%s'",
+                 PyText_AS_UTF8(s), ct_name);
+    Py_DECREF(s);
+    return -1;
+}
+
+static int _convert_to_char(PyObject *init)
+{
+    if (PyBytes_Check(init) && PyBytes_GET_SIZE(init) == 1) {
+        return (unsigned char)(PyBytes_AS_STRING(init)[0]);
+    }
+    if (CData_Check(init) &&
+           (((CDataObject *)init)->c_type->ct_flags & CT_PRIMITIVE_CHAR)) {
+        char *data = ((CDataObject *)init)->c_data;
+        /*READ(data, 1)*/
+        return *(unsigned char *)data;
+    }
+    PyErr_Format(PyExc_TypeError,
+                 "initializer for ctype 'char' must be a "STR_OR_BYTES
+                 " of length 1, not %.200s", Py_TYPE(init)->tp_name);
+    return -1;
+}
+
+//1009
+static int _convert_error(PyObject *init, const char *ct_name,
+                          const char *expected)
+{
+    if (CData_Check(init))
+        PyErr_Format(PyExc_TypeError,
+                     "initializer for ctype '%s' must be a %s, "
+                     "not cdata '%s'",
+                     ct_name, expected,
+                     ((CDataObject *)init)->c_type->ct_name);
+    else
+        PyErr_Format(PyExc_TypeError,
+                     "initializer for ctype '%s' must be a %s, "
+                     "not %.200s",
+                     ct_name, expected, Py_TYPE(init)->tp_name);
+    return -1;
+}
+
+//1031
+static Py_ssize_t get_new_array_length(PyObject **pvalue)
+{
+    PyObject *value = *pvalue;
+
+    if (PyList_Check(value) || PyTuple_Check(value)) {
+        return PySequence_Fast_GET_SIZE(value);
+    }
+    else if (PyBytes_Check(value)) {
+        /* from a string, we add the null terminator */
+        return PyBytes_GET_SIZE(value) + 1;
+    }
+    else {
+        Py_ssize_t explicitlength;
+        explicitlength = PyNumber_AsSsize_t(value, PyExc_OverflowError);
+        if (explicitlength < 0) {
+            if (!PyErr_Occurred())
+                PyErr_SetString(PyExc_ValueError, "negative array length");
+            return -1;
+        }
+        *pvalue = Py_None;
+        return explicitlength;
+    }
+}
+
+//1061
+static int convert_field_from_object(char *data, CFieldObject *cf,
+                                     PyObject *value)
+{
+    data += cf->cf_offset;
+    if (cf->cf_bitshift >= 0)
+        return convert_from_object_bitfield(data, cf, value);
+    else
+        return convert_from_object(data, cf->cf_type, value);
+}
+
+//1071
+static int convert_vfield_from_object(char *data, CFieldObject *cf,
+                                      PyObject *value, Py_ssize_t *optvarsize)
+{
+    /* a special case for var-sized C99 arrays */
+    if ((cf->cf_type->ct_flags & CT_ARRAY) && cf->cf_type->ct_size < 0) {
+        Py_ssize_t varsizelength = get_new_array_length(&value);
+        if (varsizelength < 0)
+            return -1;
+        if (optvarsize != NULL) {
+            /* in this mode, the only purpose of this function is to compute
+               the real size of the structure from a var-sized C99 array */
+            Py_ssize_t size, itemsize;
+            assert(data == NULL);
+            itemsize = cf->cf_type->ct_itemdescr->ct_size;
+            size = cf->cf_offset + itemsize * varsizelength;
+            if (size < 0 ||
+                ((size - cf->cf_offset) / itemsize) != varsizelength) {
+                PyErr_SetString(PyExc_OverflowError,
+                                "array size would overflow a Py_ssize_t");
+                return -1;
+            }
+            if (size > *optvarsize)
+                *optvarsize = size;
+            return 0;
+        }
+        /* if 'value' was only an integer, get_new_array_length() returns
+           it and convert 'value' to be None.  Detect if this was the case,
+           and if so, stop here, leaving the content uninitialized
+           (it should be zero-initialized from somewhere else). */
+        if (value == Py_None)
+            return 0;
+    }
+    if (optvarsize == NULL)
+        return convert_field_from_object(data, cf, value);
+    else
+        return 0;
+}
+
+//1110
+static int convert_array_from_object(char *data, CTypeDescrObject *ct,
+                                     PyObject *init)
+{
+    /* used by convert_from_object(), and also to decode lists/tuples/unicodes
+       passed as function arguments.  'ct' is an CT_ARRAY in the first case
+       and a CT_POINTER in the second case. */
+    const char *expected;
+    CTypeDescrObject *ctitem = ct->ct_itemdescr;
+
+    if (PyList_Check(init) || PyTuple_Check(init)) {
+        PyObject **items;
+        Py_ssize_t i, n;
+        n = PySequence_Fast_GET_SIZE(init);
+        if (ct->ct_length >= 0 && n > ct->ct_length) {
+            PyErr_Format(PyExc_IndexError,
+                         "too many initializers for '%s' (got %zd)",
+                         ct->ct_name, n);
+            return -1;
+        }
+        items = PySequence_Fast_ITEMS(init);
+        for (i=0; i<n; i++) {
+            if (convert_from_object(data, ctitem, items[i]) < 0)
+                return -1;
+            data += ctitem->ct_size;
+        }
+        return 0;
+    }
+    else if ((ctitem->ct_flags & (CT_PRIMITIVE_CHAR |
+                                  CT_PRIMITIVE_SIGNED |
+                                  CT_PRIMITIVE_UNSIGNED))
+             && (ctitem->ct_size == sizeof(char))) {
+        char *srcdata;
+        Py_ssize_t n;
+        if (!PyBytes_Check(init)) {
+            expected = STR_OR_BYTES" or list or tuple";
+            goto cannot_convert;
+        }
+        n = PyBytes_GET_SIZE(init);
+        if (ct->ct_length >= 0 && n > ct->ct_length) {
+            PyErr_Format(PyExc_IndexError,
+                         "initializer "STR_OR_BYTES" is too long for '%s' "
+                         "(got %zd characters)", ct->ct_name, n);
+            return -1;
+        }
+        if (n != ct->ct_length)
+            n++;
+        srcdata = PyBytes_AS_STRING(init);
+        memcpy(data, srcdata, n);
+        return 0;
+    }
+    else {
+        expected = "list or tuple";
+        goto cannot_convert;
+    }
+
+ cannot_convert:
+    return _convert_error(init, ct->ct_name, expected);
+}
+
+//1190
+static int convert_struct_from_object(char *data, CTypeDescrObject *ct,
+                                      PyObject *init, Py_ssize_t *optvarsize)
+{
+    const char *expected;
+
+    if (ct->ct_flags & CT_UNION) {
+        Py_ssize_t n = PyObject_Size(init);
+        if (n < 0)
+            return -1;
+        if (n > 1) {
+            PyErr_Format(PyExc_ValueError,
+                         "initializer for '%s': %zd items given, but "
+                         "only one supported (use a dict if needed)",
+                         ct->ct_name, n);
+            return -1;
+        }
+    }
+    if (PyList_Check(init) || PyTuple_Check(init)) {
+        PyObject **items = PySequence_Fast_ITEMS(init);
+        Py_ssize_t i, n = PySequence_Fast_GET_SIZE(init);
+        CFieldObject *cf = (CFieldObject *)ct->ct_extra;
+
+        for (i=0; i<n; i++) {
+            if (cf == NULL) {
+                PyErr_Format(PyExc_ValueError,
+                             "too many initializers for '%s' (got %zd)",
+                             ct->ct_name, n);
+                return -1;
+            }
+            if (convert_vfield_from_object(data, cf, items[i], optvarsize) < 0)
+                return -1;
+            cf = cf->cf_next;
+        }
+        return 0;
+    }
+    if (PyDict_Check(init)) {
+        PyObject *d_key, *d_value;
+        Py_ssize_t i = 0;
+        CFieldObject *cf;
+
+        while (PyDict_Next(init, &i, &d_key, &d_value)) {
+            cf = (CFieldObject *)PyDict_GetItem(ct->ct_stuff, d_key);
+            if (cf == NULL) {
+                PyErr_SetObject(PyExc_KeyError, d_key);
+                return -1;
+            }
+            if (convert_vfield_from_object(data, cf, d_value, optvarsize) < 0)
+                return -1;
+        }
+        return 0;
+    }
+    expected = optvarsize == NULL ? "list or tuple or dict or struct-cdata"
+                                  : "list or tuple or dict";
+    return _convert_error(init, ct->ct_name, expected);
+}
+
+#ifdef __GNUC__
+# if __GNUC__ >= 4
+/* Don't go inlining this huge function.  Needed because occasionally
+   it gets inlined in places where is causes a warning: call to
+   __builtin___memcpy_chk will always overflow destination buffer
+   (which is places where the 'ct' should never represent such a large
+   primitive type anyway). */
+__attribute__((noinline))
+# endif
+#endif
+//1257
+static int convert_from_object(char *data, CTypeDescrObject *ct, PyObject *init)
+{
+    const char *expected;
+    char buf[sizeof(PY_LONG_LONG)];
+
+    /*if (ct->ct_size > 0)*/
+        /*WRITE(data, ct->ct_size)*/
+
+    if (ct->ct_flags & CT_ARRAY) {
+        return convert_array_from_object(data, ct, init);
+    }
+    if (ct->ct_flags & CT_POINTER) {
+        char *ptrdata;
+        CTypeDescrObject *ctinit;
+
+        if (!CData_Check(init)) {
+            expected = "cdata pointer";
+            goto cannot_convert;
+        }
+        ctinit = ((CDataObject *)init)->c_type;
+        if (!(ctinit->ct_flags & CT_POINTER)) {
+            if (ctinit->ct_flags & CT_ARRAY)
+                ctinit = (CTypeDescrObject *)ctinit->ct_stuff;
+            else {
+                expected = "pointer or array";
+                goto cannot_convert;
+            }
+        }
+        if (ctinit != ct) {
+            if ((ct->ct_flags & CT_CAST_ANYTHING) ||
+                (ctinit->ct_flags & CT_CAST_ANYTHING))
+                ;   /* accept void* or char* as either source or target */
+            else {
+                expected = "pointer to same type";
+                goto cannot_convert;
+            }
+        }
+        ptrdata = ((CDataObject *)init)->c_data;
+
+        *(char **)data = ptrdata;
+        return 0;
+    }
+    if (ct->ct_flags & CT_PRIMITIVE_SIGNED) {
+        PY_LONG_LONG value = _my_PyLong_AsLongLong(init);
+        if (value == -1 && PyErr_Occurred())
+            return -1;
+        write_raw_integer_data(buf, value, ct->ct_size);
+        if (value != read_raw_signed_data(buf, ct->ct_size))
+            goto overflow;
+        write_raw_integer_data(data, value, ct->ct_size);
+        return 0;
+    }
+    if (ct->ct_flags & CT_PRIMITIVE_UNSIGNED) {
+        unsigned PY_LONG_LONG value = _my_PyLong_AsUnsignedLongLong(init, 1);
+        if (value == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())
+            return -1;
+        if (ct->ct_flags & CT_IS_BOOL)
+            if (value & ~1)      /* value != 0 && value != 1 */
+                goto overflow;
+        write_raw_integer_data(buf, value, ct->ct_size);
+        if (value != read_raw_unsigned_data(buf, ct->ct_size))
+            goto overflow;
+        write_raw_integer_data(data, value, ct->ct_size);
+        return 0;
+    }
+    if (ct->ct_flags & CT_PRIMITIVE_FLOAT) {
+        double value;
+        if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
+                CData_Check(init) &&
+                (((CDataObject *)init)->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
+            long double lvalue;
+            char *initdata = ((CDataObject *)init)->c_data;
+            /*READ(initdata, sizeof(long double))*/
+            lvalue = read_raw_longdouble_data(initdata);
+            write_raw_longdouble_data(data, lvalue);
+            return 0;
+        }
+        value = PyFloat_AsDouble(init);
+        if (value == -1.0 && PyErr_Occurred())
+            return -1;
+        if (!(ct->ct_flags & CT_IS_LONGDOUBLE))
+            write_raw_float_data(data, value, ct->ct_size);
+        else
+            write_raw_longdouble_data(data, (long double)value);
+        return 0;
+    }
+    if (ct->ct_flags & CT_PRIMITIVE_CHAR) {
+        int res;
+        assert(ct->ct_size == sizeof(char));
+        res = _convert_to_char(init);
+        if (res < 0)
+            return -1;
+        data[0] = res;
+        return 0;
+    }
+    if (ct->ct_flags & (CT_STRUCT|CT_UNION)) {
+
+        if (CData_Check(init)) {
+            if (((CDataObject *)init)->c_type == ct && ct->ct_size >= 0) {
+                memcpy(data, ((CDataObject *)init)->c_data, ct->ct_size);
+                return 0;
+            }
+        }
+        return convert_struct_from_object(data, ct, init, NULL);
+    }
+    PyErr_Format(PyExc_SystemError,
+                 "convert_from_object: '%s'", ct->ct_name);
+    return -1;
+
+ overflow:
+    return _convert_overflow(init, ct->ct_name);
+
+ cannot_convert:
+    return _convert_error(init, ct->ct_name, expected);
+}
+
+//1383
+static int convert_from_object_bitfield(char *data, CFieldObject *cf,
+                                        PyObject *init)
+{
+    CTypeDescrObject *ct = cf->cf_type;
+    PY_LONG_LONG fmin, fmax, value = PyLong_AsLongLong(init);
+    unsigned PY_LONG_LONG rawfielddata, rawvalue, rawmask;
+    if (value == -1 && PyErr_Occurred())
+        return -1;
+
+    if (ct->ct_flags & CT_PRIMITIVE_SIGNED) {
+        fmin = -(1LL << (cf->cf_bitsize-1));
+        fmax = (1LL << (cf->cf_bitsize-1)) - 1LL;
+        if (fmax == 0)
+            fmax = 1;    /* special case to let "int x:1" receive "1" */
+    }
+    else {
+        fmin = 0LL;
+        fmax = (PY_LONG_LONG)((1ULL << cf->cf_bitsize) - 1ULL);
+    }
+    if (value < fmin || value > fmax) {
+        /* phew, PyErr_Format does not support "%lld" in Python 2.6 */
+        PyObject *svalue = NULL, *sfmin = NULL, *sfmax = NULL;
+        PyObject *lfmin = NULL, *lfmax = NULL;
+        svalue = PyObject_Str(init);
+        if (svalue == NULL) goto skip;
+        lfmin = PyLong_FromLongLong(fmin);
+        if (lfmin == NULL) goto skip;
+        sfmin = PyObject_Str(lfmin);
+        if (sfmin == NULL) goto skip;
+        lfmax = PyLong_FromLongLong(fmax);
+        if (lfmax == NULL) goto skip;
+        sfmax = PyObject_Str(lfmax);
+        if (sfmax == NULL) goto skip;
+        PyErr_Format(PyExc_OverflowError,
+                     "value %s outside the range allowed by the "
+                     "bit field width: %s <= x <= %s",
+                     PyText_AS_UTF8(svalue),
+                     PyText_AS_UTF8(sfmin),
+                     PyText_AS_UTF8(sfmax));
+       skip:
+        Py_XDECREF(svalue);
+        Py_XDECREF(sfmin);
+        Py_XDECREF(sfmax);
+        Py_XDECREF(lfmin);
+        Py_XDECREF(lfmax);
+        return -1;
+    }
+
+    rawmask = ((1ULL << cf->cf_bitsize) - 1ULL) << cf->cf_bitshift;
+    rawvalue = ((unsigned PY_LONG_LONG)value) << cf->cf_bitshift;
+    /*WRITE(data, ct->ct_size)*/
+    rawfielddata = read_raw_unsigned_data(data, ct->ct_size);
+    rawfielddata = (rawfielddata & ~rawmask) | (rawvalue & rawmask);
+    write_raw_integer_data(data, rawfielddata, ct->ct_size);
+    return 0;
+}
+
+//1440
 static Py_ssize_t get_array_length(CDataObject *cd)
 {
     if (cd->c_type->ct_length < 0)
@@ -49,6 +816,7 @@
 }
 
 
+//1480
 static void cdata_dealloc(CDataObject *cd)
 {
     if (cd->c_weakreflist != NULL)
@@ -60,6 +828,7 @@
 #endif
 }
 
+//1491
 static void cdataowninggc_dealloc(CDataObject *cd)
 {
     abort();
@@ -83,6 +852,7 @@
 #endif
 }
 
+//1534
 static int cdataowninggc_traverse(CDataObject *cd, visitproc visit, void *arg)
 {
     abort();
@@ -100,6 +870,7 @@
 #endif
 }
 
+//1548
 static int cdataowninggc_clear(CDataObject *cd)
 {
     abort();
@@ -120,6 +891,7 @@
 #endif
 }
 
+//1599
 static PyObject *cdata_repr(CDataObject *cd)
 {
     char *extra;
@@ -174,6 +946,7 @@
     return result;
 }
 
+//1650
 static PyObject *_cdata_repr2(CDataObject *cd, char *text, PyObject *x)
 {
     PyObject *res, *s = PyObject_Repr(x);
@@ -185,6 +958,7 @@
     return res;
 }
 
+//1661
 static PyObject *cdataowning_repr(CDataObject *cd)
 {
     Py_ssize_t size;
@@ -223,6 +997,253 @@
     }
 }
 
+//1815
+static Py_ssize_t cdata_length(CDataObject *cd)
+{
+    if (cd->c_type->ct_flags & CT_ARRAY) {
+        return get_array_length(cd);
+    }
+    PyErr_Format(PyExc_TypeError, "cdata of type '%s' has no len()",
+                 cd->c_type->ct_name);
+    return -1;
+}
+
+//1826
+static char *_cdata_get_indexed_ptr(CDataObject *cd, PyObject *key)
+{
+    Py_ssize_t i = PyNumber_AsSsize_t(key, PyExc_IndexError);
+    if (i == -1 && PyErr_Occurred())
+        return NULL;
+
+    if (cd->c_type->ct_flags & CT_POINTER) {
+        if (CDataOwn_Check(cd) && i != 0) {
+            PyErr_Format(PyExc_IndexError,
+                         "cdata '%s' can only be indexed by 0",
+                         cd->c_type->ct_name);
+            return NULL;
+        }
+    }
+    else if (cd->c_type->ct_flags & CT_ARRAY) {
+        if (i < 0) {
+            PyErr_SetString(PyExc_IndexError,
+                            "negative index not supported");
+            return NULL;
+        }
+        if (i >= get_array_length(cd)) {
+            PyErr_Format(PyExc_IndexError,
+                         "index too large for cdata '%s' (expected %zd < %zd)",
+                         cd->c_type->ct_name,
+                         i, get_array_length(cd));
+            return NULL;
+        }
+    }
+    else {
+        PyErr_Format(PyExc_TypeError, "cdata of type '%s' cannot be indexed",
+                     cd->c_type->ct_name);
+        return NULL;
+    }
+    return cd->c_data + i * cd->c_type->ct_itemdescr->ct_size;
+}
+
+//1866
+static CTypeDescrObject *
+_cdata_getslicearg(CDataObject *cd, PySliceObject *slice, Py_ssize_t bounds[])
+{
+    Py_ssize_t start, stop;
+    CTypeDescrObject *ct;
+
+    start = PyInt_AsSsize_t(slice->start);
+    if (start == -1 && PyErr_Occurred()) {
+        if (slice->start == Py_None)
+            PyErr_SetString(PyExc_IndexError, "slice start must be specified");
+        return NULL;
+    }
+    stop = PyInt_AsSsize_t(slice->stop);
+    if (stop == -1 && PyErr_Occurred()) {
+        if (slice->stop == Py_None)
+            PyErr_SetString(PyExc_IndexError, "slice stop must be specified");
+        return NULL;
+    }
+    if (slice->step != Py_None) {
+        PyErr_SetString(PyExc_IndexError, "slice with step not supported");
+        return NULL;
+    }
+    if (start > stop) {
+        PyErr_SetString(PyExc_IndexError, "slice start > stop");
+        return NULL;
+    }
+
+    ct = cd->c_type;
+    if (ct->ct_flags & CT_ARRAY) {
+        if (start < 0) {
+            PyErr_SetString(PyExc_IndexError,
+                            "negative index not supported");
+            return NULL;
+        }
+        if (stop > get_array_length(cd)) {
+            PyErr_Format(PyExc_IndexError,
+                         "index too large (expected %zd <= %zd)",
+                         stop, get_array_length(cd));
+            return NULL;
+        }
+        ct = (CTypeDescrObject *)ct->ct_stuff;
+    }
+    else if (!(ct->ct_flags & CT_POINTER)) {
+        PyErr_Format(PyExc_TypeError, "cdata of type '%s' cannot be indexed",
+                     ct->ct_name);
+        return NULL;
+    }
+
+    bounds[0] = start;
+    bounds[1] = stop - start;
+    return ct;
+}
+
+//1919
+static PyObject *cdata_slice(CDataObject *cd, PySliceObject *slice)
+{
+    Py_ssize_t bounds[2];
+    CDataObject_own_length *scd;
+    CTypeDescrObject *ct = _cdata_getslicearg(cd, slice, bounds);
+    if (ct == NULL)
+        return NULL;
+
+    if (ct->ct_stuff == NULL) {
+        if (fill_array_type(ct) < 0)
+            return NULL;
+        assert(ct->ct_stuff != NULL);
+    }
+    ct = (CTypeDescrObject *)ct->ct_stuff;
+
+    scd = (CDataObject_own_length *)PyObject_Malloc(
+              offsetof(CDataObject_own_length, alignment));
+    if (PyObject_Init((PyObject *)scd, &CData_Type) == NULL)
+        return NULL;
+    Py_INCREF(ct);
+    scd->head.c_type = ct;
+    scd->head.c_data = cd->c_data + ct->ct_itemdescr->ct_size * bounds[0];
+    scd->head.c_weakreflist = NULL;
+    scd->length = bounds[1];
+    return (PyObject *)scd;
+}
+
+//1947
+static int cdata_ass_slice(CDataObject *cd, PySliceObject *slice, PyObject *v)
+{
+    Py_ssize_t bounds[2], i, length, itemsize;
+    PyObject *it, *item;
+    PyObject *(*iternext)(PyObject *);
+    char *cdata;
+    int err;
+    CTypeDescrObject *ct = _cdata_getslicearg(cd, slice, bounds);
+    if (ct == NULL)
+        return -1;
+    ct = ct->ct_itemdescr;
+    itemsize = ct->ct_size;
+    cdata = cd->c_data + itemsize * bounds[0];
+    length = bounds[1];
+
+    if (CData_Check(v)) {
+        CTypeDescrObject *ctv = ((CDataObject *)v)->c_type;
+        if ((ctv->ct_flags & CT_ARRAY) && (ctv->ct_itemdescr == ct) &&
+            (get_array_length((CDataObject *)v) == length)) {
+            /* fast path: copying from exactly the correct type */
+            memmove(cdata, ((CDataObject *)v)->c_data, itemsize * length);
+            return 0;
+        }
+    }
+
+    /* A fast path for <char[]>[0:N] = b"somestring", which also adds
+       support for Python 3: otherwise, you get integers while enumerating
+       the string, and you can't set them to characters :-/
+    */
+    if (PyBytes_Check(v) && (ct->ct_flags & CT_PRIMITIVE_CHAR)) {
+        if (PyBytes_GET_SIZE(v) != length) {
+            PyErr_Format(PyExc_ValueError,
+                         "need a string of length %zd, got %zd",
+                         length, PyBytes_GET_SIZE(v));
+            return -1;
+        }
+        memcpy(cdata, PyBytes_AS_STRING(v), length);
+        return 0;
+    }
+
+    it = PyObject_GetIter(v);
+    if (it == NULL)
+        return -1;
+    iternext = *it->ob_type->tp_iternext;
+
+    for (i = 0; i < length; i++) {
+        item = iternext(it);
+        if (item == NULL) {
+            if (!PyErr_Occurred())
+                PyErr_Format(PyExc_ValueError,
+                             "need %zd values to unpack, got %zd",
+                             length, i);
+            goto error;
+        }
+        err = convert_from_object(cdata, ct, item);
+        Py_DECREF(item);
+        if (err < 0)
+            goto error;
+
+        cdata += itemsize;
+    }
+    item = iternext(it);
+    if (item != NULL) {
+        Py_DECREF(item);
+        PyErr_Format(PyExc_ValueError,
+                     "got more than %zd values to unpack", length);
+    }
+ error:
+    Py_DECREF(it);
+    return PyErr_Occurred() ? -1 : 0;
+}
+
+//2044
+static PyObject *cdata_subscript(CDataObject *cd, PyObject *key)
+{
+    char *c;
+    if (PySlice_Check(key))
+        return cdata_slice(cd, (PySliceObject *)key);
+
+    c = _cdata_get_indexed_ptr(cd, key);
+    /* use 'mp_subscript' instead of 'sq_item' because we don't want
+       negative indexes to be corrected automatically */
+    if (c == NULL && PyErr_Occurred())
+        return NULL;
+    return convert_to_object(c, cd->c_type->ct_itemdescr);
+}
+
+//2059
+static int cdata_ass_sub(CDataObject *cd, PyObject *key, PyObject *v)
+{
+    char *c;
+    CTypeDescrObject *ctitem;
+    if (PySlice_Check(key))
+        return cdata_ass_slice(cd, (PySliceObject *)key, v);
+
+    c = _cdata_get_indexed_ptr(cd, key);
+    ctitem = cd->c_type->ct_itemdescr;
+    /* use 'mp_ass_subscript' instead of 'sq_ass_item' because we don't want
+       negative indexes to be corrected automatically */
+    if (c == NULL && PyErr_Occurred())
+        return -1;
+    if (v == NULL) {
+        PyErr_SetString(PyExc_TypeError,
+                        "'del x[n]' not supported for cdata objects");
+        return -1;
+    }
+    return convert_from_object(c, ctitem, v);
+}
+
+
+static PyMappingMethods CData_as_mapping = {
+    (lenfunc)cdata_length, /*mp_length*/
+    (binaryfunc)cdata_subscript, /*mp_subscript*/
+    (objobjargproc)cdata_ass_sub, /*mp_ass_subscript*/
+};
+
 static PyTypeObject CData_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "zeffir.CData",
@@ -236,7 +1257,7 @@
     (reprfunc)cdata_repr,                       /* tp_repr */
     0,//&CData_as_number,                           /* tp_as_number */
     0,                                          /* tp_as_sequence */
-    0,//&CData_as_mapping,                          /* tp_as_mapping */
+    &CData_as_mapping,                          /* tp_as_mapping */
     0,//(hashfunc)cdata_hash,                       /* tp_hash */
     0,//(ternaryfunc)cdata_call,                    /* tp_call */
     0,                                          /* tp_str */
diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -30,13 +30,12 @@
 
     CTypeDescrObject *ct_itemdescr;    /* ptrs and arrays: the item type */
     PyObject *ct_stuff;                /* structs: dict of the fields
-                                          //arrays: ctypedescr of the ptr type
+                                          arrays: ctypedescr of the ptr type
                                           function: tuple(abi, ctres, ctargs..)
                                           enum: pair {"name":x},{x:"name"}
                                           ptrs: lazily, ctypedescr of array */
-    //void *ct_extra;                    /* structs: first field (not a ref!)
-    //                                      function types: cif_description
-    //                                      primitives: prebuilt "cif" object */
+    void *ct_extra;                    /* structs: first field (not a ref!)
+                                          function types: cif_description */
 
     PyObject *ct_weakreflist;    /* weakref support */
 
@@ -75,7 +74,7 @@
 
 static PyObject *ctypedescr_repr(CTypeDescrObject *ct)
 {
-    return PyString_FromFormat("<ctype '%s'>", ct->ct_name);
+    return PyText_FromFormat("<ctype '%s'>", ct->ct_name);
 }
 
 static void ctypedescr_dealloc(CTypeDescrObject *ct)
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -121,12 +121,12 @@
     /* Returns the CTypeDescrObject from the user-supplied 'arg'.
        Does not return a new reference!
     */
-    if ((accept & ACCEPT_STRING) && PyString_Check(arg)) {
+    if ((accept & ACCEPT_STRING) && PyText_Check(arg)) {
         PyObject *x = PyDict_GetItem(ffi->types_dict, arg);
         if (x != NULL && CTypeDescr_Check(x))
             return (CTypeDescrObject *)x;
 
-        CTypeDescrObject *ct = parse_c_decl(ffi, PyString_AS_STRING(arg));
+        CTypeDescrObject *ct = parse_c_decl(ffi, PyText_AS_UTF8(arg));
         if (ct == NULL)
             return NULL;
 
diff --git a/zeffir/lib_obj.c b/zeffir/lib_obj.c
--- a/zeffir/lib_obj.c
+++ b/zeffir/lib_obj.c
@@ -20,9 +20,9 @@
 
 static PyObject *lib_repr(ZefLibObject *lib)
 {
-    return PyString_FromFormat("<zeffir.Lib object for '%.200s'%s>",
-                               lib->l_libname,
-                               lib->l_dl_lib == NULL ? " (closed)" : "");
+    return PyText_FromFormat("<zeffir.Lib object for '%.200s'%s>",
+                             lib->l_libname,
+                             lib->l_dl_lib == NULL ? " (closed)" : "");
 }
 
 static PyObject *lib_findattr(ZefLibObject *lib, PyObject *name)
@@ -40,7 +40,7 @@
                      "lib '%.200s' has no function,"
                      " global variable or constant '%.200s'",
                      lib->l_libname,
-                     PyString_Check(name) ? PyString_AS_STRING(name) : "?");
+                     PyText_Check(name) ? PyText_AS_UTF8(name) : "?");
         return NULL;
     }
     return x;
@@ -83,7 +83,7 @@
 
     PyErr_Format(PyExc_AttributeError,
                  "cannot write to function or constant '%.200s'",
-                 PyString_Check(name) ? PyString_AS_STRING(name) : "?");
+                 PyText_Check(name) ? PyText_AS_UTF8(name) : "?");
     return -1;
 }
 
@@ -152,7 +152,7 @@
         return NULL;
 
     lib->l_dl_lib = NULL;
-    lib->l_libname = PyString_AS_STRING(path);
+    lib->l_libname = PyString_AsString(path);
     Py_INCREF(path);
     lib->l_libname_obj = path;
     lib->l_dict = PyDict_New();
diff --git a/zeffir/py3support.h b/zeffir/py3support.h
new file mode 100644
--- /dev/null
+++ b/zeffir/py3support.h
@@ -0,0 +1,28 @@
+
+#if PY_MAJOR_VERSION >= 3
+# define STR_OR_BYTES "bytes"
+# define PyText_Type PyUnicode_Type
+# define PyText_Check PyUnicode_Check
+# define PyTextAny_Check PyUnicode_Check
+# define PyText_FromFormat PyUnicode_FromFormat
+# define PyText_AsUTF8 _PyUnicode_AsString   /* PyUnicode_AsUTF8 in Py3.3 */
+# define PyText_AS_UTF8 _PyUnicode_AsString
+# define PyText_GetSize PyUnicode_GetSize
+# define PyText_FromString PyUnicode_FromString
+# define PyText_FromStringAndSize PyUnicode_FromStringAndSize
+# define PyText_InternInPlace PyUnicode_InternInPlace
+# define PyIntOrLong_Check PyLong_Check
+#else
+# define STR_OR_BYTES "str"
+# define PyText_Type PyString_Type
+# define PyText_Check PyString_Check
+# define PyTextAny_Check(op) (PyString_Check(op) || PyUnicode_Check(op))
+# define PyText_FromFormat PyString_FromFormat
+# define PyText_AsUTF8 PyString_AsString
+# define PyText_AS_UTF8 PyString_AS_STRING
+# define PyText_GetSize PyString_Size
+# define PyText_FromString PyString_FromString
+# define PyText_FromStringAndSize PyString_FromStringAndSize
+# define PyText_InternInPlace PyString_InternInPlace
+# define PyIntOrLong_Check(op) (PyInt_Check(op) || PyLong_Check(op))
+#endif
diff --git a/zeffir/test/test_cdata.py b/zeffir/test/test_cdata.py
--- a/zeffir/test/test_cdata.py
+++ b/zeffir/test/test_cdata.py
@@ -6,3 +6,10 @@
     ffi = support.new_ffi()
     p = ffi.new("short *")
     assert repr(p) == "<cdata 'short *' owning 2 bytes>"
+
+def test_cdata_read_write():
+    ffi = support.new_ffi()
+    p = ffi.new("short *")
+    assert p[0] == 0
+    p[0] = 43
+    assert p[0] == 43
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -9,6 +9,7 @@
 /* Works by including all other .c files.                   */
 /* Allows all function and global symbols to remain static. */
 
+#include "py3support.h"
 #include "zeffir.h"
 #include "ctype.c"
 #include "cdata.c"
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -1,5 +1,6 @@
 
 typedef struct _crx_type_s CTypeDescrObject;
+typedef struct cfieldobject_s CFieldObject;
 typedef struct ZefLibObject_s ZefLibObject;
 typedef struct ZefFFIObject_s ZefFFIObject;
 
@@ -15,3 +16,8 @@
 static int lib_close(ZefLibObject *);
 static int load_creflect_main(ZefFFIObject *, ZefLibObject *);
 static CTypeDescrObject *parse_c_decl(ZefFFIObject *ffi, const char *str);
+
+static int convert_from_object(char *, CTypeDescrObject *, PyObject *);
+static int convert_from_object_bitfield(char *, CFieldObject *, PyObject *);
+
+static int fill_array_type(CTypeDescrObject *ctptr);

From noreply at buildbot.pypy.org  Fri Dec  5 12:41:04 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  5 Dec 2014 12:41:04 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: fix failing test
Message-ID: <20141205114104.CB3AF1D28F0@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r74831:35709774642a
Date: 2014-12-05 13:38 +0200
http://bitbucket.org/pypy/pypy/changeset/35709774642a/

Log:	fix failing test

diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -638,10 +638,8 @@
                 for i in range(len(allargs)):
                     _arg = allargs[i]
                     assert isinstance(_arg, W_NDimArray)
-                    start_dim = len(_arg.get_shape()) - len(iter_shape)
-                    assert start_dim >= 0
+                    start_dim = len(iter_shape)
                     steps += _arg.implementation.strides[start_dim:]
-                #print 'set_dims_and_steps with dims, steps',dims,steps
                 func.set_dims_and_steps(space, dims, steps)
             else:
                 # it is a function, ready to be called by the iterator,

From noreply at buildbot.pypy.org  Fri Dec  5 12:43:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 12:43:15 +0100 (CET)
Subject: [pypy-commit] creflect default: re-enable the XXX'ed parts of
	ffi.new()
Message-ID: <20141205114315.F1F4B1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r159:c6e3bdeb49c5
Date: 2014-12-05 12:35 +0100
http://bitbucket.org/cffi/creflect/changeset/c6e3bdeb49c5/

Log:	re-enable the XXX'ed parts of ffi.new()

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -349,8 +349,8 @@
     ct->ct_size = arraysize;
     ct->ct_length = length;
     ct->ct_flags = CT_ARRAY;
-    Py_INCREF(ctptr);
-    ct->ct_stuff = (PyObject *)ctptr;
+    ct->ct_itemdescr = ctitem;        Py_INCREF(ctitem);
+    ct->ct_stuff = (PyObject *)ctptr; Py_INCREF(ctptr);
 
     if (cb != NULL) {
         if (length < 0 && ctptr->ct_stuff == NULL) {
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -203,15 +203,13 @@
         if (ctitem->ct_flags & CT_PRIMITIVE_CHAR)
             datasize *= 2;   /* forcefully add another character: a null */
 
-        /* XXX
         if ((ctitem->ct_flags & CT_WITH_VAR_ARRAY) && init != Py_None) {
             Py_ssize_t optvarsize = datasize;
             if (convert_struct_from_object(NULL,ctitem, init, &optvarsize) < 0)
                 return NULL;
             datasize = optvarsize;
-        }*/
+        }
     }
-    /* XXX
     else if (ct->ct_flags & CT_ARRAY) {
         dataoffset = offsetof(CDataObject_own_nolength, alignment);
         datasize = ct->ct_size;
@@ -229,7 +227,7 @@
                 return NULL;
             }
         }
-    }*/
+    }
     else {
         PyErr_Format(PyExc_TypeError,
                      "expected a pointer or array ctype, got '%s'",
@@ -247,12 +245,11 @@
 
     memset(cd->c_data, 0, datasize);
     if (init != Py_None) {
-        /* XXX
         if (convert_from_object(cd->c_data,
               (ct->ct_flags & CT_POINTER) ? ct->ct_itemdescr : ct, init) < 0) {
             Py_DECREF(cd);
             return NULL;
-        }*/
+        }
     }
     return (PyObject *)cd;
 }
diff --git a/zeffir/test/test_cdata.py b/zeffir/test/test_cdata.py
--- a/zeffir/test/test_cdata.py
+++ b/zeffir/test/test_cdata.py
@@ -13,3 +13,15 @@
     assert p[0] == 0
     p[0] = 43
     assert p[0] == 43
+
+def test_cdata_array_fixed():
+    ffi = support.new_ffi()
+    p = ffi.new("int[5]")
+    assert p[4] == 0
+    p[4] = 43
+    assert p[4] == 43
+
+def test_cdata_array_incomplete():
+    ffi = support.new_ffi()
+    p = ffi.new("int[]", [10, 20, 30])
+    assert p[2] == 30

From noreply at buildbot.pypy.org  Fri Dec  5 12:43:17 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 12:43:17 +0100 (CET)
Subject: [pypy-commit] creflect default: the next tests to pass
Message-ID: <20141205114317.4824C1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r160:adbaf54b8357
Date: 2014-12-05 12:43 +0100
http://bitbucket.org/cffi/creflect/changeset/adbaf54b8357/

Log:	the next tests to pass

diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -48,7 +48,8 @@
     actualnargs = PyTuple_Size(args);
     if (actualnargs != nargs) {
         if (!PyErr_Occurred())
-            PyErr_Format(PyExc_TypeError, "'%s' expects %d arguments, got %zd",
+            PyErr_Format(PyExc_TypeError,
+                         "'%s()' expected %d arguments, but got %zd",
                          zfs->zfs_funcname, nargs, actualnargs);
         return NULL;
     }
diff --git a/zeffir/test/function.crx b/zeffir/test/function.crx
--- a/zeffir/test/function.crx
+++ b/zeffir/test/function.crx
@@ -3,9 +3,18 @@
     return x + 1;
 }
 
+int add_from_array(int array[], int count)
+{
+    int result = 0;
+    while (count > 0)
+        result += array[--count];
+    return result;
+}
+
 
 // CREFLECT: start
 
 int simple_function(int);
+int add_from_array(int array[], int count);
 
 // CREFLECT: end
diff --git a/zeffir/test/test_function.py b/zeffir/test/test_function.py
--- a/zeffir/test/test_function.py
+++ b/zeffir/test/test_function.py
@@ -1,3 +1,4 @@
+import py
 import support
 
 
@@ -6,3 +7,18 @@
     res = lib.simple_function(42)
     assert type(res) is int
     assert res == 43
+
+def test_function_with_pointer_arg():
+    py.test.skip("in-progress")
+    ffi, lib = support.compile_and_open('function')
+    p = ffi.new("int[]", [30, 2, 10])
+    res = lib.add_from_array(p, 3)
+    assert type(res) is int
+    assert res == 42
+
+def test_function_with_array_arg():
+    py.test.skip("in-progress")
+    ffi, lib = support.compile_and_open('function')
+    res = lib.add_from_array([30, 2, 10], 3)
+    assert type(res) is int
+    assert res == 42

From noreply at buildbot.pypy.org  Fri Dec  5 15:48:23 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 15:48:23 +0100 (CET)
Subject: [pypy-commit] creflect default: correct call argument types
Message-ID: <20141205144823.27CF21D29F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r161:081d71b47603
Date: 2014-12-05 15:43 +0100
http://bitbucket.org/cffi/creflect/changeset/081d71b47603/

Log:	correct call argument types

diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -5,12 +5,19 @@
     int                 zfs_nargs;
     _crx_trampoline0_fn zfs_trampl;
     PyMethodDef         zfs_md;
-    char                zfs_funcname[1];
+    size_t              zfs_size_args;
+    CTypeDescrObject   *zfs_ret;
+    CTypeDescrObject   *zfs_args[1];
 
 } ZefFuncSupportObject;
 
 static void zef_func_support_dealloc(ZefFuncSupportObject *zfs)
 {
+    int i;
+    for (i = zfs->zfs_nargs - 1; i >= 0; i--)
+        Py_DECREF(zfs->zfs_args[i]);
+    Py_DECREF(zfs->zfs_ret);
+
     PyObject_Del(zfs);
 }
 
@@ -37,33 +44,42 @@
     Py_TPFLAGS_DEFAULT,                         /* tp_flags */
 };
 
+#define ROUND_UP(n)   (((n) + 7) & ~7)
+
 static PyObject *zfs_call(PyObject *self, PyObject *args)
 {
     ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)self;
     int i, nargs = zfs->zfs_nargs;
     Py_ssize_t actualnargs;
     void *llargs[nargs];
-    int llinput[nargs], lloutput;   /* temp */
+    char *llbuffer;
+    size_t lloffset;
 
     actualnargs = PyTuple_Size(args);
     if (actualnargs != nargs) {
         if (!PyErr_Occurred())
             PyErr_Format(PyExc_TypeError,
                          "'%s()' expected %d arguments, but got %zd",
-                         zfs->zfs_funcname, nargs, actualnargs);
+                         zfs->zfs_md.ml_name, nargs, actualnargs);
         return NULL;
     }
 
+    llbuffer = alloca(zfs->zfs_size_args);
+    lloffset = zfs->zfs_ret->ct_size;
+
     for (i = 0; i < nargs; i++) {
-        llinput[i] = PyInt_AsLong(PyTuple_GET_ITEM(args, i));
-        if (PyErr_Occurred())
+        CTypeDescrObject *ct = zfs->zfs_args[i];
+        PyObject *x = PyTuple_GET_ITEM(args, i);
+        lloffset = ROUND_UP(lloffset);
+        llargs[i] = llbuffer + lloffset;
+        if (convert_from_object(llbuffer + lloffset, ct, x) < 0)
             return NULL;
-        llargs[i] = &llinput[i];
+        lloffset += ct->ct_size;
     }
 
-    zfs->zfs_trampl(llargs, &lloutput);
+    zfs->zfs_trampl(llargs, llbuffer);
 
-    return PyInt_FromLong(lloutput);
+    return convert_to_object(llbuffer, zfs->zfs_ret);
 }
 
 static PyObject *make_builtin_func(PyObject *libname_obj,
@@ -71,7 +87,11 @@
                                    _crx_qual_type args[], int nargs,
                                    _crx_trampoline0_fn trampl)
 {
-    size_t size = sizeof(ZefFuncSupportObject) + strlen(funcname);
+    int i;
+    char *p;
+    size_t size = (sizeof(ZefFuncSupportObject)
+                   + (nargs - 1) * sizeof(CTypeDescrObject *)
+                   + strlen(funcname) + 1);
     ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)PyObject_Malloc(size);
     if (zfs == NULL)
         return PyErr_NoMemory();
@@ -79,9 +99,26 @@
 
     zfs->zfs_nargs = nargs;
     zfs->zfs_trampl = trampl;
+    zfs->zfs_ret = ret;
+    Py_INCREF(ret);
+
+    assert(ret->ct_size >= 0);
+    size = ret->ct_size;
+
+    for (i = 0; i < nargs; i++) {
+        CTypeDescrObject *ct = args[i].type;
+        if (ct->ct_flags & CT_ARRAY)
+            ct = (CTypeDescrObject *)ct->ct_stuff;    /* array -> pointer */
+        Py_INCREF(ct);
+        zfs->zfs_args[i] = ct;
+        assert(ct->ct_size >= 0);
+        size = ROUND_UP(size) + ct->ct_size;
+    }
+    zfs->zfs_size_args = size;
+    p = (char *)(zfs->zfs_args + nargs);
 
     memset(&zfs->zfs_md, 0, sizeof(PyMethodDef));
-    zfs->zfs_md.ml_name = strcpy(zfs->zfs_funcname, funcname);
+    zfs->zfs_md.ml_name = strcpy(p, funcname);
     zfs->zfs_md.ml_meth = &zfs_call;
     zfs->zfs_md.ml_flags = METH_VARARGS;
     /*zfs->zfs_md.ml_doc = ... */
diff --git a/zeffir/test/test_function.py b/zeffir/test/test_function.py
--- a/zeffir/test/test_function.py
+++ b/zeffir/test/test_function.py
@@ -9,7 +9,6 @@
     assert res == 43
 
 def test_function_with_pointer_arg():
-    py.test.skip("in-progress")
     ffi, lib = support.compile_and_open('function')
     p = ffi.new("int[]", [30, 2, 10])
     res = lib.add_from_array(p, 3)

From noreply at buildbot.pypy.org  Fri Dec  5 15:48:24 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 15:48:24 +0100 (CET)
Subject: [pypy-commit] creflect default: more copying from _cffi_backend
Message-ID: <20141205144824.4DC9F1D29F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r162:157449ed6904
Date: 2014-12-05 15:48 +0100
http://bitbucket.org/cffi/creflect/changeset/157449ed6904/

Log:	more copying from _cffi_backend

diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -46,6 +46,67 @@
 
 #define ROUND_UP(n)   (((n) + 7) & ~7)
 
+static Py_ssize_t _prepare_pointer_call_argument(CTypeDescrObject *ctptr,
+                                                 PyObject *init,
+                                                 char **output_data)
+{
+    /* 'ctptr' is here a pointer type 'ITEM *'.  Accept as argument an
+       initializer for an array 'ITEM[]'.  This includes the case of
+       passing a Python byte string to a 'char *' argument.
+
+       This function returns -1 if an error occurred,
+       0 if conversion succeeded (into *output_data),
+       or N > 0 if conversion would require N bytes of storage.
+    */
+    Py_ssize_t length, datasize;
+    CTypeDescrObject *ctitem;
+
+    if (CData_Check(init))
+        goto convert_default;
+
+    ctitem = ctptr->ct_itemdescr;
+    /* XXX some code duplication, how to avoid it? */
+    if (PyBytes_Check(init)) {
+        /* from a string: just returning the string here is fine.
+           We assume that the C code won't modify the 'char *' data. */
+        if ((ctptr->ct_flags & CT_CAST_ANYTHING) ||
+            ((ctitem->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED))
+             && (ctitem->ct_size == sizeof(char)))) {
+#if defined(CFFI_MEM_DEBUG) || defined(CFFI_MEM_LEAK)
+            length = PyBytes_GET_SIZE(init) + 1;
+#else
+            *output_data = PyBytes_AS_STRING(init);
+            return 0;
+#endif
+        }
+        else
+            goto convert_default;
+    }
+    else if (PyList_Check(init) || PyTuple_Check(init)) {
+        length = PySequence_Fast_GET_SIZE(init);
+    }
+    else {
+        /* refuse to receive just an integer (and interpret it
+           as the array size) */
+        goto convert_default;
+    }
+
+    if (ctitem->ct_size <= 0)
+        goto convert_default;
+    datasize = length * ctitem->ct_size;
+    if ((datasize / ctitem->ct_size) != length) {
+        PyErr_SetString(PyExc_OverflowError,
+                        "array size would overflow a Py_ssize_t");
+        return -1;
+    }
+    if (datasize <= 0)
+        datasize = 1;
+    return datasize;
+
+ convert_default:
+    return convert_from_object((char *)output_data, ctptr, init);
+}
+
 static PyObject *zfs_call(PyObject *self, PyObject *args)
 {
     ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)self;
@@ -69,11 +130,34 @@
 
     for (i = 0; i < nargs; i++) {
         CTypeDescrObject *ct = zfs->zfs_args[i];
-        PyObject *x = PyTuple_GET_ITEM(args, i);
+        PyObject *obj = PyTuple_GET_ITEM(args, i);
+        char *data;
         lloffset = ROUND_UP(lloffset);
-        llargs[i] = llbuffer + lloffset;
-        if (convert_from_object(llbuffer + lloffset, ct, x) < 0)
-            return NULL;
+        data = llbuffer + lloffset;
+        llargs[i] = data;
+
+        if (ct->ct_flags & CT_POINTER) {
+            char *tmpbuf;
+            Py_ssize_t datasize = _prepare_pointer_call_argument(
+                                            ct, obj, (char **)data);
+            if (datasize == 0) {
+                /* successfully filled '*data' */
+            }
+            else if (datasize < 0) {
+                return NULL;
+            }
+            else {
+                tmpbuf = alloca(datasize);
+                memset(tmpbuf, 0, datasize);
+                *(char **)data = tmpbuf;
+                if (convert_array_from_object(tmpbuf, ct, obj) < 0)
+                    return NULL;
+            }
+        }
+        else {
+            if (convert_from_object(data, ct, obj) < 0)
+                return NULL;
+        }
         lloffset += ct->ct_size;
     }
 
diff --git a/zeffir/test/test_function.py b/zeffir/test/test_function.py
--- a/zeffir/test/test_function.py
+++ b/zeffir/test/test_function.py
@@ -16,7 +16,6 @@
     assert res == 42
 
 def test_function_with_array_arg():
-    py.test.skip("in-progress")
     ffi, lib = support.compile_and_open('function')
     res = lib.add_from_array([30, 2, 10], 3)
     assert type(res) is int

From noreply at buildbot.pypy.org  Fri Dec  5 16:22:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 16:22:13 +0100 (CET)
Subject: [pypy-commit] creflect default: globals of all types
Message-ID: <20141205152213.15C0F1D2A0C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r163:1c10444e58e6
Date: 2014-12-05 15:54 +0100
http://bitbucket.org/cffi/creflect/changeset/1c10444e58e6/

Log:	globals of all types

diff --git a/zeffir/cglob.c b/zeffir/cglob.c
--- a/zeffir/cglob.c
+++ b/zeffir/cglob.c
@@ -51,3 +51,13 @@
     assert(addr != NULL);
     return (PyObject *)zgs;
 }
+
+static PyObject *read_global_var(ZefGlobSupportObject *zgs)
+{
+    return convert_to_object(zgs->zgs_data, zgs->zgs_type);
+}
+
+static int write_global_var(ZefGlobSupportObject *zgs, PyObject *obj)
+{
+    return convert_from_object(zgs->zgs_data, zgs->zgs_type, obj);
+}
diff --git a/zeffir/lib_obj.c b/zeffir/lib_obj.c
--- a/zeffir/lib_obj.c
+++ b/zeffir/lib_obj.c
@@ -53,8 +53,7 @@
         return NULL;
 
     if (ZefGlobSupport_Check(x)) {
-        // XXX!
-        return PyInt_FromLong(*(int *)(((ZefGlobSupportObject *)x)->zgs_data));
+        return read_global_var((ZefGlobSupportObject *)x);
     }
     Py_INCREF(x);
     return x;
@@ -73,12 +72,7 @@
     }
 
     if (ZefGlobSupport_Check(x)) {
-        // XXX!
-        long y = PyInt_AsLong(val);
-        if (PyErr_Occurred())
-            return -1;
-        *(int *)(((ZefGlobSupportObject *)x)->zgs_data) = y;
-        return 0;
+        return write_global_var((ZefGlobSupportObject *)x, val);
     }
 
     PyErr_Format(PyExc_AttributeError,
diff --git a/zeffir/test/global.crx b/zeffir/test/global.crx
--- a/zeffir/test/global.crx
+++ b/zeffir/test/global.crx
@@ -1,8 +1,10 @@
 short myglob = 42;
+short *myglobptr = &myglob;
 
 
 // CREFLECT: start
 
 short myglob;
+short *myglobptr;
 
 // CREFLECT: end
diff --git a/zeffir/test/test_global.py b/zeffir/test/test_global.py
--- a/zeffir/test/test_global.py
+++ b/zeffir/test/test_global.py
@@ -13,5 +13,14 @@
     assert type(res) is int
     assert res == -43
     #
-    #py.test.raises(OverflowError, setattr, lib, 'myglob', 40000)
-    #py.test.raises(OverflowError, setattr, lib, 'myglob', -40000)
+    py.test.raises(OverflowError, setattr, lib, 'myglob', 40000)
+    py.test.raises(OverflowError, setattr, lib, 'myglob', -40000)
+
+def test_ptr_global():
+    ffi, lib = support.compile_and_open('global')
+    p = lib.myglobptr
+    assert p[0] == 42
+    #
+    q = ffi.new("short *", -555)
+    lib.myglobptr = q
+    assert lib.myglobptr[0] == -555

From noreply at buildbot.pypy.org  Fri Dec  5 16:22:16 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 16:22:16 +0100 (CET)
Subject: [pypy-commit] creflect default: simple structs work
Message-ID: <20141205152216.B66731D2E18@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r164:fca0f1dca466
Date: 2014-12-05 16:22 +0100
http://bitbucket.org/cffi/creflect/changeset/fca0f1dca466/

Log:	simple structs work

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -388,14 +388,42 @@
     return _zef_array_type(cb, ctitem, (size_t)-1, NULL);
 }
 
+static _crx_type_t *_zef_struct_or_union(_crx_builder_t *cb, const char *name,
+                                         int flag)
+{
+    if (PyErr_Occurred())
+        return NULL;
+
+    PyObject *name_obj;
+    CTypeDescrObject *ct;
+
+    name_obj = PyString_FromString(name);
+    if (name_obj == NULL)
+        return NULL;
+
+    ct = get_cached_type(cb, name_obj);
+    if (ct && (ct->ct_flags == flag))
+        goto done;
+
+    ct = ctypedescr_new(name_obj, PyString_GET_SIZE(name_obj));
+    if (ct == NULL)
+        goto done;
+
+    ct->ct_flags = flag;
+
+ done:
+    Py_DECREF(name_obj);
+    return ct;
+}
+
 static _crx_type_t *zef_get_struct_type(_crx_builder_t *cb, const char *name)
 {
-    abort();
+    return _zef_struct_or_union(cb, name, CT_STRUCT);
 }
 
 static _crx_type_t *zef_get_union_type(_crx_builder_t *cb, const char *name)
 {
-    abort();
+    return _zef_struct_or_union(cb, name, CT_UNION);
 }
 
 static _crx_type_t *zef_get_enum_type(_crx_builder_t *cb, const char *name)
@@ -426,11 +454,258 @@
     return _zef_primitive(cb, -1, name, CT_UNKNOWN);
 }
 
-static void zef_complete(_crx_builder_t *cb, _crx_type_t *t,
+static CFieldObject *_add_field(PyObject *interned_fields,
+                                const char *fname, CTypeDescrObject *ftype,
+                                Py_ssize_t offset, int bitshift, int fbitsize)
+{
+    int err;
+    PyObject *fname_obj;
+    Py_ssize_t prev_size;
+    CFieldObject *cf = PyObject_New(CFieldObject, &CField_Type);
+    if (cf == NULL)
+        return NULL;
+
+    Py_INCREF(ftype);
+    cf->cf_type = ftype;
+    cf->cf_offset = offset;
+    cf->cf_bitshift = bitshift;
+    cf->cf_bitsize = fbitsize;
+
+    fname_obj = PyString_InternFromString(fname);
+    prev_size = PyDict_Size(interned_fields);
+    err = PyDict_SetItem(interned_fields, fname_obj, (PyObject *)cf);
+    Py_DECREF(fname_obj);
+    Py_DECREF(cf);
+    if (err < 0)
+        return NULL;
+
+    if (PyDict_Size(interned_fields) != prev_size + 1) {
+        PyErr_Format(PyExc_KeyError, "duplicate field name '%s'",
+                     PyText_AS_UTF8(fname));
+        return NULL;
+    }
+    return cf;   /* borrowed reference */
+}
+
+static void zef_complete(_crx_builder_t *cb, _crx_type_t *ct,
                          size_t sz, size_t align,
                          _crx_field_t fields[], int nfields)
 {
-    abort();
+    PyObject *interned_fields;
+    CFieldObject **previous;
+    int i;
+
+    assert(ct->ct_flags & (CT_STRUCT | CT_UNION));
+    if (ct->ct_size >= 0) {
+        PyErr_Format(ZefError, "duplicate declaration of the fields of '%s'",
+                     ct->ct_name);
+        return;
+    }
+
+    interned_fields = PyDict_New();
+    if (interned_fields == NULL)
+        return;
+
+    previous = (CFieldObject **)&ct->ct_extra;
+
+    for (i = 0; i < nfields; i++) {
+        _crx_field_t *f = &fields[i];
+        CTypeDescrObject *ftype = f->type;
+
+        if (ftype->ct_size < 0) {
+            if ((ftype->ct_flags & CT_ARRAY) /* && fbitsize < 0
+                    && (i == nb_fields - 1 || foffset != -1) */ ) {
+                ct->ct_flags |= CT_WITH_VAR_ARRAY;
+            }
+            else {
+                PyErr_Format(PyExc_TypeError,
+                             "field '%s.%s' has ctype '%s' of unknown size",
+                             ct->ct_name, f->name, ftype->ct_name);
+                goto error;
+            }
+        }
+
+        if (f->numbits < 0) {
+            /* not a bitfield: common case */
+            int bs_flag;
+
+            if ((ftype->ct_flags & CT_ARRAY) && ftype->ct_length == 0)
+                bs_flag = BS_EMPTY_ARRAY;
+            else
+                bs_flag = BS_REGULAR;
+
+#if 0  /* XXX */
+            if (PyText_GetSize(fname) == 0 &&
+                    ftype->ct_flags & (CT_STRUCT|CT_UNION)) {
+                /* a nested anonymous struct or union */
+                CFieldObject *cfsrc = (CFieldObject *)ftype->ct_extra;
+                for (; cfsrc != NULL; cfsrc = cfsrc->cf_next) {
+                    /* broken complexity in the call to get_field_name(),
+                       but we'll assume you never do that with nested
+                       anonymous structures with thousand of fields */
+                    *previous = _add_field(interned_fields,
+                                           get_field_name(ftype, cfsrc),
+                                           cfsrc->cf_type,
+                                           boffset / 8 + cfsrc->cf_offset,
+                                           cfsrc->cf_bitshift,
+                                           cfsrc->cf_bitsize);
+                    if (*previous == NULL)
+                        goto error;
+                    previous = &(*previous)->cf_next;
+                }
+                /* always forbid such structures from being passed by value */
+                ct->ct_flags |= CT_CUSTOM_FIELD_POS;
+            }
+            else
+#endif
+            {
+                *previous = _add_field(interned_fields, f->name, ftype,
+                                       f->offset, bs_flag, -1);
+                if (*previous == NULL)
+                    goto error;
+                previous = &(*previous)->cf_next;
+            }
+        }
+        else {
+            abort();
+#if 0
+            /* this is the case of a bitfield */
+            Py_ssize_t field_offset_bytes;
+            int bits_already_occupied, bitshift;
+
+            if (foffset >= 0) {
+                PyErr_Format(PyExc_TypeError,
+                             "field '%s.%s' is a bitfield, "
+                             "but a fixed offset is specified",
+                             ct->ct_name, PyText_AS_UTF8(fname));
+                goto error;
+            }
+
+            if (!(ftype->ct_flags & (CT_PRIMITIVE_SIGNED |
+                                     CT_PRIMITIVE_UNSIGNED |
+                                     CT_PRIMITIVE_CHAR))) {
+                PyErr_Format(PyExc_TypeError,
+                        "field '%s.%s' declared as '%s' cannot be a bit field",
+                             ct->ct_name, PyText_AS_UTF8(fname),
+                             ftype->ct_name);
+                goto error;
+            }
+            if (fbitsize > 8 * ftype->ct_size) {
+                PyErr_Format(PyExc_TypeError,
+                             "bit field '%s.%s' is declared '%s:%d', which "
+                             "exceeds the width of the type",
+                             ct->ct_name, PyText_AS_UTF8(fname),
+                             ftype->ct_name, fbitsize);
+                goto error;
+            }
+
+            /* compute the starting position of the theoretical field
+               that covers a complete 'ftype', inside of which we will
+               locate the real bitfield */
+            field_offset_bytes = boffset / 8;
+            field_offset_bytes &= ~(falign - 1);
+
+            if (fbitsize == 0) {
+                if (PyText_GetSize(fname) > 0) {
+                    PyErr_Format(PyExc_TypeError,
+                                 "field '%s.%s' is declared with :0",
+                                 ct->ct_name, PyText_AS_UTF8(fname));
+                    goto error;
+                }
+                if (!(sflags & SF_MSVC_BITFIELDS)) {
+                    /* GCC's notion of "ftype :0;" */
+
+                    /* pad boffset to a value aligned for "ftype" */
+                    if (boffset > field_offset_bytes * 8) {
+                        field_offset_bytes += falign;
+                        assert(boffset < field_offset_bytes * 8);
+                    }
+                    boffset = field_offset_bytes * 8;
+                }
+                else {
+                    /* MSVC's notion of "ftype :0;" */
+
+                    /* Mostly ignored.  It seems they only serve as
+                       separator between other bitfields, to force them
+                       into separate words. */
+                }
+                prev_bitfield_size = 0;
+            }
+            else {
+                if (!(sflags & SF_MSVC_BITFIELDS)) {
+                    /* GCC's algorithm */
+
+                    /* Can the field start at the offset given by 'boffset'?  It
+                       can if it would entirely fit into an aligned ftype field. */
+                    bits_already_occupied = boffset - (field_offset_bytes * 8);
+
+                    if (bits_already_occupied + fbitsize > 8 * ftype->ct_size) {
+                        /* it would not fit, we need to start at the next
+                           allowed position */
+                        if ((sflags & SF_PACKED) &&
+                            (bits_already_occupied & 7)) {
+                            PyErr_Format(PyExc_NotImplementedError,
+                                "with 'packed', gcc would compile field "
+                                "'%s.%s' to reuse some bits in the previous "
+                                "field", ct->ct_name, PyText_AS_UTF8(fname));
+                            goto error;
+                        }
+                        field_offset_bytes += falign;
+                        assert(boffset < field_offset_bytes * 8);
+                        boffset = field_offset_bytes * 8;
+                        bitshift = 0;
+                    }
+                    else {
+                        bitshift = bits_already_occupied;
+                        assert(bitshift >= 0);
+                    }
+                    boffset += fbitsize;
+                }
+                else {
+                    /* MSVC's algorithm */
+
+                    /* A bitfield is considered as taking the full width
+                       of their declared type.  It can share some bits
+                       with the previous field only if it was also a
+                       bitfield and used a type of the same size. */
+                    if (prev_bitfield_size == ftype->ct_size &&
+                        prev_bitfield_free >= fbitsize) {
+                        /* yes: reuse */
+                        bitshift = 8 * prev_bitfield_size - prev_bitfield_free;
+                    }
+                    else {
+                        /* no: start a new full field */
+                        boffset = (boffset + falign*8-1) & ~(falign*8-1); /*align*/
+                        boffset += ftype->ct_size * 8;
+                        bitshift = 0;
+                        prev_bitfield_size = ftype->ct_size;
+                        prev_bitfield_free = 8 * prev_bitfield_size;
+                    }
+                    prev_bitfield_free -= fbitsize;
+                    field_offset_bytes = boffset / 8 - ftype->ct_size;
+                }
+
+                if (sflags & SF_GCC_BIG_ENDIAN)
+                    bitshift = 8 * ftype->ct_size - fbitsize - bitshift;
+
+                *previous = _add_field(interned_fields, fname, ftype,
+                                       field_offset_bytes, bitshift, fbitsize);
+                if (*previous == NULL)
+                    goto error;
+                previous = &(*previous)->cf_next;
+            }
+#endif
+        }
+    }
+    *previous = NULL;
+
+    ct->ct_size = sz;
+    /* 'align' ignored for now */
+    ct->ct_stuff = interned_fields;
+    return;
+
+ error:
+    Py_DECREF(interned_fields);
 }
 
 static void zef_complete_enum(_crx_builder_t *cb, _crx_type_t *t,
diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -6,17 +6,6 @@
     PyObject *c_weakreflist;
 } CDataObject;
 
-struct cfieldobject_s {
-    PyObject_HEAD
-    CTypeDescrObject *cf_type;
-    Py_ssize_t cf_offset;
-    short cf_bitshift;   /* >= 0: bitshift; or BS_REGULAR or BS_EMPTY_ARRAY */
-    short cf_bitsize;
-    struct cfieldobject_s *cf_next;
-};
-#define BS_REGULAR     (-1)      /* a regular field, not with bitshift */
-#define BS_EMPTY_ARRAY (-2)      /* a field which is an array 'type[0]' */
-
 typedef union {
     unsigned char m_char;
     unsigned short m_short;
@@ -52,6 +41,70 @@
 
 /************************************************************/
 
+struct cfieldobject_s {
+    PyObject_HEAD
+    CTypeDescrObject *cf_type;
+    Py_ssize_t cf_offset;
+    short cf_bitshift;   /* >= 0: bitshift; or BS_REGULAR or BS_EMPTY_ARRAY */
+    short cf_bitsize;
+    struct cfieldobject_s *cf_next;
+};
+#define BS_REGULAR     (-1)      /* a regular field, not with bitshift */
+#define BS_EMPTY_ARRAY (-2)      /* a field which is an array 'type[0]' */
+
+static void
+cfield_dealloc(CFieldObject *cf)
+{
+    Py_DECREF(cf->cf_type);
+    PyObject_Del(cf);
+}
+
+#undef OFF
+#define OFF(x) offsetof(CFieldObject, x)
+
+static PyMemberDef cfield_members[] = {
+    {"type", T_OBJECT, OFF(cf_type), READONLY},
+    {"offset", T_PYSSIZET, OFF(cf_offset), READONLY},
+    {"bitshift", T_SHORT, OFF(cf_bitshift), READONLY},
+    {"bitsize", T_SHORT, OFF(cf_bitsize), READONLY},
+    {NULL}      /* Sentinel */
+};
+#undef OFF
+
+static PyTypeObject CField_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "zeffir.CField",
+    sizeof(CFieldObject),
+    0,
+    (destructor)cfield_dealloc,                 /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    0,                                          /* tp_compare */
+    0,                                          /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                         /* tp_flags */
+    0,                                          /* tp_doc */
+    0,                                          /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    cfield_members,                             /* tp_members */
+};
+
+/************************************************************/
+
 static int
 CDataObject_Or_PyFloat_Check(PyObject *ob)
 {
@@ -343,10 +396,10 @@
     return NULL;
 }
 
+static PyObject *convert_to_object_bitfield(char *data, CFieldObject *cf)
+{
+    abort();
 #if 0
-static PyObject *
-convert_to_object_bitfield(char *data, CFieldObject *cf)
-{
     CTypeDescrObject *ct = cf->cf_type;
     /*READ(data, ct->ct_size)*/
 
@@ -377,8 +430,8 @@
         else
             return PyLong_FromUnsignedLongLong(value);
     }
+#endif
 }
-#endif
 
 static int _convert_overflow(PyObject *init, const char *ct_name)
 {
@@ -1237,6 +1290,58 @@
     return convert_from_object(c, ctitem, v);
 }
 
+//2172
+static PyObject *cdata_getattro(CDataObject *cd, PyObject *attr)
+{
+    CFieldObject *cf;
+    CTypeDescrObject *ct = cd->c_type;
+
+    if (ct->ct_flags & CT_POINTER)
+        ct = ct->ct_itemdescr;
+
+    if ((ct->ct_flags & (CT_STRUCT|CT_UNION)) && ct->ct_stuff != NULL) {
+        cf = (CFieldObject *)PyDict_GetItem(ct->ct_stuff, attr);
+        if (cf != NULL) {
+            /* read the field 'cf' */
+            char *data = cd->c_data + cf->cf_offset;
+            if (cf->cf_bitshift == BS_REGULAR)
+                return convert_to_object(data, cf->cf_type);
+            else if (cf->cf_bitshift == BS_EMPTY_ARRAY)
+                return new_simple_cdata(data,
+                    (CTypeDescrObject *)cf->cf_type->ct_stuff);
+            else
+                return convert_to_object_bitfield(data, cf);
+        }
+    }
+    return PyObject_GenericGetAttr((PyObject *)cd, attr);
+}
+
+//2198
+static int cdata_setattro(CDataObject *cd, PyObject *attr, PyObject *value)
+{
+    CFieldObject *cf;
+    CTypeDescrObject *ct = cd->c_type;
+
+    if (ct->ct_flags & CT_POINTER)
+        ct = ct->ct_itemdescr;
+
+    if ((ct->ct_flags & (CT_STRUCT|CT_UNION)) && ct->ct_stuff != NULL) {
+        cf = (CFieldObject *)PyDict_GetItem(ct->ct_stuff, attr);
+        if (cf != NULL) {
+            /* write the field 'cf' */
+            if (value != NULL) {
+                return convert_field_from_object(cd->c_data, cf, value);
+            }
+            else {
+                PyErr_SetString(PyExc_AttributeError,
+                                "cannot delete struct field");
+                return -1;
+            }
+        }
+    }
+    return PyObject_GenericSetAttr((PyObject *)cd, attr, value);
+}
+
 
 static PyMappingMethods CData_as_mapping = {
     (lenfunc)cdata_length, /*mp_length*/
@@ -1261,8 +1366,8 @@
     0,//(hashfunc)cdata_hash,                       /* tp_hash */
     0,//(ternaryfunc)cdata_call,                    /* tp_call */
     0,                                          /* tp_str */
-    0,//(getattrofunc)cdata_getattro,               /* tp_getattro */
-    0,//(setattrofunc)cdata_setattro,               /* tp_setattro */
+    (getattrofunc)cdata_getattro,               /* tp_getattro */
+    (setattrofunc)cdata_setattro,               /* tp_setattro */
     0,                                          /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */
     0,                                          /* tp_doc */
diff --git a/zeffir/test/struct.crx b/zeffir/test/struct.crx
new file mode 100644
--- /dev/null
+++ b/zeffir/test/struct.crx
@@ -0,0 +1,13 @@
+typedef struct {
+    int a;
+    long b;
+} mystruct_t;
+
+
+// CREFLECT: start
+
+typedef struct {
+    int a, b;
+} mystruct_t;
+
+// CREFLECT: end
diff --git a/zeffir/test/test_struct.py b/zeffir/test/test_struct.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_struct.py
@@ -0,0 +1,10 @@
+import support
+
+
+def test_simple_struct():
+    ffi, lib = support.compile_and_open('struct')
+    p = ffi.new('mystruct_t *', [-5, 0x600112233])
+    assert p.a == -5
+    assert p.b == 0x600112233
+    p.b += 1
+    assert p.b == 0x600112234
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -41,6 +41,8 @@
         return;
     if (PyType_Ready(&CTypeDescr_Type) < 0)
         return;
+    if (PyType_Ready(&CField_Type) < 0)
+        return;
     if (PyType_Ready(&CData_Type) < 0)
         return;
     if (PyType_Ready(&CDataOwning_Type) < 0)
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -6,7 +6,7 @@
 
 static PyTypeObject ZefFFI_Type;
 static PyTypeObject CTypeDescr_Type;
-//static PyTypeObject CField_Type;
+static PyTypeObject CField_Type;
 static PyTypeObject CData_Type;
 static PyTypeObject CDataOwning_Type;
 static PyTypeObject CDataOwningGC_Type;

From noreply at buildbot.pypy.org  Fri Dec  5 16:56:00 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 16:56:00 +0100 (CET)
Subject: [pypy-commit] creflect default: ffi.NULL
Message-ID: <20141205155600.1A3751D2A0C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r165:64bdd79c09b5
Date: 2014-12-05 16:47 +0100
http://bitbucket.org/cffi/creflect/changeset/64bdd79c09b5/

Log:	ffi.NULL

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -6,7 +6,6 @@
     _crx_builder_t cb;
     ZefLibObject *lib;
     PyObject *l_dict;
-    ZefFFIObject *ffi;
     PyObject *types_dict;
 } zeffir_builder_t;
 
@@ -837,7 +836,6 @@
         },
         lib,              /* lib */
         lib->l_dict,      /* l_dict */
-        ffi,              /* ffi */
         ffi->types_dict,  /* types_dict */
     };
     crxmain(&builder.cb);
@@ -845,7 +843,7 @@
     return PyErr_Occurred() ? -1 : 0;
 }
 
-static CTypeDescrObject *parse_c_decl(ZefFFIObject *ffi, const char *str)
+static CTypeDescrObject *parse_c_decl(PyObject *types_dict, const char *str)
 {
     zeffir_builder_t builder = {
         {
@@ -875,8 +873,7 @@
         },
         NULL,             /* lib */
         NULL,             /* l_dict */
-        ffi,              /* ffi */
-        ffi->types_dict,  /* types_dict */
+        types_dict,       /* types_dict */
     };
 
     _crx_qual_type result;
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -126,7 +126,8 @@
         if (x != NULL && CTypeDescr_Check(x))
             return (CTypeDescrObject *)x;
 
-        CTypeDescrObject *ct = parse_c_decl(ffi, PyText_AS_UTF8(arg));
+        CTypeDescrObject *ct = parse_c_decl(ffi->types_dict,
+                                            PyText_AS_UTF8(arg));
         if (ct == NULL)
             return NULL;
 
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -31,3 +31,8 @@
 def test_types():
     ffi = support.new_ffi()
     assert ffi.types == {}
+
+def test_NULL():
+    ffi = support.new_ffi()
+    assert repr(ffi.NULL) == "<cdata 'void *' NULL>"
+    assert repr(type(ffi).NULL) == "<cdata 'void *' NULL>"
diff --git a/zeffir/test/test_struct.py b/zeffir/test/test_struct.py
--- a/zeffir/test/test_struct.py
+++ b/zeffir/test/test_struct.py
@@ -1,3 +1,4 @@
+import py
 import support
 
 
@@ -8,3 +9,9 @@
     assert p.b == 0x600112233
     p.b += 1
     assert p.b == 0x600112234
+
+def test_addressof():
+    py.test.skip("in-progress")
+    ffi, lib = support.compile_and_open('struct')
+    p = ffi.new('mystruct_t *')
+    ffi.addressof(p[0], 'a')
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -70,6 +70,24 @@
     if (PyModule_AddObject(m, "error", ZefError) < 0)
         return;
 
+    {
+        CTypeDescrObject *ctvoidp;
+        PyObject *cdnull;
+        PyObject *d1 = PyDict_New();
+        if (d1 == NULL)
+            return;
+
+        ctvoidp = parse_c_decl(d1, "void*");
+        if (ctvoidp) {
+            cdnull = new_simple_cdata(NULL, ctvoidp);
+            if (cdnull)
+                PyDict_SetItemString(ZefFFI_Type.tp_dict, "NULL", cdnull);
+        }
+        Py_DECREF(d1);
+        if (PyErr_Occurred())
+            return;
+    }
+
     if (PyDict_SetItemString(ZefFFI_Type.tp_dict, "error", ZefError) < 0)
         return;
 }
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -15,7 +15,7 @@
 
 static int lib_close(ZefLibObject *);
 static int load_creflect_main(ZefFFIObject *, ZefLibObject *);
-static CTypeDescrObject *parse_c_decl(ZefFFIObject *ffi, const char *str);
+static CTypeDescrObject *parse_c_decl(PyObject *types_dict, const char *str);
 
 static int convert_from_object(char *, CTypeDescrObject *, PyObject *);
 static int convert_from_object_bitfield(char *, CFieldObject *, PyObject *);

From noreply at buildbot.pypy.org  Fri Dec  5 16:56:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 16:56:01 +0100 (CET)
Subject: [pypy-commit] creflect default: ffi.string
Message-ID: <20141205155601.6E8DA1D2A0C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r166:030aea977e7e
Date: 2014-12-05 16:52 +0100
http://bitbucket.org/cffi/creflect/changeset/030aea977e7e/

Log:	ffi.string

diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -255,12 +255,75 @@
     return (PyObject *)cd;
 }
 
+static PyObject *ffi_string(PyObject *self, PyObject *args)
+{
+    CDataObject *cd;
+    Py_ssize_t maxlen = -1;
+    if (!PyArg_ParseTuple(args, "O!|n:string",
+                          &CData_Type, &cd, &maxlen))
+        return NULL;
+
+    if (cd->c_type->ct_itemdescr != NULL &&
+        cd->c_type->ct_itemdescr->ct_flags & (CT_PRIMITIVE_CHAR |
+                                              CT_PRIMITIVE_SIGNED |
+                                              CT_PRIMITIVE_UNSIGNED)) {
+        Py_ssize_t length = maxlen;
+        if (cd->c_data == NULL) {
+            PyObject *s = cdata_repr(cd);
+            if (s != NULL) {
+                PyErr_Format(PyExc_RuntimeError,
+                             "cannot use string() on %s",
+                             PyText_AS_UTF8(s));
+                Py_DECREF(s);
+            }
+            return NULL;
+        }
+        if (length < 0 && cd->c_type->ct_flags & CT_ARRAY) {
+            length = get_array_length(cd);
+        }
+        if (cd->c_type->ct_itemdescr->ct_size == sizeof(char)) {
+            const char *start = cd->c_data;
+            if (length < 0) {
+                /*READ(start, 1)*/
+                length = strlen(start);
+                /*READ(start, length)*/
+            }
+            else {
+                const char *end;
+                /*READ(start, length)*/
+                end = (const char *)memchr(start, 0, length);
+                if (end != NULL)
+                    length = end - start;
+            }
+            return PyBytes_FromStringAndSize(start, length);
+        }
+    }
+    else if (cd->c_type->ct_flags & CT_IS_ENUM) {
+        abort();
+        //return convert_cdata_to_enum_string(cd, 0);
+    }
+    else if (cd->c_type->ct_flags & CT_IS_BOOL) {
+        /* fall through to TypeError */
+    }
+    else if (cd->c_type->ct_flags & (CT_PRIMITIVE_CHAR |
+                                     CT_PRIMITIVE_SIGNED |
+                                     CT_PRIMITIVE_UNSIGNED)) {
+        /*READ(cd->c_data, cd->c_type->ct_size)*/
+        if (cd->c_type->ct_size == sizeof(char))
+            return PyBytes_FromStringAndSize(cd->c_data, 1);
+    }
+    PyErr_Format(PyExc_TypeError, "string(): unexpected cdata '%s' argument",
+                 cd->c_type->ct_name);
+    return NULL;
+}
+
 static PyMethodDef ffi_methods[] = {
     {"close_library", ffi_close_library,         METH_VARARGS | METH_STATIC},
     {"load_library",  (PyCFunction)ffi_load_library,
                                                  METH_VARARGS | METH_KEYWORDS},
     {"new",           (PyCFunction)ffi_new,      METH_VARARGS},
     {"sizeof",        (PyCFunction)ffi_sizeof,   METH_O},
+    {"string",        (PyCFunction)ffi_string,   METH_VARARGS},
     {"typeof",        (PyCFunction)ffi_typeof,   METH_O},
     {NULL}
 };
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -36,3 +36,11 @@
     ffi = support.new_ffi()
     assert repr(ffi.NULL) == "<cdata 'void *' NULL>"
     assert repr(type(ffi).NULL) == "<cdata 'void *' NULL>"
+
+def test_string():
+    ffi = support.new_ffi()
+    p = ffi.new("char[]", "hello\x00world")
+    assert ffi.string(p) == "hello"
+    for i in range(50):
+        p = ffi.new("char[]", "12345678" * i)
+        assert ffi.string(p) == "12345678" * i

From noreply at buildbot.pypy.org  Fri Dec  5 16:56:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 16:56:02 +0100 (CET)
Subject: [pypy-commit] creflect default: equality on cdata's
Message-ID: <20141205155602.95F351D2CF0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r167:6aee42c289d3
Date: 2014-12-05 16:56 +0100
http://bitbucket.org/cffi/creflect/changeset/6aee42c289d3/

Log:	equality on cdata's

diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1050,6 +1050,52 @@
     }
 }
 
+//1769
+static PyObject *cdata_richcompare(PyObject *v, PyObject *w, int op)
+{
+    int res;
+    PyObject *pyres;
+    char *v_cdata, *w_cdata;
+
+    assert(CData_Check(v));
+    if (!CData_Check(w)) {
+        pyres = Py_NotImplemented;
+        goto done;
+    }
+
+    if ((op != Py_EQ && op != Py_NE) &&
+        ((((CDataObject *)v)->c_type->ct_flags & CT_PRIMITIVE_ANY) ||
+         (((CDataObject *)w)->c_type->ct_flags & CT_PRIMITIVE_ANY)))
+        goto Error;
+
+    v_cdata = ((CDataObject *)v)->c_data;
+    w_cdata = ((CDataObject *)w)->c_data;
+
+    switch (op) {
+    case Py_EQ: res = (v_cdata == w_cdata); break;
+    case Py_NE: res = (v_cdata != w_cdata); break;
+    case Py_LT: res = (v_cdata <  w_cdata); break;
+    case Py_LE: res = (v_cdata <= w_cdata); break;
+    case Py_GT: res = (v_cdata >  w_cdata); break;
+    case Py_GE: res = (v_cdata >= w_cdata); break;
+    default: res = -1;
+    }
+    pyres = res ? Py_True : Py_False;
+ done:
+    Py_INCREF(pyres);
+    return pyres;
+
+ Error:
+    PyErr_SetString(PyExc_TypeError,
+                    "cannot do comparison on a primitive cdata");
+    return NULL;
+}
+
+static long cdata_hash(CDataObject *cd)
+{
+    return _Py_HashPointer(cd->c_data);
+}
+
 //1815
 static Py_ssize_t cdata_length(CDataObject *cd)
 {
@@ -1363,7 +1409,7 @@
     0,//&CData_as_number,                           /* tp_as_number */
     0,                                          /* tp_as_sequence */
     &CData_as_mapping,                          /* tp_as_mapping */
-    0,//(hashfunc)cdata_hash,                       /* tp_hash */
+    (hashfunc)cdata_hash,                       /* tp_hash */
     0,//(ternaryfunc)cdata_call,                    /* tp_call */
     0,                                          /* tp_str */
     (getattrofunc)cdata_getattro,               /* tp_getattro */
@@ -1373,7 +1419,7 @@
     0,                                          /* tp_doc */
     0,                                          /* tp_traverse */
     0,                                          /* tp_clear */
-    0,//cdata_richcompare,                          /* tp_richcompare */
+    cdata_richcompare,                          /* tp_richcompare */
     offsetof(CDataObject, c_weakreflist),       /* tp_weaklistoffset */
     0,//(getiterfunc)cdata_iter,                    /* tp_iter */
     0,                                          /* tp_iternext */
diff --git a/zeffir/test/test_global.py b/zeffir/test/test_global.py
--- a/zeffir/test/test_global.py
+++ b/zeffir/test/test_global.py
@@ -24,3 +24,9 @@
     q = ffi.new("short *", -555)
     lib.myglobptr = q
     assert lib.myglobptr[0] == -555
+
+def test_eq():
+    ffi, lib = support.compile_and_open('global')
+    assert lib.myglobptr is not lib.myglobptr
+    assert lib.myglobptr == lib.myglobptr
+    assert hash(lib.myglobptr) == hash(lib.myglobptr)

From noreply at buildbot.pypy.org  Fri Dec  5 17:06:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 17:06:15 +0100 (CET)
Subject: [pypy-commit] creflect default: functions returning void
Message-ID: <20141205160615.B23F01D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r168:c56076ff1eb2
Date: 2014-12-05 17:06 +0100
http://bitbucket.org/cffi/creflect/changeset/c56076ff1eb2/

Log:	functions returning void

diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -115,6 +115,7 @@
     void *llargs[nargs];
     char *llbuffer;
     size_t lloffset;
+    int returns_void;
 
     actualnargs = PyTuple_Size(args);
     if (actualnargs != nargs) {
@@ -126,7 +127,8 @@
     }
 
     llbuffer = alloca(zfs->zfs_size_args);
-    lloffset = zfs->zfs_ret->ct_size;
+    returns_void = (zfs->zfs_ret->ct_flags & CT_VOID);
+    lloffset = returns_void ? 0 : zfs->zfs_ret->ct_size;
 
     for (i = 0; i < nargs; i++) {
         CTypeDescrObject *ct = zfs->zfs_args[i];
@@ -163,6 +165,10 @@
 
     zfs->zfs_trampl(llargs, llbuffer);
 
+    if (returns_void) {
+        Py_INCREF(Py_None);
+        return Py_None;
+    }
     return convert_to_object(llbuffer, zfs->zfs_ret);
 }
 
@@ -186,8 +192,8 @@
     zfs->zfs_ret = ret;
     Py_INCREF(ret);
 
-    assert(ret->ct_size >= 0);
-    size = ret->ct_size;
+    size = (ret->ct_flags & CT_VOID) ? 0 : ret->ct_size;
+    assert(size >= 0);
 
     for (i = 0; i < nargs; i++) {
         CTypeDescrObject *ct = args[i].type;
diff --git a/zeffir/test/function.crx b/zeffir/test/function.crx
--- a/zeffir/test/function.crx
+++ b/zeffir/test/function.crx
@@ -11,10 +11,18 @@
     return result;
 }
 
+void returning_nothing(char *p)
+{
+    *p++ = 'O';
+    *p++ = 'K';
+    *p++ = 0;
+}
+
 
 // CREFLECT: start
 
 int simple_function(int);
 int add_from_array(int array[], int count);
+void returning_nothing(char *);
 
 // CREFLECT: end
diff --git a/zeffir/test/test_function.py b/zeffir/test/test_function.py
--- a/zeffir/test/test_function.py
+++ b/zeffir/test/test_function.py
@@ -20,3 +20,9 @@
     res = lib.add_from_array([30, 2, 10], 3)
     assert type(res) is int
     assert res == 42
+
+def test_function_returning_nothing():
+    ffi, lib = support.compile_and_open('function')
+    p = ffi.new("char[]", 10)
+    lib.returning_nothing(p)
+    assert ffi.string(p) == "OK"
diff --git a/zeffir/test/test_global.py b/zeffir/test/test_global.py
--- a/zeffir/test/test_global.py
+++ b/zeffir/test/test_global.py
@@ -30,3 +30,5 @@
     assert lib.myglobptr is not lib.myglobptr
     assert lib.myglobptr == lib.myglobptr
     assert hash(lib.myglobptr) == hash(lib.myglobptr)
+    lib.myglobptr = ffi.NULL
+    assert lib.myglobptr == ffi.NULL

From noreply at buildbot.pypy.org  Fri Dec  5 17:16:00 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 17:16:00 +0100 (CET)
Subject: [pypy-commit] creflect default: Demo files (direct ports from
	hg/cffi/demo)
Message-ID: <20141205161600.498EA1D2A0C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r169:fe2fdb378e56
Date: 2014-12-05 17:15 +0100
http://bitbucket.org/cffi/creflect/changeset/fe2fdb378e56/

Log:	Demo files (direct ports from hg/cffi/demo)

diff --git a/demo/gmp.crx b/demo/gmp.crx
new file mode 100644
--- /dev/null
+++ b/demo/gmp.crx
@@ -0,0 +1,13 @@
+#include <gmp.h>
+
+
+/* CREFLECT: start */
+
+typedef struct { } MP_INT;
+typedef MP_INT mpz_t[1];
+
+int mpz_init_set_str (MP_INT *dest_integer, char *src_cstring, int base);
+void mpz_add (MP_INT *sum, MP_INT *addend1, MP_INT *addend2);
+char * mpz_get_str (char *string, int base, MP_INT *integer);
+
+/* CREFLECT: end */
diff --git a/demo/gmp.py b/demo/gmp.py
new file mode 100644
--- /dev/null
+++ b/demo/gmp.py
@@ -0,0 +1,15 @@
+import sys
+import zeffir
+
+ffi = zeffir.FFI()
+lib = ffi.load_library("gmp-creflect", relative_to=__file__)
+
+a = ffi.new("mpz_t")
+b = ffi.new("mpz_t")
+
+lib.mpz_init_set_str(a, sys.argv[1], 10)	# Assume decimal integers
+lib.mpz_init_set_str(b, sys.argv[2], 10)	# Assume decimal integers
+lib.mpz_add(a, a, b)			# a=a+b
+
+s = lib.mpz_get_str(ffi.NULL, 10, a)
+print ffi.string(s)
diff --git a/demo/readdir2.crx b/demo/readdir2.crx
new file mode 100644
--- /dev/null
+++ b/demo/readdir2.crx
@@ -0,0 +1,29 @@
+#ifndef _ATFILE_SOURCE
+#  define _ATFILE_SOURCE
+#endif
+#ifndef _BSD_SOURCE
+#  define _BSD_SOURCE
+#endif
+#include <fcntl.h>
+#include <sys/types.h>
+#include <dirent.h>
+
+
+/* CREFLECT: start */
+
+typedef ... DIR;
+
+struct dirent {
+    unsigned char  d_type;      /* type of file; not supported
+                                   by all file system types */
+    char           d_name[];    /* filename */
+};
+
+int readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result);
+int openat(int dirfd, const char *pathname, int flags);
+DIR *fdopendir(int fd);
+int closedir(DIR *dirp);
+
+#define DT_DIR ...
+
+/* CREFLECT: end */
diff --git a/demo/readdir2.py b/demo/readdir2.py
new file mode 100644
--- /dev/null
+++ b/demo/readdir2.py
@@ -0,0 +1,32 @@
+import zeffir
+
+ffi = zeffir.FFI()
+lib = ffi.load_library("readdir2-creflect", relative_to=__file__)
+
+p = ffi.new("struct dirent *")
+print p
+
+
+def walk(basefd, path):
+    print '{', path
+    dirfd = lib.openat(basefd, path, 0)
+    if dirfd < 0:
+        # error in openat()
+        return
+    dir = lib.fdopendir(dirfd)
+    dirent = ffi.new("struct dirent *")
+    result = ffi.new("struct dirent **")
+    while True:
+        if lib.readdir_r(dir, dirent, result):
+            # error in readdir_r()
+            break
+        if result[0] == ffi.NULL:
+            break
+        name = ffi.string(dirent.d_name)
+        print '%3d %s' % (dirent.d_type, name)
+        if dirent.d_type == lib.DT_DIR and name != '.' and name != '..':
+            walk(dirfd, name)
+    lib.closedir(dir)
+    print '}'
+
+walk(-1, "/tmp")

From noreply at buildbot.pypy.org  Fri Dec  5 17:31:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 17:31:13 +0100 (CET)
Subject: [pypy-commit] creflect default: add a Makefile
Message-ID: <20141205163113.5F2D61D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r170:fa167c323433
Date: 2014-12-05 17:25 +0100
http://bitbucket.org/cffi/creflect/changeset/fa167c323433/

Log:	add a Makefile

diff --git a/demo/Makefile b/demo/Makefile
new file mode 100644
--- /dev/null
+++ b/demo/Makefile
@@ -0,0 +1,16 @@
+ALL = zeffir.so libreaddir2-creflect.so libgmp-creflect.so
+
+all: $(ALL)
+
+%-creflect.c: %.crx
+	PYTHONPATH=.. ../bin/creflect $*.crx $*-creflect.c
+
+lib%-creflect.so: %-creflect.c
+	gcc -g -shared -fPIC $*-creflect.c -o lib$*-creflect.so -I../creflect
+
+zeffir.so: ../zeffir/*.h ../zeffir/*.c
+	gcc -g -shared -fPIC ../zeffir/zeffir.c -o zeffir.so -I/usr/include/python2.7
+
+
+clean:
+	rm -f $(ALL) readdir2-creflect.c

From noreply at buildbot.pypy.org  Fri Dec  5 17:31:14 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 17:31:14 +0100 (CET)
Subject: [pypy-commit] creflect default: fix
Message-ID: <20141205163114.791421D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r171:60242be937cb
Date: 2014-12-05 17:31 +0100
http://bitbucket.org/cffi/creflect/changeset/60242be937cb/

Log:	fix

diff --git a/demo/Makefile b/demo/Makefile
--- a/demo/Makefile
+++ b/demo/Makefile
@@ -5,6 +5,9 @@
 %-creflect.c: %.crx
 	PYTHONPATH=.. ../bin/creflect $*.crx $*-creflect.c
 
+libgmp-creflect.so: gmp-creflect.c
+	gcc -g -shared -fPIC gmp-creflect.c -o libgmp-creflect.so -lgmp -I../creflect
+
 lib%-creflect.so: %-creflect.c
 	gcc -g -shared -fPIC $*-creflect.c -o lib$*-creflect.so -I../creflect
 

From noreply at buildbot.pypy.org  Fri Dec  5 17:44:59 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri,  5 Dec 2014 17:44:59 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Add dummy analyze_signals pass to
 bc_reader.build_flow()
Message-ID: <20141205164459.4E4941D28F0@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74832:3d32a761abce
Date: 2014-12-03 13:50 +0000
http://bitbucket.org/pypy/pypy/changeset/3d32a761abce/

Log:	Add dummy analyze_signals pass to bc_reader.build_flow()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -214,9 +214,16 @@
             block = self.curr_block
             graph.pos_index[instr.offset] = block, len(block.operations)
             instr.bc_flow(self)
+        self.analyze_signals(graph)
         self.check_graph()
         return graph
 
+    def analyze_signals(self, graph):
+        for block in graph.iterblocks():
+            self.curr_block = block
+            for instr in block:
+                instr.do_signals(self)
+
     def check_graph(self):
         for b in self.blocks:
             if not b._exits:
@@ -263,6 +270,17 @@
             instr = self.read(offset)
             yield instr
 
+    def iterblocks(self):
+        block = self.entry
+        seen = set()
+        stack = block._exits[:]
+        while stack:
+            block = stack.pop()
+            if block not in seen:
+                yield block
+                seen.add(block)
+                stack.extend(block._exits[:])
+
     def all_blocks(self):
         return set(x[0] for x in self.pos_index.values())
 
@@ -338,15 +356,18 @@
         self.arg = arg
         self.offset = offset
 
-    def eval(self, ctx):
-        pass
-
     def bc_flow(self, reader):
         reader.curr_block.operations.append(self)
         if self.has_jump():
             reader.end_block()
             reader.get_block_at(self.arg)
 
+    def do_signals(self, reader):
+        pass
+
+    def eval(self, ctx):
+        pass
+
     def has_jump(self):
         return self.num in opcode.hasjrel or self.num in opcode.hasjabs
 

From noreply at buildbot.pypy.org  Fri Dec  5 17:45:00 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri,  5 Dec 2014 17:45:00 +0100 (CET)
Subject: [pypy-commit] pypy framestate: begin implementing analyze_signals()
Message-ID: <20141205164500.9DBA91D28F0@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74833:3fbf13809a73
Date: 2014-12-03 17:32 +0000
http://bitbucket.org/pypy/pypy/changeset/3fbf13809a73/

Log:	begin implementing analyze_signals()

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -221,8 +221,12 @@
     def analyze_signals(self, graph):
         for block in graph.iterblocks():
             self.curr_block = block
+            block.init_blockstack()
+            self.blockstack = block.blockstack[:]
             for instr in block:
                 instr.do_signals(self)
+            for exit in block._exits:
+                exit.set_blockstack(self.blockstack)
 
     def check_graph(self):
         for b in self.blocks:
@@ -294,7 +298,7 @@
     def __init__(self):
         self.parents = set()
         self._exits = []
-        self.blockstack = []
+        self.blockstack = None
 
     def __getitem__(self, i):
         return self.operations[i]
@@ -320,6 +324,16 @@
     def startpos(self):
         return self.operations[0].offset
 
+    def init_blockstack(self):
+        if self.blockstack is None:
+            self.blockstack = []
+
+    def set_blockstack(self, blockstack):
+        if self.blockstack is None:
+            self.blockstack = blockstack
+        else:
+            assert self.blockstack == blockstack
+
     def split_at(self, i):
         if i == 0 or i == len(self.operations):
             return [self]
@@ -546,7 +560,9 @@
     def bc_flow(self, reader):
         reader.curr_block.operations.append(self)
         self.target = reader.get_block_at(self.arg)
-        reader.curr_block.blockstack.append(self.make_block(-1))
+
+    def do_signals(self, reader):
+        reader.blockstack.append(self.make_block(-1))
 
     def eval(self, ctx):
         block = self.make_block(ctx.stackdepth)
@@ -590,6 +606,16 @@
         ctx.blockstack.append(block)
         ctx.pushvalue(w_result)
 
+ at bc_reader.register_opcode
+class POP_BLOCK(BCInstruction):
+    def do_signals(self, reader):
+        reader.blockstack.pop()
+
+    def eval(self, ctx):
+        block = ctx.blockstack.pop()
+        block.cleanupstack(ctx)  # the block knows how to clean up the value stack
+
+
 _unary_ops = [
     ('UNARY_POSITIVE', op.pos),
     ('UNARY_NEGATIVE', op.neg),
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -652,10 +652,6 @@
             signal = self.popvalue()
             raise signal
 
-    def POP_BLOCK(self, oparg):
-        block = self.blockstack.pop()
-        block.cleanupstack(self)  # the block knows how to clean up the value stack
-
     def YIELD_VALUE(self, _):
         assert self.pycode.is_generator
         w_result = self.popvalue()

From noreply at buildbot.pypy.org  Fri Dec  5 17:45:01 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri,  5 Dec 2014 17:45:01 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move stuff around
Message-ID: <20141205164501.E6A9F1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74834:ac849f367719
Date: 2014-12-03 23:57 +0000
http://bitbucket.org/pypy/pypy/changeset/ac849f367719/

Log:	move stuff around

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -312,15 +312,18 @@
             return w_condition.value
         return self.recorder.guessbool(self, w_condition)
 
-    def record(self, spaceop):
+    def maybe_merge(self):
         recorder = self.recorder
         if getattr(recorder, 'final_state', None) is not None:
             self.mergeblock(recorder.crnt_block, recorder.final_state)
             raise StopFlowing
+
+    def record(self, spaceop):
         spaceop.offset = self.last_offset
-        recorder.append(spaceop)
+        self.recorder.append(spaceop)
 
     def do_op(self, op):
+        self.maybe_merge()
         self.record(op)
         self.guessexception(op.canraise)
         return op.result
@@ -410,14 +413,8 @@
             if newstate is not None:
                 break
         else:
-            newstate = currentstate.copy()
-            newblock = SpamBlock(newstate)
-            # unconditionally link the current block to the newblock
-            outputargs = currentstate.getoutputargs(newstate)
-            link = Link(outputargs, newblock)
-            currentblock.closeblock(link)
+            newblock = self.make_next_block(currentblock, currentstate)
             candidates.insert(0, newblock)
-            self.pendingblocks.append(newblock)
             return
 
         if newstate.matches(block.framestate):
@@ -447,6 +444,16 @@
         candidates.insert(0, newblock)
         self.pendingblocks.append(newblock)
 
+    def make_next_block(self, block, state):
+        newstate = state.copy()
+        newblock = SpamBlock(newstate)
+        # unconditionally link the current block to the newblock
+        outputargs = state.getoutputargs(newstate)
+        link = Link(outputargs, newblock)
+        block.closeblock(link)
+        self.pendingblocks.append(newblock)
+        return newblock
+
     # hack for unrolling iterables, don't use this
     def replace_in_stack(self, oldvalue, newvalue):
         w_new = Constant(newvalue)

From noreply at buildbot.pypy.org  Fri Dec  5 17:45:03 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri,  5 Dec 2014 17:45:03 +0100 (CET)
Subject: [pypy-commit] pypy framestate: kill pos_index
Message-ID: <20141205164503.4EACD1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74835:d3503f9756b8
Date: 2014-12-04 00:42 +0000
http://bitbucket.org/pypy/pypy/changeset/d3503f9756b8/

Log:	kill pos_index

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -176,10 +176,6 @@
         if offset < self.next_offset:
             i_block, i_instr = self.find_position(offset)
             split = self.blocks[i_block].split_at(i_instr)
-            if len(split) == 2:
-                new_block = split[1]
-                for i, instr in enumerate(new_block.operations):
-                    self.graph.pos_index[instr.offset] = new_block, i
             self.blocks[i_block:i_block + 1] = split
             return split[-1]
         else:
@@ -211,8 +207,6 @@
         self.needs_new_block = False
         self.graph = graph = BytecodeGraph(self.blocks[0])
         for instr in self._iter_instr(code):
-            block = self.curr_block
-            graph.pos_index[instr.offset] = block, len(block.operations)
             instr.bc_flow(self)
         self.analyze_signals(graph)
         self.check_graph()
@@ -245,10 +239,9 @@
     def __init__(self, startblock):
         self.entry = EntryBlock()
         self.entry.set_exits([startblock])
-        self.pos_index = {}
 
     def read(self, pos):
-        bc_block, i = self.pos_index[pos]
+        bc_block, i = pos
         return bc_block[i]
 
     def next_pos(self):
@@ -261,17 +254,9 @@
         else:
             return block, i
 
-    def get_position(self, offset):
-        return self.pos_index[offset]
-
-    def get_offset(self, position):
-        block, i = position
-        return block[i].offset
-
     def iter_instr(self):
         while True:
-            offset = self.get_offset(self.curr_position)
-            instr = self.read(offset)
+            instr = self.read(self.curr_position)
             yield instr
 
     def iterblocks(self):
@@ -286,7 +271,7 @@
                 stack.extend(block._exits[:])
 
     def all_blocks(self):
-        return set(x[0] for x in self.pos_index.values())
+        return list(self.iterblocks())
 
     def dump(self):
         blocks = sorted(self.all_blocks(), key=lambda b: b.startpos)

From noreply at buildbot.pypy.org  Fri Dec  5 17:45:04 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri,  5 Dec 2014 17:45:04 +0100 (CET)
Subject: [pypy-commit] pypy framestate: rm force keyword from
	ctx.guessexception()
Message-ID: <20141205164504.8976D1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74836:1c2990ad5a58
Date: 2014-12-04 19:28 +0000
http://bitbucket.org/pypy/pypy/changeset/1c2990ad5a58/

Log:	rm force keyword from ctx.guessexception()

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -328,14 +328,14 @@
         self.guessexception(op.canraise)
         return op.result
 
-    def guessexception(self, exceptions, force=False):
+    def guessexception(self, exceptions):
         """
         Catch possible exceptions implicitly.
         """
         if not exceptions:
             return
-        if not force and not any(isinstance(block, (ExceptBlock, FinallyBlock))
-                                 for block in self.blockstack):
+        if not any(isinstance(block, (ExceptBlock, FinallyBlock))
+                for block in self.blockstack):
             # The implicit exception wouldn't be caught and would later get
             # removed, so don't bother creating it.
             return
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -517,7 +517,7 @@
                     ctx.replace_in_stack(it, next_unroller)
                     return const(v)
         w_item = ctx.do_op(self)
-        ctx.guessexception([StopIteration, RuntimeError], force=True)
+        ctx.recorder.guessexception(ctx, StopIteration, RuntimeError)
         return w_item
 
 class GetAttr(SingleDispatchMixin, HLOperation):

From noreply at buildbot.pypy.org  Fri Dec  5 17:45:05 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri,  5 Dec 2014 17:45:05 +0100 (CET)
Subject: [pypy-commit] pypy framestate: extract method ctx.has_exc_handler()
Message-ID: <20141205164505.C99541D28F0@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74837:1488ee7e209b
Date: 2014-12-05 15:23 +0000
http://bitbucket.org/pypy/pypy/changeset/1488ee7e209b/

Log:	extract method ctx.has_exc_handler()

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -334,12 +334,13 @@
         """
         if not exceptions:
             return
-        if not any(isinstance(block, (ExceptBlock, FinallyBlock))
-                for block in self.blockstack):
-            # The implicit exception wouldn't be caught and would later get
-            # removed, so don't bother creating it.
-            return
-        self.recorder.guessexception(self, *exceptions)
+        # Implicit exceptions are ignored unless they are caught explicitly
+        if self.has_exc_handler():
+            self.recorder.guessexception(self, *exceptions)
+
+    def has_exc_handler(self):
+        return any(isinstance(block, (ExceptBlock, FinallyBlock))
+                for block in self.blockstack)
 
     def build_flow(self):
         graph = self.graph

From noreply at buildbot.pypy.org  Fri Dec  5 17:45:07 2014
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri,  5 Dec 2014 17:45:07 +0100 (CET)
Subject: [pypy-commit] pypy framestate: move handling of escaping
 FlowSignals to nomoreblocks()
Message-ID: <20141205164507.1AE3B1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: framestate
Changeset: r74838:7c14497ac340
Date: 2014-12-05 16:44 +0000
http://bitbucket.org/pypy/pypy/changeset/7c14497ac340/

Log:	move handling of escaping FlowSignals to nomoreblocks()

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -368,35 +368,8 @@
                     next_position = next_block, 0
                 bc_graph.curr_position = next_position
                 self.recorder.final_state = self.getstate(next_position)
-
-        except RaiseImplicit as e:
-            w_exc = e.w_exc
-            if isinstance(w_exc.w_type, Constant):
-                exc_cls = w_exc.w_type.value
-            else:
-                exc_cls = Exception
-            msg = "implicit %s shouldn't occur" % exc_cls.__name__
-            w_type = Constant(AssertionError)
-            w_value = Constant(AssertionError(msg))
-            link = Link([w_type, w_value], self.graph.exceptblock)
-            self.recorder.crnt_block.closeblock(link)
-
-        except Raise as e:
-            w_exc = e.w_exc
-            if w_exc.w_type == const(ImportError):
-                msg = 'import statement always raises %s' % e
-                raise ImportError(msg)
-            link = Link([w_exc.w_type, w_exc.w_value], self.graph.exceptblock)
-            self.recorder.crnt_block.closeblock(link)
-
         except StopFlowing:
             pass
-
-        except Return as exc:
-            w_result = exc.w_value
-            link = Link([w_result], self.graph.returnblock)
-            self.recorder.crnt_block.closeblock(link)
-
         except FlowingError as exc:
             if exc.ctx is None:
                 exc.ctx = self
@@ -474,7 +447,7 @@
             if isinstance(signal, block.handles):
                 return block.handle(self, signal)
             block.cleanupstack(self)
-        return signal.nomoreblocks()
+        return signal.nomoreblocks(self)
 
     def getlocalvarname(self, index):
         return self.pycode.co_varnames[index]
@@ -1049,7 +1022,7 @@
                 WHY_CONTINUE,   Continue
                 WHY_YIELD       not needed
     """
-    def nomoreblocks(self):
+    def nomoreblocks(self, ctx):
         raise BytecodeCorruption("misplaced bytecode - should not return")
 
     def __eq__(self, other):
@@ -1063,8 +1036,10 @@
     def __init__(self, w_value):
         self.w_value = w_value
 
-    def nomoreblocks(self):
-        raise Return(self.w_value)
+    def nomoreblocks(self, ctx):
+        w_result = self.w_value
+        link = Link([w_result], ctx.graph.returnblock)
+        ctx.recorder.crnt_block.closeblock(link)
 
     @property
     def args(self):
@@ -1081,8 +1056,14 @@
     def __init__(self, w_exc):
         self.w_exc = w_exc
 
-    def nomoreblocks(self):
-        raise self
+    def nomoreblocks(self, ctx):
+        w_exc = self.w_exc
+        if w_exc.w_type == const(ImportError):
+            msg = 'import statement always raises %s' % self
+            raise ImportError(msg)
+        link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
     @property
     def args(self):
@@ -1094,6 +1075,18 @@
 
 class RaiseImplicit(Raise):
     """Signals an exception raised implicitly"""
+    def nomoreblocks(self, ctx):
+        w_exc = self.w_exc
+        if isinstance(w_exc.w_type, Constant):
+            exc_cls = w_exc.w_type.value
+        else:
+            exc_cls = Exception
+        msg = "implicit %s shouldn't occur" % exc_cls.__name__
+        w_type = Constant(AssertionError)
+        w_value = Constant(AssertionError(msg))
+        link = Link([w_type, w_value], ctx.graph.exceptblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
 
 class Break(FlowSignal):

From noreply at buildbot.pypy.org  Fri Dec  5 18:22:58 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 18:22:58 +0100 (CET)
Subject: [pypy-commit] cffi default: typo
Message-ID: <20141205172258.5C18C1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1586:dac8897a6503
Date: 2014-12-05 18:23 +0100
http://bitbucket.org/cffi/cffi/changeset/dac8897a6503/

Log:	typo

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -4765,7 +4765,7 @@
     CFieldObject *cf;
     Py_ssize_t offset;
 
-    if (!PyArg_ParseTuple(args, "O!O:typeof",
+    if (!PyArg_ParseTuple(args, "O!O:typeoffsetof",
                           &CTypeDescr_Type, &ct, &fieldname))
         return NULL;
 

From noreply at buildbot.pypy.org  Fri Dec  5 18:27:28 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 18:27:28 +0100 (CET)
Subject: [pypy-commit] creflect default: ffi.offsetof
Message-ID: <20141205172728.9F06C1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r172:233cdddde8f1
Date: 2014-12-05 18:27 +0100
http://bitbucket.org/cffi/creflect/changeset/233cdddde8f1/

Log:	ffi.offsetof

diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -255,7 +255,7 @@
     return (PyObject *)cd;
 }
 
-static PyObject *ffi_string(PyObject *self, PyObject *args)
+static PyObject *ffi_string(ZefFFIObject *self, PyObject *args)
 {
     CDataObject *cd;
     Py_ssize_t maxlen = -1;
@@ -317,10 +317,49 @@
     return NULL;
 }
 
+static PyObject *ffi_offsetof(ZefFFIObject *self, PyObject *args)
+{
+    PyObject *arg;
+    char *fieldname;
+    CTypeDescrObject *ct;
+    CFieldObject *cf;
+
+    if (!PyArg_ParseTuple(args, "Os:offsetof", &arg, &fieldname))
+        return NULL;
+
+    ct = _ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CTYPE);
+    if (ct == NULL)
+        return NULL;
+
+    if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) {
+        PyErr_Format(PyExc_TypeError,
+                     "expected a struct or union ctype, got '%s'",
+                     ct->ct_name);
+        return NULL;
+    }
+    if (ct->ct_stuff == NULL) {
+        PyErr_Format(PyExc_TypeError, "'%s' is incomplete", ct->ct_name);
+        return NULL;
+    }
+
+    cf = (CFieldObject *)PyDict_GetItemString(ct->ct_stuff, fieldname);
+    if (cf == NULL) {
+        PyErr_Format(PyExc_KeyError, "'%s' has got no field '%s'",
+                     ct->ct_name, fieldname);
+        return NULL;
+    }
+    if (cf->cf_bitshift >= 0) {
+        PyErr_SetString(PyExc_TypeError, "not supported for bitfields");
+        return NULL;
+    }
+    return PyInt_FromSsize_t(cf->cf_offset);
+}
+
 static PyMethodDef ffi_methods[] = {
     {"close_library", ffi_close_library,         METH_VARARGS | METH_STATIC},
     {"load_library",  (PyCFunction)ffi_load_library,
                                                  METH_VARARGS | METH_KEYWORDS},
+    {"offsetof",      (PyCFunction)ffi_offsetof, METH_VARARGS},
     {"new",           (PyCFunction)ffi_new,      METH_VARARGS},
     {"sizeof",        (PyCFunction)ffi_sizeof,   METH_O},
     {"string",        (PyCFunction)ffi_string,   METH_VARARGS},
diff --git a/zeffir/test/test_struct.py b/zeffir/test/test_struct.py
--- a/zeffir/test/test_struct.py
+++ b/zeffir/test/test_struct.py
@@ -10,8 +10,7 @@
     p.b += 1
     assert p.b == 0x600112234
 
-def test_addressof():
-    py.test.skip("in-progress")
+def test_offsetof():
     ffi, lib = support.compile_and_open('struct')
-    p = ffi.new('mystruct_t *')
-    ffi.addressof(p[0], 'a')
+    assert ffi.offsetof("mystruct_t", "a") == 0
+    assert ffi.offsetof("mystruct_t", "b") == ffi.sizeof("long")

From noreply at buildbot.pypy.org  Fri Dec  5 18:49:34 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 18:49:34 +0100 (CET)
Subject: [pypy-commit] creflect default: ffi.addressof
Message-ID: <20141205174934.31DFF1D29F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r173:832fea94ecfa
Date: 2014-12-05 18:40 +0100
http://bitbucket.org/cffi/creflect/changeset/832fea94ecfa/

Log:	ffi.addressof

diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -317,6 +317,26 @@
     return NULL;
 }
 
+static CFieldObject *_ffi_field(CTypeDescrObject *ct, const char *fieldname)
+{
+    CFieldObject *cf;
+    if (ct->ct_stuff == NULL) {
+        PyErr_Format(PyExc_TypeError, "'%s' is incomplete", ct->ct_name);
+        return NULL;
+    }
+    cf = (CFieldObject *)PyDict_GetItemString(ct->ct_stuff, fieldname);
+    if (cf == NULL) {
+        PyErr_Format(PyExc_KeyError, "'%s' has got no field '%s'",
+                     ct->ct_name, fieldname);
+        return NULL;
+    }
+    if (cf->cf_bitshift >= 0) {
+        PyErr_SetString(PyExc_TypeError, "not supported for bitfields");
+        return NULL;
+    }
+    return cf;
+}
+
 static PyObject *ffi_offsetof(ZefFFIObject *self, PyObject *args)
 {
     PyObject *arg;
@@ -337,25 +357,47 @@
                      ct->ct_name);
         return NULL;
     }
-    if (ct->ct_stuff == NULL) {
-        PyErr_Format(PyExc_TypeError, "'%s' is incomplete", ct->ct_name);
+    cf = _ffi_field(ct, fieldname);
+    if (cf == NULL)
+        return NULL;
+    return PyInt_FromSsize_t(cf->cf_offset);
+}
+
+static PyObject *ffi_addressof(ZefFFIObject *self, PyObject *args)
+{
+    char *fieldname = NULL;
+    CDataObject *cd;
+    CTypeDescrObject *ct;
+    Py_ssize_t offset;
+
+    if (!PyArg_ParseTuple(args, "O!|z:addressof", &CData_Type, &cd, &fieldname))
+        return NULL;
+
+    ct = cd->c_type;
+    if (fieldname != NULL && ct->ct_flags & CT_POINTER)
+        ct = ct->ct_itemdescr;
+
+    if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) {
+        PyErr_Format(PyExc_TypeError,
+                     "expected a struct or union cdata, got '%s'",
+                     ct->ct_name);
         return NULL;
     }
 
-    cf = (CFieldObject *)PyDict_GetItemString(ct->ct_stuff, fieldname);
-    if (cf == NULL) {
-        PyErr_Format(PyExc_KeyError, "'%s' has got no field '%s'",
-                     ct->ct_name, fieldname);
-        return NULL;
+    if (fieldname == NULL) {
+        offset = 0;
     }
-    if (cf->cf_bitshift >= 0) {
-        PyErr_SetString(PyExc_TypeError, "not supported for bitfields");
-        return NULL;
+    else {
+        CFieldObject *cf = _ffi_field(ct, fieldname);
+        if (cf == NULL)
+            return NULL;
+        offset = cf->cf_offset;
     }
-    return PyInt_FromSsize_t(cf->cf_offset);
+    return new_simple_cdata(cd->c_data + offset, ct);
 }
 
 static PyMethodDef ffi_methods[] = {
+    {"addressof",     (PyCFunction)ffi_addressof,METH_VARARGS},
     {"close_library", ffi_close_library,         METH_VARARGS | METH_STATIC},
     {"load_library",  (PyCFunction)ffi_load_library,
                                                  METH_VARARGS | METH_KEYWORDS},
diff --git a/zeffir/test/test_struct.py b/zeffir/test/test_struct.py
--- a/zeffir/test/test_struct.py
+++ b/zeffir/test/test_struct.py
@@ -14,3 +14,15 @@
     ffi, lib = support.compile_and_open('struct')
     assert ffi.offsetof("mystruct_t", "a") == 0
     assert ffi.offsetof("mystruct_t", "b") == ffi.sizeof("long")
+
+def test_addressof():
+    ffi, lib = support.compile_and_open('struct')
+    p = ffi.new("mystruct_t *")
+    assert ffi.addressof(p[0]) == p
+    assert ffi.addressof(p[0], None) == p
+    assert ffi.addressof(p[0], "a") == p
+    assert ffi.addressof(p[0], "b") != p
+    assert ffi.addressof(p[0], "b") == ffi.addressof(p, "b")
+    assert ffi.addressof(p, "a") == p
+    py.test.raises(TypeError, ffi.addressof, p)
+    py.test.raises(TypeError, ffi.addressof, p, None)

From noreply at buildbot.pypy.org  Fri Dec  5 18:49:35 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 18:49:35 +0100 (CET)
Subject: [pypy-commit] creflect default: addressof(Lib, "some_global_var")
Message-ID: <20141205174935.7A2B61D29F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r174:a9718f1fb634
Date: 2014-12-05 18:49 +0100
http://bitbucket.org/cffi/creflect/changeset/a9718f1fb634/

Log:	addressof(Lib, "some_global_var")

diff --git a/zeffir/cglob.c b/zeffir/cglob.c
--- a/zeffir/cglob.c
+++ b/zeffir/cglob.c
@@ -61,3 +61,8 @@
 {
     return convert_from_object(zgs->zgs_data, zgs->zgs_type, obj);
 }
+
+static PyObject *addressof_global_var(ZefGlobSupportObject *zgs)
+{
+    return new_simple_cdata(zgs->zgs_data, zgs->zgs_type);
+}
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -365,35 +365,69 @@
 
 static PyObject *ffi_addressof(ZefFFIObject *self, PyObject *args)
 {
+    PyObject *obj;
     char *fieldname = NULL;
-    CDataObject *cd;
-    CTypeDescrObject *ct;
-    Py_ssize_t offset;
 
-    if (!PyArg_ParseTuple(args, "O!|z:addressof", &CData_Type, &cd, &fieldname))
+    if (!PyArg_ParseTuple(args, "O|z:addressof", &obj, &fieldname))
         return NULL;
 
-    ct = cd->c_type;
-    if (fieldname != NULL && ct->ct_flags & CT_POINTER)
-        ct = ct->ct_itemdescr;
+    if (CData_Check(obj)) {
+        CDataObject *cd = (CDataObject *)obj;
+        CTypeDescrObject *ct;
+        Py_ssize_t offset;
 
-    if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) {
+        ct = cd->c_type;
+        if (fieldname != NULL && ct->ct_flags & CT_POINTER)
+            ct = ct->ct_itemdescr;
+
+        if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) {
+            PyErr_Format(PyExc_TypeError,
+                         "expected a struct or union cdata, got '%s'",
+                         ct->ct_name);
+            return NULL;
+        }
+
+        if (fieldname == NULL) {
+            offset = 0;
+        }
+        else {
+            CFieldObject *cf = _ffi_field(ct, fieldname);
+            if (cf == NULL)
+                return NULL;
+            offset = cf->cf_offset;
+        }
+        return new_simple_cdata(cd->c_data + offset, ct);
+    }
+    else if (ZefLib_Check(obj)) {
+        PyObject *attr, *name;
+
+        if (fieldname == NULL) {
+            PyErr_SetString(PyExc_TypeError, "addressof(Lib, fieldname) "
+                            "cannot be used with only one argument");
+            return NULL;
+        }
+        name = PyString_FromString(fieldname);
+        if (name == NULL)
+            return NULL;
+        attr = lib_findattr((ZefLibObject *)obj, name);
+        Py_DECREF(name);
+        if (attr == NULL)
+            return NULL;
+
+        if (ZefGlobSupport_Check(attr))
+            return addressof_global_var((ZefGlobSupportObject *)attr);
+
         PyErr_Format(PyExc_TypeError,
-                     "expected a struct or union cdata, got '%s'",
-                     ct->ct_name);
+                     "cannot take the address of constant '%s'",
+                     fieldname);
         return NULL;
     }
-
-    if (fieldname == NULL) {
-        offset = 0;
+    else {
+        PyErr_SetString(PyExc_TypeError, "addressof() first argument must be "
+                        "a cdata struct or union, a pointer to one, or a Lib "
+                        "object");
+        return NULL;
     }
-    else {
-        CFieldObject *cf = _ffi_field(ct, fieldname);
-        if (cf == NULL)
-            return NULL;
-        offset = cf->cf_offset;
-    }
-    return new_simple_cdata(cd->c_data + offset, ct);
 }
 
 static PyMethodDef ffi_methods[] = {
diff --git a/zeffir/lib_obj.c b/zeffir/lib_obj.c
--- a/zeffir/lib_obj.c
+++ b/zeffir/lib_obj.c
@@ -12,6 +12,8 @@
     PyObject *l_libname_obj;  /* same, as a PyObject */
 };
 
+#define ZefLib_Check(ob)  ((Py_TYPE(ob) == &ZefLib_Type))
+
 static void lib_dealloc(ZefLibObject *lib)
 {
     (void)lib_close(lib);
diff --git a/zeffir/test/test_global.py b/zeffir/test/test_global.py
--- a/zeffir/test/test_global.py
+++ b/zeffir/test/test_global.py
@@ -32,3 +32,10 @@
     assert hash(lib.myglobptr) == hash(lib.myglobptr)
     lib.myglobptr = ffi.NULL
     assert lib.myglobptr == ffi.NULL
+
+def test_addressof():
+    ffi, lib = support.compile_and_open('global')
+    p = ffi.addressof(lib, 'myglob')
+    assert p == lib.myglobptr
+    lib.myglobptr = ffi.NULL
+    assert p != lib.myglobptr

From noreply at buildbot.pypy.org  Fri Dec  5 19:09:23 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 19:09:23 +0100 (CET)
Subject: [pypy-commit] creflect default: more fixes to addressof()
Message-ID: <20141205180923.AB3621D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r175:6fc30b084312
Date: 2014-12-05 19:09 +0100
http://bitbucket.org/cffi/creflect/changeset/6fc30b084312/

Log:	more fixes to addressof()

diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -11,6 +11,8 @@
 
 } ZefFuncSupportObject;
 
+#define ZefFuncSupport_Check(ob)  (Py_TYPE(ob) == &ZefFuncSupport_Type)
+
 static void zef_func_support_dealloc(ZefFuncSupportObject *zfs)
 {
     int i;
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -400,6 +400,7 @@
     }
     else if (ZefLib_Check(obj)) {
         PyObject *attr, *name;
+        char *reason;
 
         if (fieldname == NULL) {
             PyErr_SetString(PyExc_TypeError, "addressof(Lib, fieldname) "
@@ -409,17 +410,23 @@
         name = PyString_FromString(fieldname);
         if (name == NULL)
             return NULL;
-        attr = lib_findattr((ZefLibObject *)obj, name);
+        attr = lib_findattr((ZefLibObject *)obj, name, ZefError);
         Py_DECREF(name);
         if (attr == NULL)
             return NULL;
 
-        if (ZefGlobSupport_Check(attr))
+        if (ZefGlobSupport_Check(attr)) {
             return addressof_global_var((ZefGlobSupportObject *)attr);
+        }
 
-        PyErr_Format(PyExc_TypeError,
-                     "cannot take the address of constant '%s'",
-                     fieldname);
+        if (PyCFunction_Check(attr))
+            reason = "declare that function as a function pointer instead";
+        else
+            reason = "numeric constants don't have addresses";
+
+        PyErr_Format(PyExc_NotImplementedError,
+                     "cannot take the address of '%s' (%s)",
+                     fieldname, reason);
         return NULL;
     }
     else {
diff --git a/zeffir/lib_obj.c b/zeffir/lib_obj.c
--- a/zeffir/lib_obj.c
+++ b/zeffir/lib_obj.c
@@ -27,7 +27,7 @@
                              lib->l_dl_lib == NULL ? " (closed)" : "");
 }
 
-static PyObject *lib_findattr(ZefLibObject *lib, PyObject *name)
+static PyObject *lib_findattr(ZefLibObject *lib, PyObject *name, PyObject *exc)
 {
     /* does not return a new reference! */
 
@@ -38,7 +38,7 @@
 
     PyObject *x = PyDict_GetItem(lib->l_dict, name);
     if (x == NULL) {
-        PyErr_Format(PyExc_AttributeError,
+        PyErr_Format(exc,
                      "lib '%.200s' has no function,"
                      " global variable or constant '%.200s'",
                      lib->l_libname,
@@ -50,7 +50,7 @@
 
 static PyObject *lib_getattr(ZefLibObject *lib, PyObject *name)
 {
-    PyObject *x = lib_findattr(lib, name);
+    PyObject *x = lib_findattr(lib, name, PyExc_AttributeError);
     if (x == NULL)
         return NULL;
 
@@ -63,7 +63,7 @@
 
 static int lib_setattr(ZefLibObject *lib, PyObject *name, PyObject *val)
 {
-    PyObject *x = lib_findattr(lib, name);
+    PyObject *x = lib_findattr(lib, name, PyExc_AttributeError);
     if (x == NULL)
         return -1;
 
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -44,3 +44,7 @@
     for i in range(50):
         p = ffi.new("char[]", "12345678" * i)
         assert ffi.string(p) == "12345678" * i
+
+def test_addressof_missing_glob():
+    ffi, lib = support.compile_and_open('basic')
+    py.test.raises(ffi.error, ffi.addressof, lib, "some_missing_name")
diff --git a/zeffir/test/test_function.py b/zeffir/test/test_function.py
--- a/zeffir/test/test_function.py
+++ b/zeffir/test/test_function.py
@@ -26,3 +26,7 @@
     p = ffi.new("char[]", 10)
     lib.returning_nothing(p)
     assert ffi.string(p) == "OK"
+
+def test_addressof_function():
+    ffi, lib = support.compile_and_open('function')
+    py.test.raises(NotImplementedError, ffi.addressof, lib, 'simple_function')
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -15,8 +15,8 @@
 #include "cdata.c"
 #include "cglob.c"
 #include "lib_obj.c"
+#include "cfunc.c"
 #include "ffi_obj.c"
-#include "cfunc.c"
 #include "builder.c"
 #include "../creflect/creflect_cdecl.c"
 

From noreply at buildbot.pypy.org  Fri Dec  5 19:38:48 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 19:38:48 +0100 (CET)
Subject: [pypy-commit] creflect default: fix ffi.addressof(). Also allows
 ffi.new("struct_type") directly, 
Message-ID: <20141205183848.68C971D2DD3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r176:620bef5903b4
Date: 2014-12-05 19:39 +0100
http://bitbucket.org/cffi/creflect/changeset/620bef5903b4/

Log:	fix ffi.addressof(). Also allows ffi.new("struct_type") directly,
	as an extension that seems to make sense for now

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -45,13 +45,18 @@
     return result;
 }
 
-static CTypeDescrObject *get_cached_type(_crx_builder_t *cb, PyObject *name_obj)
+static PyObject *get_types_dict(_crx_builder_t *cb)
+{
+    return ((zeffir_builder_t *)cb)->types_dict;
+}
+
+static CTypeDescrObject *get_cached_type(PyObject *types_dict,
+                                         PyObject *name_obj)
 {
     /* Look up the type called 'name_obj' and check that it's a type with the
        right flags.  Returns either the type or NULL, without changing
        the reference counts.  Never raises an exception.
     */
-    PyObject *types_dict = ((zeffir_builder_t *)cb)->types_dict;
     PyObject *x;
 
     assert(name_obj != NULL);
@@ -62,13 +67,13 @@
         return (CTypeDescrObject *)x;
 }
 
-static void put_cached_type(_crx_builder_t *cb, PyObject *name_obj,
+static int put_cached_type(PyObject *types_dict, PyObject *name_obj,
                             CTypeDescrObject *ct)
 {
-    PyObject *types_dict = ((zeffir_builder_t *)cb)->types_dict;
-    PyDict_SetItem(types_dict, name_obj, (PyObject *)ct);
+    int err = PyDict_SetItem(types_dict, name_obj, (PyObject *)ct);
     Py_DECREF(ct);   /* still a reference in the dict,
                         unless PyDict_SetItem failed */
+    return err;
 }
 
 static _crx_type_t *_zef_primitive(_crx_builder_t *cb, Py_ssize_t size,
@@ -84,7 +89,7 @@
     if (name_obj == NULL)
         return NULL;
 
-    ct = get_cached_type(cb, name_obj);
+    ct = get_cached_type(get_types_dict(cb), name_obj);
     if (ct && (ct->ct_flags == flags) && (ct->ct_size == size))
         goto done;
 
@@ -95,7 +100,8 @@
     ct->ct_size = size;
     ct->ct_flags = flags;
 
-    put_cached_type(cb, name_obj, ct);
+    if (put_cached_type(get_types_dict(cb), name_obj, ct) < 0)
+        ct = NULL;
 
  done:
     Py_DECREF(name_obj);
@@ -207,7 +213,7 @@
                == extra_len);
     }
 
-    ct = get_cached_type(cb, name_obj);
+    ct = get_cached_type(get_types_dict(cb), name_obj);
     if (ct && (ct->ct_flags == CT_FUNCTION))
         goto done;
 
@@ -219,7 +225,7 @@
     ct->ct_flags = CT_FUNCTION;
     /* XXX more... */
 
-    put_cached_type(cb, name_obj, ct);
+    put_cached_type(get_types_dict(cb), name_obj, ct);
 
  done:
     Py_DECREF(name_obj);
@@ -241,12 +247,9 @@
     return _zef_function(cb, ret, args, nargs, 1);
 }
 
-static _crx_type_t *zef_get_pointer_type(_crx_builder_t *cb,
-                                         _crx_type_t *totype, int toquals)
+static CTypeDescrObject *fetch_pointer_type(PyObject *types_dict,
+                                            CTypeDescrObject *totype)
 {
-    if (PyErr_Occurred())
-        return NULL;
-
     const char *extra;
     PyObject *name_obj;
     CTypeDescrObject *ct;
@@ -260,7 +263,7 @@
     if (name_obj == NULL)
         return NULL;
 
-    ct = get_cached_type(cb, name_obj);
+    ct = get_cached_type(types_dict, name_obj);
     if (ct && (ct->ct_flags & CT_POINTER))
         goto done;
 
@@ -280,13 +283,22 @@
          totype->ct_size == sizeof(char)))
         ct->ct_flags |= CT_CAST_ANYTHING;   /* 'void *' or 'char *' only */
 
-    put_cached_type(cb, name_obj, ct);
+    put_cached_type(types_dict, name_obj, ct);
 
  done:
     Py_DECREF(name_obj);
     return ct;
 }
 
+static _crx_type_t *zef_get_pointer_type(_crx_builder_t *cb,
+                                         _crx_type_t *totype, int toquals)
+{
+    if (PyErr_Occurred())
+        return NULL;
+
+    return fetch_pointer_type(get_types_dict(cb), totype);
+}
+
 static _crx_type_t *_zef_array_type(_crx_builder_t *cb, _crx_type_t *ctitem,
                                     size_t len, CTypeDescrObject *ctptr)
 {
@@ -318,7 +330,7 @@
         return NULL;
 
     if (cb != NULL) {
-        ct = get_cached_type(cb, name_obj);
+        ct = get_cached_type(get_types_dict(cb), name_obj);
         if (ct && (ct->ct_flags & CT_ARRAY))
             goto done;
     }
@@ -356,7 +368,7 @@
             Py_INCREF(ct);
             ctptr->ct_stuff = (PyObject *)ct;
         }
-        put_cached_type(cb, name_obj, ct);
+        put_cached_type(get_types_dict(cb), name_obj, ct);
     }
     else {
         assert(length < 0);
@@ -400,7 +412,7 @@
     if (name_obj == NULL)
         return NULL;
 
-    ct = get_cached_type(cb, name_obj);
+    ct = get_cached_type(get_types_dict(cb), name_obj);
     if (ct && (ct->ct_flags == flag))
         goto done;
 
@@ -442,7 +454,7 @@
     if (name_obj == NULL)
         return result;
 
-    result.type = get_cached_type(cb, name_obj);
+    result.type = get_cached_type(get_types_dict(cb), name_obj);
     if (result.type == NULL)
         PyErr_Format(ZefError, "undefined type: '%s'", name);
     return result;
diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1207,6 +1207,7 @@
     if (ct == NULL)
         return NULL;
 
+    assert(ct->ct_flags & CT_POINTER);
     if (ct->ct_stuff == NULL) {
         if (fill_array_type(ct) < 0)
             return NULL;
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -191,9 +191,9 @@
         return NULL;
 
     explicitlength = -1;
-    if (ct->ct_flags & CT_POINTER) {
+    if (ct->ct_flags & (CT_POINTER | CT_STRUCT | CT_UNION)) {
         dataoffset = offsetof(CDataObject_own_nolength, alignment);
-        ctitem = ct->ct_itemdescr;
+        ctitem = (ct->ct_flags & CT_POINTER) ? ct->ct_itemdescr : ct;
         datasize = ctitem->ct_size;
         if (datasize < 0) {
             PyErr_Format(PyExc_TypeError,
@@ -395,7 +395,11 @@
             if (cf == NULL)
                 return NULL;
             offset = cf->cf_offset;
+            ct = cf->cf_type;
         }
+        ct = fetch_pointer_type(self->types_dict, ct);
+        if (ct == NULL)
+            return NULL;
         return new_simple_cdata(cd->c_data + offset, ct);
     }
     else if (ZefLib_Check(obj)) {
diff --git a/zeffir/test/struct.crx b/zeffir/test/struct.crx
--- a/zeffir/test/struct.crx
+++ b/zeffir/test/struct.crx
@@ -3,6 +3,11 @@
     long b;
 } mystruct_t;
 
+typedef struct {
+    int foo[5];
+    //int bar[];
+} myarrstr_t;
+
 
 // CREFLECT: start
 
@@ -10,4 +15,9 @@
     int a, b;
 } mystruct_t;
 
+typedef struct {
+    int foo[5];
+    //int bar[];
+} myarrstr_t;
+
 // CREFLECT: end
diff --git a/zeffir/test/test_struct.py b/zeffir/test/test_struct.py
--- a/zeffir/test/test_struct.py
+++ b/zeffir/test/test_struct.py
@@ -6,6 +6,7 @@
     ffi, lib = support.compile_and_open('struct')
     p = ffi.new('mystruct_t *', [-5, 0x600112233])
     assert p.a == -5
+    assert p[0].a == -5
     assert p.b == 0x600112233
     p.b += 1
     assert p.b == 0x600112234
@@ -19,10 +20,27 @@
     ffi, lib = support.compile_and_open('struct')
     p = ffi.new("mystruct_t *")
     assert ffi.addressof(p[0]) == p
+    assert ffi.typeof(ffi.addressof(p[0])) == ffi.typeof("mystruct_t *")
     assert ffi.addressof(p[0], None) == p
     assert ffi.addressof(p[0], "a") == p
+    assert ffi.typeof(ffi.addressof(p[0], "a")) == ffi.typeof("int *")
     assert ffi.addressof(p[0], "b") != p
     assert ffi.addressof(p[0], "b") == ffi.addressof(p, "b")
+    assert ffi.typeof(ffi.addressof(p[0], "b")) == ffi.typeof("long *")
     assert ffi.addressof(p, "a") == p
     py.test.raises(TypeError, ffi.addressof, p)
     py.test.raises(TypeError, ffi.addressof, p, None)
+    #
+    p = ffi.new("myarrstr_t *")
+    assert ffi.typeof(ffi.addressof(p, "foo")) == ffi.typeof("int(*)[5]")
+    assert ffi.typeof(p.foo) == ffi.typeof("int[5]")
+    #assert ffi.typeof(ffi.addressof(p, "bar")) == ffi.typeof("int(*)[]")
+
+def test_new_nonptr_struct():
+    ffi, lib = support.compile_and_open('struct')
+    s = ffi.new("mystruct_t", [-5, 0x600112233])
+    assert s.a == -5
+    assert s.b == 0x600112233
+    s.b += 1
+    assert s.b == 0x600112234
+    assert ffi.addressof(s)[0] == s
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -21,3 +21,5 @@
 static int convert_from_object_bitfield(char *, CFieldObject *, PyObject *);
 
 static int fill_array_type(CTypeDescrObject *ctptr);
+static CTypeDescrObject *fetch_pointer_type(PyObject *types_dict,
+                                            CTypeDescrObject *totype);

From noreply at buildbot.pypy.org  Fri Dec  5 19:44:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 19:44:44 +0100 (CET)
Subject: [pypy-commit] creflect default: test and fix for
	structs-with-incomplete-array
Message-ID: <20141205184444.488D51D257D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r177:c652e11796b0
Date: 2014-12-05 19:45 +0100
http://bitbucket.org/cffi/creflect/changeset/c652e11796b0/

Log:	test and fix for structs-with-incomplete-array

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -471,7 +471,7 @@
         else:
             star_p1 = None
         t1, q1 = self.item.inspect_type(block, inspect, qualifiers)
-        if star_p1 is not None:
+        if star_p1 is not None and self.length is not None:
             expr = 'cb->get_array_type(cb, %s, sizeof(%s) / sizeof(*%s))' % (
                 t1, star_p1, star_p1)
         else:
diff --git a/creflect/test/codegen/struct-004b.c b/creflect/test/codegen/struct-004b.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/struct-004b.c
@@ -0,0 +1,54 @@
+struct foo_s {
+    int nn;
+    int aa[];
+};
+
+# ____________________________________________________________
+
+void teststruct_004b(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2, *t3, *t4;
+    _crx_field_t d1[2];
+    t1 = cb->get_struct_type(cb, "foo_s");
+    {
+        struct foo_s *p1;
+        char b[sizeof(p1->nn)];
+        size_t o = ((char *)&((struct foo_s *)0)->nn) - (char *)0;  /* check that 'struct foo_s::nn' is not an array */
+        p1 = (void *)(((char *)b) - o);
+        (void)(p1->nn << 1);  /* check that 'struct foo_s::nn' is an integer type */
+        p1->nn = -1;  /* check that 'struct foo_s::nn' is not declared 'const' */
+        t2 = _CRX_INT_TYPE(cb, p1->nn, _crx_sc_int);
+        d1[0].name = "nn";
+        d1[0].type = t2;
+        d1[0].qualifiers = 0;
+        d1[0].offset = o;
+        d1[0].numbits = -1;
+        d1[0].bitshift = -1;
+    }
+    {
+        struct foo_s *p1;
+        char b[sizeof(*p1->aa)];  /* check that 'struct foo_s::aa[]' is a valid type */
+        size_t o = offsetof(struct foo_s, aa[0]);  /* check that 'struct foo_s::aa' is an array */
+        p1 = (void *)(((char *)b) - o);
+        if ((void *)&p1->aa != (void *)p1->aa) {
+            cb->error(cb, "type 'struct foo_s::aa' is not an array, but a pointer type");
+            return;
+        }
+        (void)(*p1->aa << 1);  /* check that 'struct foo_s::aa[]' is an integer type */
+        *p1->aa = -1;  /* check that 'struct foo_s::aa[]' is not declared 'const' */
+        t3 = _CRX_INT_TYPE(cb, *p1->aa, _crx_sc_int);
+        t4 = cb->get_incomplete_array_type(cb, t3);
+        d1[1].name = "aa";
+        d1[1].type = t4;
+        d1[1].qualifiers = 0;
+        d1[1].offset = o;
+        d1[1].numbits = -1;
+        d1[1].bitshift = -1;
+    }
+    cb->complete(cb, t1, sizeof(struct foo_s),
+                 ((char *)&((struct{char a; struct foo_s b;} *)0)->b) - (char *)0,
+                 d1, 2);
+#expect STRUCT foo_s:
+#expect | nn: int
+#expect | aa: ARRAY[] int
+}

From noreply at buildbot.pypy.org  Fri Dec  5 19:46:04 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 19:46:04 +0100 (CET)
Subject: [pypy-commit] creflect default: uncomment these tests, now passing
Message-ID: <20141205184604.AF42E1D257D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r178:a3463cd2c8f0
Date: 2014-12-05 19:46 +0100
http://bitbucket.org/cffi/creflect/changeset/a3463cd2c8f0/

Log:	uncomment these tests, now passing

diff --git a/zeffir/test/struct.crx b/zeffir/test/struct.crx
--- a/zeffir/test/struct.crx
+++ b/zeffir/test/struct.crx
@@ -5,7 +5,8 @@
 
 typedef struct {
     int foo[5];
-    //int bar[];
+    int bar[7];
+    int baz[];
 } myarrstr_t;
 
 
@@ -17,7 +18,8 @@
 
 typedef struct {
     int foo[5];
-    //int bar[];
+    int bar[];
+    int baz[];
 } myarrstr_t;
 
 // CREFLECT: end
diff --git a/zeffir/test/test_struct.py b/zeffir/test/test_struct.py
--- a/zeffir/test/test_struct.py
+++ b/zeffir/test/test_struct.py
@@ -32,9 +32,12 @@
     py.test.raises(TypeError, ffi.addressof, p, None)
     #
     p = ffi.new("myarrstr_t *")
+    assert ffi.typeof(p.foo) == ffi.typeof("int[5]")
+    assert ffi.typeof(p.bar) == ffi.typeof("int *")
+    assert ffi.typeof(p.baz) == ffi.typeof("int *")
     assert ffi.typeof(ffi.addressof(p, "foo")) == ffi.typeof("int(*)[5]")
-    assert ffi.typeof(p.foo) == ffi.typeof("int[5]")
-    #assert ffi.typeof(ffi.addressof(p, "bar")) == ffi.typeof("int(*)[]")
+    assert ffi.typeof(ffi.addressof(p, "bar")) == ffi.typeof("int(*)[]")
+    assert ffi.typeof(ffi.addressof(p, "baz")) == ffi.typeof("int(*)[]")
 
 def test_new_nonptr_struct():
     ffi, lib = support.compile_and_open('struct')

From noreply at buildbot.pypy.org  Fri Dec  5 21:08:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 21:08:44 +0100 (CET)
Subject: [pypy-commit] creflect default: fix the error message
Message-ID: <20141205200844.9C58C1D2E18@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r179:8cf4c557d1f1
Date: 2014-12-05 20:49 +0100
http://bitbucket.org/cffi/creflect/changeset/8cf4c557d1f1/

Log:	fix the error message

diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -231,8 +231,8 @@
     }
     else {
         PyErr_Format(PyExc_TypeError,
-                     "expected a pointer or array ctype, got '%s'",
-                     ct->ct_name);
+                     "expected a pointer, struct, union, or array ctype, "
+                     "got '%s'", ct->ct_name);
         return NULL;
     }
 

From noreply at buildbot.pypy.org  Fri Dec  5 21:08:45 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 21:08:45 +0100 (CET)
Subject: [pypy-commit] creflect default: copy the logic for ffi.cast()
Message-ID: <20141205200845.BA32C1D2E18@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r180:9eb2b89d1adc
Date: 2014-12-05 21:09 +0100
http://bitbucket.org/cffi/creflect/changeset/9eb2b89d1adc/

Log:	copy the logic for ffi.cast()

diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -255,6 +255,208 @@
     return (PyObject *)cd;
 }
 
+static int _my_PyObject_AsBool(PyObject *ob)
+{
+    /* convert and cast a Python object to a boolean.  Accept an integer
+       or a float object, up to a CData 'long double'. */
+    PyObject *io;
+    PyNumberMethods *nb;
+    int res;
+
+#if PY_MAJOR_VERSION < 3
+    if (PyInt_Check(ob)) {
+        return PyInt_AS_LONG(ob) != 0;
+    }
+    else
+#endif
+    if (PyLong_Check(ob)) {
+        return _PyLong_Sign(ob) != 0;
+    }
+    else if (PyFloat_Check(ob)) {
+        return PyFloat_AS_DOUBLE(ob) != 0.0;
+    }
+    else if (CData_Check(ob)) {
+        CDataObject *cd = (CDataObject *)ob;
+        if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
+            /*READ(cd->c_data, cd->c_type->ct_size)*/
+            if (cd->c_type->ct_flags & CT_IS_LONGDOUBLE) {
+                /* 'long double' objects: return the answer directly */
+                return read_raw_longdouble_data(cd->c_data) != 0.0;
+            }
+            else {
+                /* 'float'/'double' objects: return the answer directly */
+                return read_raw_float_data(cd->c_data,
+                                           cd->c_type->ct_size) != 0.0;
+            }
+        }
+    }
+    nb = ob->ob_type->tp_as_number;
+    if (nb == NULL || (nb->nb_float == NULL && nb->nb_int == NULL)) {
+        PyErr_SetString(PyExc_TypeError, "integer/float expected");
+        return -1;
+    }
+    if (nb->nb_float && !CData_Check(ob))
+        io = (*nb->nb_float) (ob);
+    else
+        io = (*nb->nb_int) (ob);
+    if (io == NULL)
+        return -1;
+
+    if (PyIntOrLong_Check(io) || PyFloat_Check(io)) {
+        res = _my_PyObject_AsBool(io);
+    }
+    else {
+        PyErr_SetString(PyExc_TypeError, "integer/float conversion failed");
+        res = -1;
+    }
+    Py_DECREF(io);
+    return res;
+}
+
+static CDataObject *cast_to_integer_or_char(CTypeDescrObject *ct, PyObject *ob)
+{
+    unsigned PY_LONG_LONG value;
+    CDataObject *cd;
+
+    if (CData_Check(ob) &&
+            ((CDataObject *)ob)->c_type->ct_flags & (CT_POINTER | CT_ARRAY)) {
+        value = (Py_intptr_t)((CDataObject *)ob)->c_data;
+    }
+    else if (PyString_Check(ob)) {
+        if (PyString_GET_SIZE(ob) != 1) {
+            PyErr_Format(PyExc_TypeError,
+                         "cannot cast string of length %zd to ctype '%s'",
+                         PyString_GET_SIZE(ob), ct->ct_name);
+            return NULL;
+        }
+        value = (unsigned char)PyString_AS_STRING(ob)[0];
+    }
+    else if (ct->ct_flags & CT_IS_BOOL) {
+        int res = _my_PyObject_AsBool(ob);
+        if (res < 0)
+            return NULL;
+        value = res;
+    }
+    else {
+        value = _my_PyLong_AsUnsignedLongLong(ob, 0);
+        if (value == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())
+            return NULL;
+    }
+    if (ct->ct_flags & CT_IS_BOOL)
+        value = !!value;
+    cd = _new_casted_primitive(ct);
+    if (cd != NULL)
+        write_raw_integer_data(cd->c_data, value, ct->ct_size);
+    return cd;
+}
+
+static PyObject *ffi_cast(ZefFFIObject *self, PyObject *args)
+{
+    CTypeDescrObject *ct;
+    CDataObject *cd;
+    PyObject *ob, *arg;
+    if (!PyArg_ParseTuple(args, "OO:cast", &arg, &ob))
+        return NULL;
+
+    ct = _ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CTYPE);
+    if (ct == NULL)
+        return NULL;
+
+    if (ct->ct_flags & (CT_POINTER | CT_ARRAY) &&
+        ct->ct_size >= 0) {
+        /* cast to a pointer or to an array.
+           Note that casting to an array is an extension to the C language,
+           which seems to be necessary in order to sanely get a
+           <cdata 'int[3]'> at some address. */
+        unsigned PY_LONG_LONG value;
+
+        if (CData_Check(ob)) {
+            CDataObject *cdsrc = (CDataObject *)ob;
+            if (cdsrc->c_type->ct_flags & (CT_POINTER | CT_ARRAY)) {
+                return new_simple_cdata(cdsrc->c_data, ct);
+            }
+        }
+        value = _my_PyLong_AsUnsignedLongLong(ob, 0);
+        if (value == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())
+            return NULL;
+        return new_simple_cdata((char *)(Py_intptr_t)value, ct);
+    }
+    else if (ct->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED
+                             |CT_PRIMITIVE_CHAR)) {
+        /* cast to an integer type or a char */
+        return (PyObject *)cast_to_integer_or_char(ct, ob);
+    }
+    else if (ct->ct_flags & CT_PRIMITIVE_FLOAT) {
+        /* cast to a float */
+        double value;
+        PyObject *io;
+
+        if (CData_Check(ob)) {
+            CDataObject *cdsrc = (CDataObject *)ob;
+
+            if (!(cdsrc->c_type->ct_flags & CT_PRIMITIVE_ANY))
+                goto cannot_cast;
+            io = convert_to_object(cdsrc->c_data, cdsrc->c_type);
+            if (io == NULL)
+                return NULL;
+        }
+        else {
+            io = ob;
+            Py_INCREF(io);
+        }
+
+        if (PyBytes_Check(io)) {
+            if (PyBytes_GET_SIZE(io) != 1) {
+                Py_DECREF(io);
+                goto cannot_cast;
+            }
+            value = (unsigned char)PyBytes_AS_STRING(io)[0];
+        }
+        else if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
+                 CData_Check(io) &&
+                 (((CDataObject *)io)->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
+            long double lvalue;
+            char *data = ((CDataObject *)io)->c_data;
+            /*READ(data, sizeof(long double)*/
+            lvalue = read_raw_longdouble_data(data);
+            cd = _new_casted_primitive(ct);
+            if (cd != NULL)
+                write_raw_longdouble_data(cd->c_data, lvalue);
+            return (PyObject *)cd;
+        }
+        else {
+            value = PyFloat_AsDouble(io);
+        }
+        Py_DECREF(io);
+        if (value == -1.0 && PyErr_Occurred())
+            return NULL;
+
+        cd = _new_casted_primitive(ct);
+        if (cd != NULL) {
+            if (!(ct->ct_flags & CT_IS_LONGDOUBLE))
+                write_raw_float_data(cd->c_data, value, ct->ct_size);
+            else
+                write_raw_longdouble_data(cd->c_data, (long double)value);
+        }
+        return (PyObject *)cd;
+    }
+    else {
+        PyErr_Format(PyExc_TypeError, "cannot cast to ctype '%s'",
+                     ct->ct_name);
+        return NULL;
+    }
+
+ cannot_cast:
+    if (CData_Check(ob))
+        PyErr_Format(PyExc_TypeError, "cannot cast ctype '%s' to ctype '%s'",
+                     ((CDataObject *)ob)->c_type->ct_name, ct->ct_name);
+    else
+        PyErr_Format(PyExc_TypeError,
+                     "cannot cast %.200s object to ctype '%s'",
+                     Py_TYPE(ob)->tp_name, ct->ct_name);
+    return NULL;
+}
+
 static PyObject *ffi_string(ZefFFIObject *self, PyObject *args)
 {
     CDataObject *cd;
@@ -443,6 +645,7 @@
 
 static PyMethodDef ffi_methods[] = {
     {"addressof",     (PyCFunction)ffi_addressof,METH_VARARGS},
+    {"cast",          (PyCFunction)ffi_cast,     METH_VARARGS},
     {"close_library", ffi_close_library,         METH_VARARGS | METH_STATIC},
     {"load_library",  (PyCFunction)ffi_load_library,
                                                  METH_VARARGS | METH_KEYWORDS},
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -48,3 +48,9 @@
 def test_addressof_missing_glob():
     ffi, lib = support.compile_and_open('basic')
     py.test.raises(ffi.error, ffi.addressof, lib, "some_missing_name")
+
+def test_cast():
+    ffi = support.new_ffi()
+    p = ffi.new("char[]", "----abcd")
+    q = ffi.cast("int *", p)
+    assert q[1] in [0x61626364, 0x64636261]

From noreply at buildbot.pypy.org  Fri Dec  5 21:19:54 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 21:19:54 +0100 (CET)
Subject: [pypy-commit] pypy default: cleanup references to numpypy
Message-ID: <20141205201954.694511D257D@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74839:642a5d4965d6
Date: 2014-12-05 15:12 -0500
http://bitbucket.org/pypy/pypy/changeset/642a5d4965d6/

Log:	cleanup references to numpypy

diff too long, truncating to 2000 out of 3730 lines

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -59,9 +59,9 @@
     _mixin_ = True
 
     def reduce(self, space):
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
+        _numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(_numpypy, MixedModule)
+        multiarray = _numpypy.get("multiarray")
         assert isinstance(multiarray, MixedModule)
         scalar = multiarray.get("scalar")
 
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -1139,9 +1139,9 @@
         from pypy.interpreter.mixedmodule import MixedModule
         from pypy.module.micronumpy.concrete import SliceArray
 
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
+        _numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(_numpypy, MixedModule)
+        multiarray = _numpypy.get("multiarray")
         assert isinstance(multiarray, MixedModule)
         reconstruct = multiarray.get("_reconstruct")
         parameters = space.newtuple([self.getclass(space), space.newtuple(
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -3,13 +3,13 @@
 
 class AppTestNumSupport(BaseNumpyAppTest):
     def test_zeros(self):
-        from numpypy import zeros
+        from numpy import zeros
         a = zeros(3)
         assert len(a) == 3
         assert a[0] == a[1] == a[2] == 0
 
     def test_empty(self):
-        from numpypy import empty
+        from numpy import empty
         import gc
         for i in range(1000):
             a = empty(3)
@@ -26,26 +26,26 @@
                 "empty() returned a zeroed out array every time")
 
     def test_where(self):
-        from numpypy import where, ones, zeros, array
+        from numpy import where, ones, zeros, array
         a = [1, 2, 3, 0, -3]
         a = where(array(a) > 0, ones(5), zeros(5))
         assert (a == [1, 1, 1, 0, 0]).all()
 
     def test_where_differing_dtypes(self):
-        from numpypy import array, ones, zeros, where
+        from numpy import array, ones, zeros, where
         a = [1, 2, 3, 0, -3]
         a = where(array(a) > 0, ones(5, dtype=int), zeros(5, dtype=float))
         assert (a == [1, 1, 1, 0, 0]).all()
 
     def test_where_broadcast(self):
-        from numpypy import array, where
+        from numpy import array, where
         a = where(array([[1, 2, 3], [4, 5, 6]]) > 3, [1, 1, 1], 2)
         assert (a == [[2, 2, 2], [1, 1, 1]]).all()
         a = where(True, [1, 1, 1], 2)
         assert (a == [1, 1, 1]).all()
 
     def test_where_errors(self):
-        from numpypy import where, array
+        from numpy import where, array
         raises(ValueError, "where([1, 2, 3], [3, 4, 5])")
         raises(ValueError, "where([1, 2, 3], [3, 4, 5], [6, 7])")
         assert where(True, 1, 2) == array(1)
@@ -58,14 +58,14 @@
     #    xxx
 
     def test_where_invalidates(self):
-        from numpypy import where, ones, zeros, array
+        from numpy import where, ones, zeros, array
         a = array([1, 2, 3, 0, -3])
         b = where(a > 0, ones(5), zeros(5))
         a[0] = 0
         assert (b == [1, 1, 1, 0, 0]).all()
 
     def test_dot_basic(self):
-        from numpypy import array, dot, arange
+        from numpy import array, dot, arange
         a = array(range(5))
         assert dot(a, a) == 30.0
 
@@ -100,7 +100,7 @@
         assert (dot([[1,2],[3,4]],[5,6]) == [17, 39]).all()
 
     def test_dot_constant(self):
-        from numpypy import array, dot
+        from numpy import array, dot
         a = array(range(5))
         b = a.dot(2.5)
         for i in xrange(5):
@@ -111,7 +111,7 @@
         assert c == 12.0
 
     def test_dot_out(self):
-        from numpypy import arange, dot
+        from numpy import arange, dot
         a = arange(12).reshape(3, 4)
         b = arange(12).reshape(4, 3)
         out = arange(9).reshape(3, 3)
@@ -124,19 +124,19 @@
                                 'right type, nr dimensions, and be a C-Array)')
 
     def test_choose_basic(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), array([4, 5, 6]), array([7, 8, 9])
         r = array([2, 1, 0]).choose([a, b, c])
         assert (r == [7, 5, 3]).all()
 
     def test_choose_broadcast(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         r = array([2, 1, 0]).choose([a, b, c])
         assert (r == [13, 5, 3]).all()
 
     def test_choose_out(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         r = array([2, 1, 0]).choose([a, b, c], out=None)
         assert (r == [13, 5, 3]).all()
@@ -146,7 +146,7 @@
         assert (a == [13, 5, 3]).all()
 
     def test_choose_modes(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         raises(ValueError, "array([3, 1, 0]).choose([a, b, c])")
         raises(ValueError, "array([3, 1, 0]).choose([a, b, c], mode='raises')")
@@ -158,20 +158,20 @@
         assert (r == [4, 5, 3]).all()
 
     def test_choose_dtype(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1.2, 2, 3]), [4, 5, 6], 13
         r = array([2, 1, 0]).choose([a, b, c])
         assert r.dtype == float
 
     def test_choose_dtype_out(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         x = array([0, 0, 0], dtype='i2')
         r = array([2, 1, 0]).choose([a, b, c], out=x)
         assert r.dtype == 'i2'
 
     def test_put_basic(self):
-        from numpypy import arange, array
+        from numpy import arange, array
         a = arange(5)
         a.put([0, 2], [-44, -55])
         assert (a == array([-44, 1, -55, 3, 4])).all()
@@ -183,7 +183,7 @@
         assert (a == array([0, 7, 2, 3, 4])).all()
 
     def test_put_modes(self):
-        from numpypy import array, arange
+        from numpy import array, arange
         a = arange(5)
         a.put(22, -5, mode='clip')
         assert (a == array([0, 1, 2, 3, -5])).all()
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -14,15 +14,12 @@
             else:
                 from . import dummy_module as numpy
                 sys.modules['numpy'] = numpy
-            sys.modules['numpypy'] = numpy
         else:
             import os
             path = os.path.dirname(__file__) + '/dummy_module.py'
             cls.space.appexec([cls.space.wrap(path)], """(path):
             import imp
-            numpy = imp.load_source('numpy', path)
-            import sys
-            sys.modules['numpypy'] = numpy
+            imp.load_source('numpy', path)
             """)
         cls.w_non_native_prefix = cls.space.wrap(NPY.OPPBYTE)
         cls.w_native_prefix = cls.space.wrap(NPY.NATBYTE)
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -132,13 +132,13 @@
             cls.w_c_pow = cls.space.wrap(interp2app(cls_c_pow))
 
     def test_fabs(self):
-        from numpypy import fabs, dtype
+        from numpy import fabs, dtype
 
         a = dtype('complex128').type(complex(-5., 5.))
         raises(TypeError, fabs, a)
 
     def test_fmax(self):
-        from numpypy import fmax, array
+        from numpy import fmax, array
         nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
         a = array((complex(ninf, 10), complex(10, ninf),
                    complex( inf, 10), complex(10,  inf),
@@ -165,7 +165,7 @@
         assert (fmax(a, b) == res).all()
 
     def test_fmin(self):
-        from numpypy import fmin, array
+        from numpy import fmin, array
         nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
         a = array((complex(ninf, 10), complex(10, ninf),
                    complex( inf, 10), complex(10,  inf),
@@ -192,11 +192,11 @@
         assert (fmin(a, b) == res).all()
 
     def test_signbit(self):
-        from numpypy import signbit
+        from numpy import signbit
         raises(TypeError, signbit, complex(1,1))
 
     def test_reciprocal(self):
-        from numpypy import array, reciprocal
+        from numpy import array, reciprocal
         inf = float('inf')
         nan = float('nan')
         #complex
@@ -218,21 +218,21 @@
                 assert (a[0].imag - e.imag) < rel_err
 
     def test_floorceiltrunc(self):
-        from numpypy import array, floor, ceil, trunc
+        from numpy import array, floor, ceil, trunc
         a = array([ complex(-1.4, -1.4), complex(-1.5, -1.5)])
         raises(TypeError, floor, a)
         raises(TypeError, ceil, a)
         raises(TypeError, trunc, a)
 
     def test_copysign(self):
-        from numpypy import copysign, dtype
+        from numpy import copysign, dtype
         complex128 = dtype('complex128').type
         a = complex128(complex(-5., 5.))
         b = complex128(complex(0., 0.))
         raises(TypeError, copysign, a, b)
 
     def test_exp2(self):
-        from numpypy import array, exp2
+        from numpy import array, exp2
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
@@ -268,7 +268,7 @@
 
     def test_expm1(self):
         import math, cmath
-        from numpypy import array, expm1
+        from numpy import array, expm1
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
@@ -307,7 +307,7 @@
                 self.rAlmostEqual(t1, t2, rel_err=rel_err, msg=msg)
 
     def test_not_complex(self):
-        from numpypy import (radians, deg2rad, degrees, rad2deg,
+        from numpy import (radians, deg2rad, degrees, rad2deg,
                   logaddexp, logaddexp2, fmod,
                   arctan2)
         raises(TypeError, radians, complex(90,90))
@@ -320,7 +320,7 @@
         raises (TypeError, fmod, complex(90,90), 3)
 
     def test_isnan_isinf(self):
-        from numpypy import isnan, isinf, array
+        from numpy import isnan, isinf, array
         assert (isnan(array([0.2+2j, complex(float('inf'),0),
                 complex(0,float('inf')), complex(0,float('nan')),
                 complex(float('nan'), 0)], dtype=complex)) == \
@@ -333,7 +333,7 @@
 
 
     def test_square(self):
-        from numpypy import square
+        from numpy import square
         assert square(complex(3, 4)) == complex(3,4) * complex(3, 4)
 
     def test_power_simple(self):
@@ -364,8 +364,8 @@
             self.rAlmostEqual(float(n_r_a[i].imag), float(p_r[i].imag), msg=msg)
 
     def test_conjugate(self):
-        from numpypy import conj, conjugate, dtype
-        import numpypy as np
+        from numpy import conj, conjugate, dtype
+        import numpy as np
 
         c0 = dtype('complex128').type(complex(2.5, 0))
         c1 = dtype('complex64').type(complex(1, 2))
@@ -390,7 +390,7 @@
     def test_logn(self):
         import math, cmath
         # log and log10 are tested in math (1:1 from rcomplex)
-        from numpypy import log2, array, log1p
+        from numpy import log2, array, log1p
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
@@ -447,7 +447,7 @@
                 self.rAlmostEqual(t1, t2, rel_err=rel_err, msg=msg)
 
     def test_logical_ops(self):
-        from numpypy import logical_and, logical_or, logical_xor, logical_not
+        from numpy import logical_and, logical_or, logical_xor, logical_not
 
         c1 = complex(1, 1)
         c3 = complex(3, 0)
@@ -461,7 +461,7 @@
         assert (logical_not([c1, c0]) == [False, True]).all()
 
     def test_minimum(self):
-        from numpypy import array, minimum
+        from numpy import array, minimum
 
         a = array([-5.0+5j, -5.0-5j, -0.0-10j, 1.0+10j])
         b = array([ 3.0+10.0j, 3.0, -2.0+2.0j, -3.0+4.0j])
@@ -470,7 +470,7 @@
             assert c[i] == min(a[i], b[i])
 
     def test_maximum(self):
-        from numpypy import array, maximum
+        from numpy import array, maximum
 
         a = array([-5.0+5j, -5.0-5j, -0.0-10j, 1.0+10j])
         b = array([ 3.0+10.0j, 3.0, -2.0+2.0j, -3.0+4.0j])
@@ -480,10 +480,10 @@
 
     def test_basic(self):
         import sys
-        from numpypy import (dtype, add, array, dtype,
+        from numpy import (dtype, add, array, dtype,
             subtract as sub, multiply, divide, negative, absolute as abs,
             floor_divide, real, imag, sign)
-        from numpypy import (equal, not_equal, greater, greater_equal, less,
+        from numpy import (equal, not_equal, greater, greater_equal, less,
                 less_equal, isnan)
         assert real(4.0) == 4.0
         assert imag(0.0) == 0.0
@@ -579,7 +579,7 @@
             assert repr(abs(inf_c)) == 'inf'
             assert repr(abs(complex(float('nan'), float('nan')))) == 'nan'
             # numpy actually raises an AttributeError,
-            # but numpypy raises a TypeError
+            # but numpy.raises a TypeError
             if '__pypy__' in sys.builtin_module_names:
                 exct, excm = TypeError, 'readonly attribute'
             else:
@@ -592,7 +592,7 @@
             assert(imag(c2) == 4.0)
 
     def test_conj(self):
-        from numpypy import array
+        from numpy import array
 
         a = array([1 + 2j, 1 - 2j])
         assert (a.conj() == [1 - 2j, 1 + 2j]).all()
@@ -603,7 +603,7 @@
         if self.isWindows:
             skip('windows does not support c99 complex')
         import sys
-        import numpypy as np
+        import numpy as np
         rAlmostEqual = self.rAlmostEqual
 
         for t, testcases in (
@@ -658,6 +658,6 @@
             sys.stderr.write('\n')
 
     def test_complexbox_to_pycomplex(self):
-        from numpypy import dtype
+        from numpy import dtype
         x = dtype('complex128').type(3.4j)
         assert complex(x) == 3.4j
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -23,7 +23,7 @@
             from numpy.core.multiarray import typeinfo
         except ImportError:
             # running on dummy module
-            from numpypy import typeinfo
+            from numpy import typeinfo
         assert typeinfo['Number'] == np.number
         assert typeinfo['LONGLONG'] == ('q', 9, 64, 8, 9223372036854775807L,
                                         -9223372036854775808L, np.longlong)
@@ -39,7 +39,7 @@
                                     np.dtype('int').type)
 
     def test_dtype_basic(self):
-        from numpypy import dtype
+        from numpy import dtype
         import sys
 
         d = dtype('?')
@@ -104,7 +104,7 @@
         assert d.shape == ()
 
     def test_dtype_eq(self):
-        from numpypy import dtype
+        from numpy import dtype
 
         assert dtype("int8") == "int8"
         assert "int8" == dtype("int8")
@@ -112,7 +112,7 @@
         assert dtype(bool) == bool
 
     def test_dtype_aliases(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype('bool8') is dtype('bool')
         assert dtype('byte') is dtype('int8')
         assert dtype('ubyte') is dtype('uint8')
@@ -132,7 +132,7 @@
         assert dtype('clongdouble').num in (15, 16)
 
     def test_dtype_with_types(self):
-        from numpypy import dtype
+        from numpy import dtype
 
         assert dtype(bool).num == 0
         if self.ptr_size == 4:
@@ -156,7 +156,7 @@
         assert dtype('complex').num == 15
 
     def test_array_dtype_attr(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
 
         a = array(range(5), long)
         assert a.dtype is dtype(long)
@@ -182,7 +182,7 @@
         assert np.dtype('S5').isbuiltin == 0
 
     def test_repr_str(self):
-        from numpypy import dtype
+        from numpy import dtype
         b = dtype(int).newbyteorder().newbyteorder().byteorder
         assert '.dtype' in repr(dtype)
         d = dtype('?')
@@ -208,7 +208,7 @@
         assert str(d) == "|V16"
 
     def test_bool_array(self):
-        from numpypy import array, False_, True_
+        from numpy import array, False_, True_
 
         a = array([0, 1, 2, 2.5], dtype='?')
         assert a[0] is False_
@@ -216,7 +216,7 @@
             assert a[i] is True_
 
     def test_copy_array_with_dtype(self):
-        from numpypy import array, longlong, False_
+        from numpy import array, longlong, False_
 
         a = array([0, 1, 2, 3], dtype=long)
         # int on 64-bit, long in 32-bit
@@ -230,35 +230,35 @@
         assert b[0] is False_
 
     def test_zeros_bool(self):
-        from numpypy import zeros, False_
+        from numpy import zeros, False_
 
         a = zeros(10, dtype=bool)
         for i in range(10):
             assert a[i] is False_
 
     def test_ones_bool(self):
-        from numpypy import ones, True_
+        from numpy import ones, True_
 
         a = ones(10, dtype=bool)
         for i in range(10):
             assert a[i] is True_
 
     def test_zeros_long(self):
-        from numpypy import zeros, longlong
+        from numpy import zeros, longlong
         a = zeros(10, dtype=long)
         for i in range(10):
             assert isinstance(a[i], longlong)
             assert a[1] == 0
 
     def test_ones_long(self):
-        from numpypy import ones, longlong
+        from numpy import ones, longlong
         a = ones(10, dtype=long)
         for i in range(10):
             assert isinstance(a[i], longlong)
             assert a[1] == 1
 
     def test_overflow(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         assert array([128], 'b')[0] == -128
         assert array([256], 'B')[0] == 0
         assert array([32768], 'h')[0] == -32768
@@ -273,7 +273,7 @@
         raises(OverflowError, "array([2**64], 'Q')")
 
     def test_bool_binop_types(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         types = [
             '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd',
             'e'
@@ -285,7 +285,7 @@
             assert (a + array([0], t)).dtype is dtype(t)
 
     def test_binop_types(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
                  ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
                  ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
@@ -309,7 +309,7 @@
             assert (d1, d2) == (d1, d2) and d3 is dtype(dout)
 
     def test_add(self):
-        import numpypy as np
+        import numpy as np
         for dtype in ["int8", "int16", "I"]:
             a = np.array(range(5), dtype=dtype)
             b = a + a
@@ -325,22 +325,22 @@
         assert len(d) == 2
 
     def test_shape(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype(long).shape == ()
 
     def test_cant_subclass(self):
-        from numpypy import dtype
+        from numpy import dtype
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
     def test_can_subclass(self):
-        import numpypy
-        class xyz(numpypy.void):
+        import numpy
+        class xyz(numpy.void):
             pass
         assert True
 
     def test_index(self):
-        import numpypy as np
+        import numpy as np
         for dtype in [np.int8, np.int16, np.int32, np.int64]:
             a = np.array(range(10), dtype=dtype)
             b = np.array([0] * 10, dtype=dtype)
@@ -348,7 +348,7 @@
                 a[idx] += 1
 
     def test_hash(self):
-        import numpypy as numpy
+        import numpy
         for tp, value in [
             (numpy.int8, 4),
             (numpy.int16, 5),
@@ -395,7 +395,7 @@
 
     def test_pickle(self):
         import numpy as np
-        from numpypy import array, dtype
+        from numpy import array, dtype
         from cPickle import loads, dumps
         a = array([1,2,3])
         if self.ptr_size == 8:
@@ -408,7 +408,7 @@
         assert np.dtype(('<f8', 2)).__reduce__() == (dtype, ('V16', 0, 1), (3, '|', (dtype('float64'), (2,)), None, None, 16, 1, 0))
 
     def test_newbyteorder(self):
-        import numpypy as np
+        import numpy as np
         import sys
         sys_is_le = sys.byteorder == 'little'
         native_code = sys_is_le and '<' or '>'
@@ -480,7 +480,7 @@
 
 class AppTestTypes(BaseAppTestDtypes):
     def test_abstract_types(self):
-        import numpypy as numpy
+        import numpy
 
         raises(TypeError, numpy.generic, 0)
         raises(TypeError, numpy.number, 0)
@@ -526,16 +526,16 @@
         #assert a.dtype is numpy.dtype('|V4')
 
     def test_new(self):
-        import numpypy as np
+        import numpy as np
         assert np.int_(4) == 4
         assert np.float_(3.4) == 3.4
 
     def test_pow(self):
-        from numpypy import int_
+        from numpy import int_
         assert int_(4) ** 2 == 16
 
     def test_bool(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.bool_.mro() == [numpy.bool_, numpy.generic, object]
         assert numpy.bool_(3) is numpy.True_
@@ -550,7 +550,7 @@
         assert numpy.bool_("False") is numpy.True_
 
     def test_int8(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int8.mro() == [numpy.int8, numpy.signedinteger,
                                     numpy.integer, numpy.number,
@@ -574,7 +574,7 @@
         assert numpy.int8('128') == -128
 
     def test_uint8(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.uint8.mro() == [numpy.uint8, numpy.unsignedinteger,
                                      numpy.integer, numpy.number,
@@ -599,7 +599,7 @@
         assert numpy.uint8('256') == 0
 
     def test_int16(self):
-        import numpypy as numpy
+        import numpy
 
         x = numpy.int16(3)
         assert x == 3
@@ -641,7 +641,7 @@
             assert numpy.uint32('4294967296') == 0
 
     def test_int_(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int_ is numpy.dtype(int).type
         assert numpy.int_.mro() == [numpy.int_, numpy.signedinteger,
@@ -752,7 +752,7 @@
 
     def test_complex_format(self):
         import sys
-        import numpypy as numpy
+        import numpy
 
         for complex_ in (numpy.complex128, numpy.complex64,):
             for real, imag, should in [
@@ -792,7 +792,7 @@
             assert "{:g}".format(numpy.complex_(0.5+1.5j)) == '{:g}'.format(0.5+1.5j)
 
     def test_complex(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.complex_ is numpy.complex128
         assert numpy.csingle is numpy.complex64
@@ -812,7 +812,7 @@
         assert d.char == 'F'
 
     def test_subclass_type(self):
-        import numpypy as numpy
+        import numpy
 
         class X(numpy.float64):
             def m(self):
@@ -826,12 +826,12 @@
         assert b.dtype.type is numpy.float64
 
     def test_long_as_index(self):
-        from numpypy import int_, float64
+        from numpy import int_, float64
         assert (1, 2, 3)[int_(1)] == 2
         raises(TypeError, lambda: (1, 2, 3)[float64(1)])
 
     def test_int(self):
-        from numpypy import int32, int64, int_
+        from numpy import int32, int64, int_
         import sys
         assert issubclass(int_, int)
         if sys.maxint == (1<<31) - 1:
@@ -842,7 +842,7 @@
             assert int_ is int64
 
     def test_various_types(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int16 is numpy.short
         assert numpy.int8 is numpy.byte
@@ -864,7 +864,7 @@
         assert not issubclass(numpy.longfloat, numpy.float64)
 
     def test_mro(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int16.__mro__ == (numpy.int16, numpy.signedinteger,
                                        numpy.integer, numpy.number,
@@ -873,7 +873,7 @@
 
     def test_operators(self):
         from operator import truediv
-        from numpypy import float64, int_, True_, False_
+        from numpy import float64, int_, True_, False_
         assert 5 / int_(2) == int_(2)
         assert truediv(int_(3), int_(2)) == float64(1.5)
         assert truediv(3, int_(2)) == float64(1.5)
@@ -899,7 +899,7 @@
 
     def test_alternate_constructs(self):
         import numpy as np
-        from numpypy import dtype
+        from numpy import dtype
         nnp = self.non_native_prefix
         byteorder = self.native_prefix
         assert dtype('i8') == dtype(byteorder + 'i8') == dtype('=i8') == dtype(long)
@@ -922,7 +922,7 @@
         assert d.str == '|S0'
 
     def test_dtype_str(self):
-        from numpypy import dtype
+        from numpy import dtype
         byteorder = self.native_prefix
         assert dtype('i8').str == byteorder + 'i8'
         assert dtype('<i8').str == '<i8'
@@ -949,7 +949,7 @@
         assert dtype(('f8', 2)).str == "|V16"
 
     def test_intp(self):
-        from numpypy import dtype
+        from numpy import dtype
         for s in ['p', 'int']:
             assert dtype(s) is dtype('intp')
         for s in ['P', 'uint']:
@@ -968,16 +968,16 @@
             assert dtype('P').name == 'uint64'
 
     def test_alignment(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype('i4').alignment == 4
 
     def test_isnative(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype('i4').isnative == True
         assert dtype('>i8').isnative == False
 
     def test_any_all_nonzero(self):
-        import numpypy as numpy
+        import numpy
         x = numpy.bool_(True)
         assert x.any() is numpy.True_
         assert x.all() is numpy.True_
@@ -1000,7 +1000,7 @@
         assert x.__nonzero__() is True
 
     def test_ravel(self):
-        from numpypy import float64, int8, array
+        from numpy import float64, int8, array
         x = float64(42.5).ravel()
         assert x.dtype == float64
         assert (x == array([42.5])).all()
@@ -1023,7 +1023,7 @@
 
 class AppTestStrUnicodeDtypes(BaseNumpyAppTest):
     def test_mro(self):
-        from numpypy import str_, unicode_, character, flexible, generic
+        from numpy import str_, unicode_, character, flexible, generic
         import sys
         if '__pypy__' in sys.builtin_module_names:
             assert str_.mro() == [str_, character, flexible, generic,
@@ -1037,7 +1037,7 @@
                                       flexible, generic, object]
 
     def test_str_dtype(self):
-        from numpypy import dtype, str_
+        from numpy import dtype, str_
 
         raises(TypeError, "dtype('Sx')")
         for t in ['S8', '|S8', '=S8']:
@@ -1058,7 +1058,7 @@
             assert d.str == '|S1'
 
     def test_unicode_dtype(self):
-        from numpypy import dtype, unicode_
+        from numpy import dtype, unicode_
 
         raises(TypeError, "dtype('Ux')")
         d = dtype('U8')
@@ -1070,11 +1070,11 @@
         assert d.num == 19
 
     def test_string_boxes(self):
-        from numpypy import str_
+        from numpy import str_
         assert isinstance(str_(3), str_)
 
     def test_unicode_boxes(self):
-        from numpypy import unicode_
+        from numpy import unicode_
         import sys
         if '__pypy__' in sys.builtin_module_names:
             exc = raises(NotImplementedError, unicode_, 3)
@@ -1085,7 +1085,7 @@
 
     def test_character_dtype(self):
         import numpy as np
-        from numpypy import array, character
+        from numpy import array, character
         x = array([["A", "B"], ["C", "D"]], character)
         assert (x == [["A", "B"], ["C", "D"]]).all()
         d = np.dtype('c')
@@ -1098,7 +1098,7 @@
     spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
 
     def test_create(self):
-        from numpypy import dtype, void
+        from numpy import dtype, void
 
         raises(ValueError, "dtype([('x', int), ('x', float)])")
         d = dtype([("x", "<i4"), ("y", "<f4"), ("z", "<u2"), ("v", "<f8")])
@@ -1157,7 +1157,7 @@
             raises(NotImplementedError, np.dtype, d)
 
     def test_create_subarrays(self):
-        from numpypy import dtype
+        from numpy import dtype
         d = dtype([("x", "float", (2,)), ("y", "int64", (2,))])
         assert d.itemsize == 32
         assert d.name == "void256"
@@ -1278,7 +1278,7 @@
         assert repr(d) == "dtype([('f0', '<f8'), ('f1', '<f8')])"
 
     def test_pickle_record(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         from cPickle import loads, dumps
 
         d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)])
@@ -1289,7 +1289,7 @@
         assert new_d.__reduce__() == d.__reduce__()
 
     def test_pickle_record_subarrays(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         from cPickle import loads, dumps
 
         d = dtype([("x", "int32", (3,)), ("y", "int32", (2,)), ("z", "int32", (4,)), ("value", float, (5,))])
@@ -1321,7 +1321,7 @@
             cls.w_check_non_native = cls.space.wrap(interp2app(check_non_native))
 
     def test_non_native(self):
-        from numpypy import array
+        from numpy import array
         a = array([1, 2, 3], dtype=self.non_native_prefix + 'i2')
         assert a[0] == 1
         assert (a + a)[1] == 4
@@ -1344,7 +1344,7 @@
 
 class AppTestObjectDtypes(BaseNumpyAppTest):
     def test_scalar_from_object(self):
-        from numpypy import array
+        from numpy import array
         import sys
         class Polynomial(object):
             pass
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -269,13 +269,13 @@
                 assert not a.flags['F']
 
     def test_ndmin(self):
-        from numpypy import array
+        from numpy import array
 
         arr = array([[[1]]], ndmin=1)
         assert arr.shape == (1, 1, 1)
 
     def test_noop_ndmin(self):
-        from numpypy import array
+        from numpy import array
         arr = array([1], ndmin=3)
         assert arr.shape == (1, 1, 1)
 
@@ -289,7 +289,7 @@
         assert a.dtype == np.intp
 
     def test_array_copy(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(12)).reshape(3,4)
         b = array(a, ndmin=4)
         assert b.shape == (1, 1, 3, 4)
@@ -353,12 +353,12 @@
         assert str(exc.value) == 'buffer is read-only'
 
     def test_type(self):
-        from numpypy import array
+        from numpy import array
         ar = array(range(5))
         assert type(ar) is type(ar + ar)
 
     def test_ndim(self):
-        from numpypy import array
+        from numpy import array
         x = array(0.2)
         assert x.ndim == 0
         x = array([1, 2])
@@ -367,12 +367,12 @@
         assert x.ndim == 2
         x = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
         assert x.ndim == 3
-        # numpy actually raises an AttributeError, but numpypy raises an
+        # numpy actually raises an AttributeError, but numpy.raises an
         # TypeError
         raises((TypeError, AttributeError), 'x.ndim = 3')
 
     def test_zeros(self):
-        from numpypy import zeros
+        from numpy import zeros
         a = zeros(15)
         # Check that storage was actually zero'd.
         assert a[10] == 0.0
@@ -431,7 +431,7 @@
         assert type(b) is np.ndarray
 
     def test_size(self):
-        from numpypy import array,arange,cos
+        from numpy import array,arange,cos
         assert array(3).size == 1
         a = array([1, 2, 3])
         assert a.size == 3
@@ -440,13 +440,13 @@
         assert ten == 10
 
     def test_empty(self):
-        from numpypy import empty
+        from numpy import empty
         a = empty(2)
         a[1] = 1.0
         assert a[1] == 1.0
 
     def test_ones(self):
-        from numpypy import ones, dtype
+        from numpy import ones, dtype
         a = ones(3)
         assert len(a) == 3
         assert a[0] == 1
@@ -458,7 +458,7 @@
         assert b.dtype is dtype(complex)
 
     def test_arange(self):
-        from numpypy import arange, dtype
+        from numpy import arange, dtype
         a = arange(3)
         assert (a == [0, 1, 2]).all()
         assert a.dtype is dtype(int)
@@ -480,7 +480,7 @@
         assert arange(False, True, True).dtype is dtype(int)
 
     def test_copy(self):
-        from numpypy import arange, array
+        from numpy import arange, array
         a = arange(5)
         b = a.copy()
         for i in xrange(5):
@@ -522,7 +522,7 @@
             raises(NotImplementedError, a.copy, order=True)
 
     def test_iterator_init(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert a[3] == 3
 
@@ -546,7 +546,7 @@
         assert (a == [[1], [2]]).all()
 
     def test_getitem(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         raises(IndexError, "a[5]")
         a = a + a
@@ -555,14 +555,14 @@
         raises(IndexError, "a[-6]")
 
     def test_getitem_float(self):
-        from numpypy import array
+        from numpy import array
         a = array([1, 2, 3, 4])
         assert a[1.2] == 2
         assert a[1.6] == 2
         assert a[-1.2] == 4
 
     def test_getitem_tuple(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         raises(IndexError, "a[(1,2)]")
         for i in xrange(5):
@@ -572,28 +572,28 @@
             assert a[i] == b[i]
 
     def test_getitem_nd(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(15).reshape(3, 5)
         assert a[1, 3] == 8
         assert a.T[1, 2] == 11
 
     def test_getitem_obj_index(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         assert a[self.CustomIndexObject(1)] == 1
 
     def test_getitem_obj_prefer_index_to_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         assert a[self.CustomIndexIntObject(0, 1)] == 0
 
     def test_getitem_obj_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         assert a[self.CustomIntObject(1)] == 1
 
     def test_setitem(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         a[-1] = 5.0
         assert a[4] == 5.0
@@ -612,7 +612,7 @@
         assert a[2] == -0.005
 
     def test_setitem_tuple(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         raises(IndexError, "a[(1,2)] = [0,1]")
         for i in xrange(5):
@@ -630,25 +630,25 @@
         assert a[2] == 100
 
     def test_setitem_obj_index(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         a[self.CustomIndexObject(1)] = 100
         assert a[1] == 100
 
     def test_setitem_obj_prefer_index_to_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         a[self.CustomIndexIntObject(0, 1)] = 100
         assert a[0] == 100
 
     def test_setitem_obj_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         a[self.CustomIntObject(1)] = 100
         assert a[1] == 100
 
     def test_delitem(self):
-        import numpypy as np
+        import numpy as np
         a = np.arange(10)
         exc = raises(ValueError, 'del a[2]')
         assert exc.value.message == 'cannot delete array elements'
@@ -665,7 +665,7 @@
         # numpy will swallow errors in __int__ and __index__ and
         # just raise IndexError.
 
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         exc = raises(IndexError, "a[ErrorIndex()] == 0")
         assert exc.value.message == 'cannot convert index to integer'
@@ -673,7 +673,7 @@
         assert exc.value.message == 'cannot convert index to integer'
 
     def test_setslice_array(self):
-        from numpypy import array
+        from numpy import array
         a = array(5)
         exc = raises(ValueError, "a[:] = 4")
         assert exc.value[0] == "cannot slice a 0-d array"
@@ -687,7 +687,7 @@
         assert b[1] == 0.
 
     def test_setslice_of_slice_array(self):
-        from numpypy import array, zeros
+        from numpy import array, zeros
         a = zeros(5)
         a[::2] = array([9., 10., 11.])
         assert a[0] == 9.
@@ -706,7 +706,7 @@
         assert a[0] == 3.
 
     def test_setslice_list(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5), float)
         b = [0., 1.]
         a[1:4:2] = b
@@ -714,7 +714,7 @@
         assert a[3] == 1.
 
     def test_setslice_constant(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5), float)
         a[1:4:2] = 0.
         assert a[1] == 0.
@@ -722,7 +722,7 @@
 
     def test_newaxis(self):
         import math
-        from numpypy import array, cos, zeros, newaxis
+        from numpy import array, cos, zeros, newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
@@ -743,7 +743,7 @@
         assert o == 3
 
     def test_newaxis_slice(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
 
         a = array(range(5))
         b = array(range(1,5))
@@ -755,14 +755,14 @@
         assert (a[newaxis,1:] == c).all()
 
     def test_newaxis_assign(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
 
         a = array(range(5))
         a[newaxis,1] = [2]
         assert a[1] == 2
 
     def test_newaxis_virtual(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
 
         a = array(range(5))
         b = (a + a)[newaxis]
@@ -770,20 +770,20 @@
         assert (b == c).all()
 
     def test_newaxis_then_slice(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
         a = array(range(5))
         b = a[newaxis]
         assert b.shape == (1, 5)
         assert (b[0,1:] == a[1:]).all()
 
     def test_slice_then_newaxis(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
         a = array(range(5))
         b = a[2:]
         assert (b[newaxis] == [[2, 3, 4]]).all()
 
     def test_scalar(self):
-        from numpypy import array, dtype, int_
+        from numpy import array, dtype, int_
         a = array(3)
         exc = raises(IndexError, "a[0]")
         assert exc.value[0] == "0-d arrays can't be indexed"
@@ -816,13 +816,13 @@
         assert a == 2.5
 
     def test_len(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert len(a) == 5
         assert len(a + a) == 5
 
     def test_shape(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert a.shape == (5,)
         b = a + a
@@ -832,7 +832,7 @@
         assert array([]).shape == (0,)
 
     def test_set_shape(self):
-        from numpypy import array, zeros
+        from numpy import array, zeros
         a = array([])
         raises(ValueError, "a.shape = []")
         a = array(range(12))
@@ -855,7 +855,7 @@
         raises(AttributeError, 'a.shape = 6')
 
     def test_reshape(self):
-        from numpypy import array, zeros
+        from numpy import array, zeros
         for a in [array(1), array([1])]:
             for s in [(), (1,)]:
                 b = a.reshape(s)
@@ -902,7 +902,7 @@
             raises(NotImplementedError, a.reshape, (0,), order='F')
 
     def test_slice_reshape(self):
-        from numpypy import zeros, arange
+        from numpy import zeros, arange
         a = zeros((4, 2, 3))
         b = a[::2, :, :]
         b.shape = (2, 6)
@@ -938,32 +938,32 @@
         raises(ValueError, arange(10).reshape, (5, -1, -1))
 
     def test_reshape_varargs(self):
-        from numpypy import arange
+        from numpy import arange
         z = arange(96).reshape(12, -1)
         y = z.reshape(4, 3, 8)
         assert y.shape == (4, 3, 8)
 
     def test_scalar_reshape(self):
-        from numpypy import array
+        from numpy import array
         a = array(3)
         assert a.reshape([1, 1]).shape == (1, 1)
         assert a.reshape([1]).shape == (1,)
         raises(ValueError, "a.reshape(3)")
 
     def test_strides(self):
-        from numpypy import array
+        from numpy import array
         a = array([[1.0, 2.0],
                    [3.0, 4.0]])
         assert a.strides == (16, 8)
         assert a[1:].strides == (16, 8)
 
     def test_strides_scalar(self):
-        from numpypy import array
+        from numpy import array
         a = array(42)
         assert a.strides == ()
 
     def test_add(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = a + a
         for i in range(5):
@@ -976,7 +976,7 @@
             assert c[i] == bool(a[i] + b[i])
 
     def test_add_other(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = array([i for i in reversed(range(5))])
         c = a + b
@@ -984,14 +984,14 @@
             assert c[i] == 4
 
     def test_add_constant(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = a + 5
         for i in range(5):
             assert b[i] == i + 5
 
     def test_radd(self):
-        from numpypy import array
+        from numpy import array
         r = 3 + array(range(3))
         for i in range(3):
             assert r[i] == i + 3
@@ -999,7 +999,7 @@
         assert (r == [2, 4]).all()
 
     def test_inline_op_scalar(self):
-        from numpypy import array
+        from numpy import array
         for op in [
                 '__iadd__',
                 '__isub__',
@@ -1018,7 +1018,7 @@
             assert id(a) == id(b)
 
     def test_inline_op_array(self):
-        from numpypy import array
+        from numpy import array
         for op in [
                 '__iadd__',
                 '__isub__',
@@ -1042,7 +1042,7 @@
                 assert a[i] == getattr(c[i], reg_op).__call__(d[i])
 
     def test_add_list(self):
-        from numpypy import array, ndarray
+        from numpy import array, ndarray
         a = array(range(5))
         b = list(reversed(range(5)))
         c = a + b
@@ -1051,14 +1051,14 @@
             assert c[i] == 4
 
     def test_subtract(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = a - a
         for i in range(5):
             assert b[i] == 0
 
     def test_subtract_other(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = array([1, 1, 1, 1, 1])
         c = a - b
@@ -1066,37 +1066,37 @@
             assert c[i] == i - 1
 
     def test_subtract_constant(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = a - 5
         for i in range(5):
             assert b[i] == i - 5
 
     def test_scalar_subtract(self):
-        from numpypy import dtype
+        from numpy import dtype
         int32 = dtype('int32').type
         assert int32(2) - 1 == 1
         assert 1 - int32(2) == -1
 
     def test_mul(self):
-        import numpypy
-
-        a = numpypy.array(range(5))
+        import numpy
+
+        a = numpy.array(range(5))
         b = a * a
         for i in range(5):
             assert b[i] == i * i
         assert b.dtype is a.dtype
 
-        a = numpypy.array(range(5), dtype=bool)
+        a = numpy.array(range(5), dtype=bool)
         b = a * a
-        assert b.dtype is numpypy.dtype(bool)
-        bool_ = numpypy.dtype(bool).type
+        assert b.dtype is numpy.dtype(bool)
+        bool_ = numpy.dtype(bool).type
         assert b[0] is bool_(False)
         for i in range(1, 5):
             assert b[i] is bool_(True)
 
     def test_mul_constant(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = a * 5
         for i in range(5):
@@ -1104,7 +1104,7 @@
 
     def test_div(self):
         from math import isnan
-        from numpypy import array, dtype
+        from numpy import array, dtype
 
         a = array(range(1, 6))
         b = a / a
@@ -1136,7 +1136,7 @@
         assert c[2] == float('-inf')
 
     def test_div_other(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = array([2, 2, 2, 2, 2], float)
         c = a / b
@@ -1144,7 +1144,7 @@
             assert c[i] == i / 2.0
 
     def test_div_constant(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = a / 5.0
         for i in range(5):
@@ -1152,7 +1152,7 @@
 
     def test_floordiv(self):
         from math import isnan
-        from numpypy import array, dtype
+        from numpy import array, dtype
 
         a = array(range(1, 6))
         b = a // a
@@ -1182,26 +1182,26 @@
         assert c[2] == float('-inf')
 
     def test_floordiv_other(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = array([2, 2, 2, 2, 2], float)
         c = a // b
         assert (c == [0, 0, 1, 1, 2]).all()
 
     def test_rfloordiv(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(1, 6))
         b = 3 // a
         assert (b == [3, 1, 1, 0, 0]).all()
 
     def test_floordiv_constant(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = a // 2
         assert (b == [0, 0, 1, 1, 2]).all()
 
     def test_signed_integer_division_overflow(self):
-        import numpypy as np
+        import numpy as np
         for s in (8, 16, 32, 64):
             for o in ['__div__', '__floordiv__']:
                 a = np.array([-2**(s-1)], dtype='int%d' % s)
@@ -1209,27 +1209,27 @@
 
     def test_truediv(self):
         from operator import truediv
-        from numpypy import arange
+        from numpy import arange
 
         assert (truediv(arange(5), 2) == [0., .5, 1., 1.5, 2.]).all()
         assert (truediv(2, arange(3)) == [float("inf"), 2., 1.]).all()
 
     def test_divmod(self):
-        from numpypy import arange
+        from numpy import arange
 
         a, b = divmod(arange(10), 3)
         assert (a == [0, 0, 0, 1, 1, 1, 2, 2, 2, 3]).all()
         assert (b == [0, 1, 2, 0, 1, 2, 0, 1, 2, 0]).all()
 
     def test_rdivmod(self):
-        from numpypy import arange
+        from numpy import arange
 
         a, b = divmod(3, arange(1, 5))
         assert (a == [3, 1, 1, 0]).all()
         assert (b == [0, 1, 0, 3]).all()
 
     def test_lshift(self):
-        from numpypy import array
+        from numpy import array
 
         a = array([0, 1, 2, 3])
         assert (a << 2 == [0, 4, 8, 12]).all()
@@ -1239,7 +1239,7 @@
         raises(TypeError, lambda: a << 2)
 
     def test_rlshift(self):
-        from numpypy import arange
+        from numpy import arange
 
         a = arange(3)
         assert (2 << a == [2, 4, 8]).all()
@@ -1261,13 +1261,13 @@
         assert 'not supported for the input types' in exc.value.message
 
     def test_rrshift(self):
-        from numpypy import arange
+        from numpy import arange
 
         a = arange(5)
         assert (2 >> a == [2, 1, 0, 0, 0]).all()
 
     def test_pow(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5), float)
         b = a ** a
         for i in range(5):
@@ -1277,7 +1277,7 @@
         assert (a ** 2 == a * a).all()
 
     def test_pow_other(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5), float)
         b = array([2, 2, 2, 2, 2])
         c = a ** b
@@ -1285,14 +1285,14 @@
             assert c[i] == i ** 2
 
     def test_pow_constant(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5), float)
         b = a ** 2
         for i in range(5):
             assert b[i] == i ** 2
 
     def test_mod(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(1, 6))
         b = a % a
         for i in range(5):
@@ -1305,7 +1305,7 @@
             assert b[i] == 1
 
     def test_mod_other(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = array([2, 2, 2, 2, 2])
         c = a % b
@@ -1313,38 +1313,38 @@
             assert c[i] == i % 2
 
     def test_mod_constant(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = a % 2
         for i in range(5):
             assert b[i] == i % 2
 
     def test_rand(self):
-        from numpypy import arange
+        from numpy import arange
 
         a = arange(5)
         assert (3 & a == [0, 1, 2, 3, 0]).all()
 
     def test_ror(self):
-        from numpypy import arange
+        from numpy import arange
 
         a = arange(5)
         assert (3 | a == [3, 3, 3, 3, 7]).all()
 
     def test_xor(self):
-        from numpypy import arange
+        from numpy import arange
 
         a = arange(5)
         assert (a ^ 3 == [3, 2, 1, 0, 7]).all()
 
     def test_rxor(self):
-        from numpypy import arange
+        from numpy import arange
 
         a = arange(5)
         assert (3 ^ a == [3, 2, 1, 0, 7]).all()
 
     def test_pos(self):
-        from numpypy import array
+        from numpy import array
         a = array([1., -2., 3., -4., -5.])
         b = +a
         for i in range(5):
@@ -1355,7 +1355,7 @@
             assert a[i] == i
 
     def test_neg(self):
-        from numpypy import array
+        from numpy import array
         a = array([1., -2., 3., -4., -5.])
         b = -a
         for i in range(5):
@@ -1366,7 +1366,7 @@
             assert a[i] == -i
 
     def test_abs(self):
-        from numpypy import array
+        from numpy import array
         a = array([1., -2., 3., -4., -5.])
         b = abs(a)
         for i in range(5):
@@ -1377,7 +1377,7 @@
             assert a[i + 5] == abs(i)
 
     def test_auto_force(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         b = a - 1
         a[2] = 3
@@ -1391,7 +1391,7 @@
         assert c[1] == 4
 
     def test_getslice(self):
-        from numpypy import array
+        from numpy import array
         a = array(5)
         exc = raises(ValueError, "a[:]")
         assert exc.value[0] == "cannot slice a 0-d array"
@@ -1408,7 +1408,7 @@
         assert s[0] == 5
 
     def test_getslice_step(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(10))
         s = a[1:9:2]
         assert len(s) == 4
@@ -1416,7 +1416,7 @@
             assert s[i] == a[2 * i + 1]
 
     def test_slice_update(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         s = a[0:3]
         s[1] = 10
@@ -1426,7 +1426,7 @@
 
     def test_slice_invaidate(self):
         # check that slice shares invalidation list with
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         s = a[0:2]
         b = array([10, 11])
@@ -1440,7 +1440,7 @@
         assert d[1] == 12
 
     def test_sum(self):
-        from numpypy import array, zeros, float16, complex64, str_
+        from numpy import array, zeros, float16, complex64, str_
         a = array(range(5))
         assert a.sum() == 10
         assert a[:4].sum() == 6
@@ -1470,7 +1470,7 @@
         assert list(zeros((0, 2)).sum(axis=1)) == []
 
     def test_reduce_nd(self):
-        from numpypy import arange, array
+        from numpy import arange, array
         a = arange(15).reshape(5, 3)
         assert a.sum() == 105
         assert a.sum(keepdims=True) == 105
@@ -1504,7 +1504,7 @@
         assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all()
 
     def test_prod(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         a = array(range(1, 6))
         assert a.prod() == 120.0
         assert a.prod(keepdims=True) == 120.0
@@ -1520,7 +1520,7 @@
             assert a.prod().dtype is dtype(dt)
 
     def test_max(self):
-        from numpypy import array, zeros
+        from numpy import array, zeros
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert a.max() == 5.7
         assert a.max().shape == ()
@@ -1533,12 +1533,12 @@
         assert list(zeros((0, 2)).max(axis=1)) == []
 
     def test_max_add(self):
-        from numpypy import array
+        from numpy import array
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert (a + a).max() == 11.4
 
     def test_min(self):
-        from numpypy import array, zeros
+        from numpy import array, zeros
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert a.min() == -3.0
         assert a.min().shape == ()
@@ -1551,7 +1551,7 @@
         assert list(zeros((0, 2)).min(axis=1)) == []
 
     def test_argmax(self):
-        from numpypy import array
+        from numpy import array
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         r = a.argmax()
         assert r == 2
@@ -1580,7 +1580,7 @@
             raises(NotImplementedError, a.argmax, axis=0)
 
     def test_argmin(self):
-        from numpypy import array
+        from numpy import array
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert a.argmin() == 3
         assert a.argmin(axis=None, out=None) == 3
@@ -1591,7 +1591,7 @@
             raises(NotImplementedError, a.argmin, axis=0)
 
     def test_all(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert a.all() == False
         a[0] = 3.0
@@ -1602,7 +1602,7 @@
         assert b.all() == True
 
     def test_any(self):
-        from numpypy import array, zeros
+        from numpy import array, zeros
         a = array(range(5))
         assert a.any() == True
         assert a.any(keepdims=True) == True
@@ -1613,7 +1613,7 @@
         assert c.any() == False
 
     def test_dtype_guessing(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         import sys
         assert array([True]).dtype is dtype(bool)
         assert array([True, False]).dtype is dtype(bool)
@@ -1639,7 +1639,7 @@
 
     def test_comparison(self):
         import operator
-        from numpypy import array, dtype
+        from numpy import array, dtype
 
         a = array(range(5))
         b = array(range(5), float)
@@ -1658,7 +1658,7 @@
                 assert c[i] == func(b[i], 3)
 
     def test___nonzero__(self):
-        from numpypy import array
+        from numpy import array
         a = array([1, 2])
         raises(ValueError, bool, a)
         raises(ValueError, bool, a == a)
@@ -1668,7 +1668,7 @@
         assert not bool(array([0]))
 
     def test_slice_assignment(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         a[::-1] = a
         assert (a == [4, 3, 2, 1, 0]).all()
@@ -1678,7 +1678,7 @@
         assert (a == [8, 6, 4, 2, 0]).all()
 
     def test_virtual_views(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(15)
         c = (a + a)
         d = c[::2]
@@ -1696,7 +1696,7 @@
         assert b[1] == 2
 
     def test_realimag_views(self):
-        from numpypy import arange, array
+        from numpy import arange, array
         a = array(1.5)
         assert a.real == 1.5
         assert a.imag == 0.0
@@ -1743,7 +1743,7 @@
         assert arange(4, dtype='<c8').real.max() == 3.0
 
     def test_scalar_view(self):
-        from numpypy import array
+        from numpy import array
         a = array(3, dtype='int32')
         b = a.view(dtype='float32')
         assert b.shape == ()
@@ -1770,7 +1770,7 @@
         assert b.imag.tostring() == '\x00' * 4
 
     def test_array_view(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         x = array((1, 2), dtype='int8')
         assert x.shape == (2,)
         y = x.view(dtype='int16')
@@ -1803,12 +1803,12 @@
         assert a.view('S16')[0] == '\x01' + '\x00' * 7 + '\x02'
 
     def test_ndarray_view_empty(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         x = array([], dtype=[('a', 'int8'), ('b', 'int8')])
         y = x.view(dtype='int16')
 
     def test_tolist_scalar(self):
-        from numpypy import dtype
+        from numpy import dtype
         int32 = dtype('int32').type
         bool_ = dtype('bool').type
         x = int32(23)
@@ -1818,13 +1818,13 @@
         assert y.tolist() is True
 
     def test_tolist_zerodim(self):
-        from numpypy import array
+        from numpy import array
         x = array(3)
         assert x.tolist() == 3
         assert type(x.tolist()) is int
 
     def test_tolist_singledim(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert a.tolist() == [0, 1, 2, 3, 4]
         assert type(a.tolist()[0]) is int
@@ -1832,23 +1832,23 @@
         assert b.tolist() == [0.2, 0.4, 0.6]
 
     def test_tolist_multidim(self):
-        from numpypy import array
+        from numpy import array
         a = array([[1, 2], [3, 4]])
         assert a.tolist() == [[1, 2], [3, 4]]
 
     def test_tolist_view(self):
-        from numpypy import array
+        from numpy import array
         a = array([[1, 2], [3, 4]])
         assert (a + a).tolist() == [[2, 4], [6, 8]]
 
     def test_tolist_slice(self):
-        from numpypy import array
+        from numpy import array
         a = array([[17.1, 27.2], [40.3, 50.3]])
         assert a[:, 0].tolist() == [17.1, 40.3]
         assert a[0].tolist() == [17.1, 27.2]
 
     def test_concatenate(self):
-        from numpypy import array, concatenate, dtype
+        from numpy import array, concatenate, dtype
         exc = raises(ValueError, concatenate, (array(1.5), array(2.5)))
         assert exc.value[0] == 'zero-dimensional arrays cannot be concatenated'
         a = concatenate((array(1.5), array(2.5)), axis=None)
@@ -1933,7 +1933,7 @@
 
     def test_record_concatenate(self):
         # only an exact match can succeed
-        from numpypy import zeros, concatenate
+        from numpy import zeros, concatenate
         a = concatenate((zeros((2,),dtype=[('x', int), ('y', float)]),
                          zeros((2,),dtype=[('x', int), ('y', float)])))
         assert a.shape == (4,)
@@ -1949,7 +1949,7 @@
         assert str(exc.value).startswith('invalid type promotion')
 
     def test_flatten(self):
-        from numpypy import array
+        from numpy import array
 
         assert array(3).flatten().shape == (1,)
         a = array([[1, 2], [3, 4]])
@@ -1972,7 +1972,7 @@
         assert (a.T.flatten() == [1, 3, 2, 4]).all()
 
     def test_itemsize(self):
-        from numpypy import ones, dtype, array
+        from numpy import ones, dtype, array
 
         for obj in [float, bool, int]:
             assert ones(1, dtype=obj).itemsize == dtype(obj).itemsize
@@ -1981,7 +1981,7 @@
         assert ones(1)[:].itemsize == 8
 
     def test_nbytes(self):
-        from numpypy import array, ones
+        from numpy import array, ones
 
         assert ones(1).nbytes == 8
         assert ones((2, 2)).nbytes == 32
@@ -1990,7 +1990,7 @@
         assert array(3.0).nbytes == 8
 
     def test_repeat(self):
-        from numpypy import array
+        from numpy import array
         a = array([[1, 2], [3, 4]])
         assert (a.repeat(3) == [1, 1, 1, 2, 2, 2,
                                  3, 3, 3, 4, 4, 4]).all()
@@ -2033,7 +2033,7 @@
         assert exc.value.message == "duplicate value in 'axis'"
 
     def test_swapaxes(self):
-        from numpypy import array
+        from numpy import array
         x = array([])
         assert x.swapaxes(0, 2) is x
         x = array([[1, 2]])
@@ -2071,19 +2071,19 @@
         assert array(1).swapaxes(10, 12) == 1
 
     def test_filter_bug(self):
-        from numpypy import array
+        from numpy import array
         a = array([1.0,-1.0])
         a[a<0] = -a[a<0]
         assert (a == [1, 1]).all()
 
     def test_int_list_index(slf):
-        from numpypy import array, arange
+        from numpy import array, arange
         assert (array([10,11,12,13])[[1,2]] == [11, 12]).all()
         assert (arange(6).reshape((2,3))[[0,1]] == [[0, 1, 2], [3, 4, 5]]).all()
         assert arange(6).reshape((2,3))[(0,1)] == 1
 
     def test_int_array_index(self):
-        from numpypy import array, arange, zeros
+        from numpy import array, arange, zeros
         b = arange(10)[array([3, 2, 1, 5])]
         assert (b == [3, 2, 1, 5]).all()
         raises(IndexError, "arange(10)[array([10])]")
@@ -2095,7 +2095,7 @@
         assert (zeros(1)[[]] == []).all()
 
     def test_int_array_index_setitem(self):
-        from numpypy import arange, zeros, array
+        from numpy import arange, zeros, array
         a = arange(10)
         a[[3, 2, 1, 5]] = zeros(4, dtype=int)
         assert (a == [0, 0, 0, 0, 4, 0, 6, 7, 8, 9]).all()
@@ -2109,7 +2109,7 @@
         assert (a == [0, 0, 0, 1, 1, 1, 0, 0, 0, 0]).all()
 
     def test_array_scalar_index(self):
-        import numpypy as np
+        import numpy as np
         a = np.array([[1, 2, 3],
                       [4, 5, 6],
                       [7, 8, 9]])
@@ -2130,7 +2130,7 @@
                                     'integer (or boolean) type'
 
     def test_bool_array_index(self):
-        from numpypy import arange, array
+        from numpy import arange, array
         b = arange(10)
         assert (b[array([True, False, True])] == [0, 2]).all()

From noreply at buildbot.pypy.org  Fri Dec  5 21:19:56 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 21:19:56 +0100 (CET)
Subject: [pypy-commit] pypy default: remove old test_lib_pypy tests of
	numpypy
Message-ID: <20141205201956.097491D257D@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74840:0123a8ee6ab6
Date: 2014-12-05 15:16 -0500
http://bitbucket.org/pypy/pypy/changeset/0123a8ee6ab6/

Log:	remove old test_lib_pypy tests of numpypy

diff --git a/pypy/module/test_lib_pypy/numpypy/__init__.py b/pypy/module/test_lib_pypy/numpypy/__init__.py
deleted file mode 100644
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ /dev/null
@@ -1,202 +0,0 @@
-from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
-
-
-class AppTestFromNumeric(BaseNumpyAppTest):
-    def test_argmax(self):
-        # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import arange, argmax
-        a = arange(6).reshape((2,3))
-        assert argmax(a) == 5
-        # assert (argmax(a, axis=0) == array([1, 1, 1])).all()
-        # assert (argmax(a, axis=1) == array([2, 2])).all()
-        b = arange(6)
-        b[1] = 5
-        assert argmax(b) == 1
-
-    def test_argmin(self):
-        # tests adapted from test_argmax
-        from numpypy import arange, argmin
-        a = arange(6).reshape((2,3))
-        assert argmin(a) == 0
-        #assert (argmin(a, axis=0) == array([0, 0, 0])).all()
-        #assert (argmin(a, axis=1) == array([0, 0])).all()
-        b = arange(6)
-        b[1] = 0
-        assert argmin(b) == 0
-
-    def test_ravel(self):
-        import numpypy as np
-        a = np.ravel(np.float64(1))
-        assert np.array_equal(a, [1.])
-        a = np.ravel(np.array([[1, 2, 3], [4, 5, 6]]))
-        assert np.array_equal(a, [1, 2, 3, 4, 5, 6])
-
-    def test_shape(self):
-        # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import identity, shape
-        assert shape(identity(3)) == (3, 3)
-        assert shape([[1, 2]]) == (1, 2)
-        assert shape([0]) ==  (1,)
-        assert shape(0) == ()
-        # a = array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
-        # assert shape(a) == (2,)
-
-    def test_clip(self):
-        import numpypy as np
-        a = np.arange(10)
-        b = np.clip(a, 1, 8)
-        assert (b == [1, 1, 2, 3, 4, 5, 6, 7, 8, 8]).all()
-        assert (a == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]).all()
-        b = np.clip(a, 3, 6, out=a)
-        assert (b == [3, 3, 3, 3, 4, 5, 6, 6, 6, 6]).all()
-        assert (a == [3, 3, 3, 3, 4, 5, 6, 6, 6, 6]).all()
-        a = np.arange(10)
-        b = np.clip(a, [3,4,1,1,1,4,4,4,4,4], 8)
-        assert (b == [3, 4, 2, 3, 4, 5, 6, 7, 8, 8]).all()
-        assert (a == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]).all()
-
-    def test_sum(self):
-        # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import sum, ones, zeros, array
-        assert sum([0.5, 1.5])== 2.0
-        assert sum([[0, 1], [0, 5]]) == 6
-        # assert sum([0.5, 0.7, 0.2, 1.5], dtype=int32) == 1
-        assert (sum([[0, 1], [0, 5]], axis=0) == array([0, 6])).all()
-        assert (sum([[0, 1], [0, 5]], axis=1) == array([1, 5])).all()
-        # If the accumulator is too small, overflow occurs:
-        # assert ones(128, dtype=int8).sum(dtype=int8) == -128
-
-        assert sum(range(10)) == 45
-        assert sum(array(range(10))) == 45
-        assert list(sum(zeros((0, 2)), axis=1)) == []
-
-        a = array([[1, 2], [3, 4]])
-        out = array([[0, 0], [0, 0]])
-        c = sum(a, axis=0, out=out[0])
-        assert (c == [4, 6]).all()
-        assert (c == out[0]).all()
-        assert (c != out[1]).all()
-
-    def test_amin(self):
-        # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import array, arange, amin, zeros
-        a = arange(4).reshape((2,2))
-        assert amin(a) == 0
-        # # Minima along the first axis
-        # assert (amin(a, axis=0) == array([0, 1])).all()
-        # # Minima along the second axis
-        # assert (amin(a, axis=1) == array([0, 2])).all()
-        # # NaN behaviour
-        # b = arange(5, dtype=float)
-        # b[2] = NaN
-        # assert amin(b) == nan
-        # assert nanmin(b) == 0.0
-
-        assert amin(range(10)) == 0
-        assert amin(array(range(10))) == 0
-        assert list(amin(zeros((0, 2)), axis=1)) == []
-
-        a = array([[1, 2], [3, 4]])
-        out = array([[0, 0], [0, 0]])
-        c = amin(a, axis=1, out=out[0])
-        assert (c == [1, 3]).all()
-        assert (c == out[0]).all()
-        assert (c != out[1]).all()
-
-    def test_amax(self):
-        # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import array, arange, amax, zeros
-        a = arange(4).reshape((2,2))
-        assert amax(a) == 3
-        # assert (amax(a, axis=0) == array([2, 3])).all()
-        # assert (amax(a, axis=1) == array([1, 3])).all()
-        # # NaN behaviour
-        # b = arange(5, dtype=float)
-        # b[2] = NaN
-        # assert amax(b) == nan
-        # assert nanmax(b) == 4.0
-
-        assert amax(range(10)) == 9
-        assert amax(array(range(10))) == 9
-        assert list(amax(zeros((0, 2)), axis=1)) == []
-
-        a = array([[1, 2], [3, 4]])
-        out = array([[0, 0], [0, 0]])
-        c = amax(a, axis=1, out=out[0])
-        assert (c == [2, 4]).all()
-        assert (c == out[0]).all()
-        assert (c != out[1]).all()
-
-    def test_alen(self):
-        # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import array, zeros, alen
-        a = zeros((7,4,5))
-        assert a.shape[0] == 7
-        assert alen(a)    == 7
-
-    def test_ndim(self):
-        # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import array, ndim
-        assert ndim([[1,2,3],[4,5,6]]) == 2
-        assert ndim(array([[1,2,3],[4,5,6]])) == 2
-        assert ndim(1) == 0
-
-    def test_rank(self):
-        # tests taken from numpy/core/fromnumeric.py docstring
-        from numpypy import array, rank
-        assert rank([[1,2,3],[4,5,6]]) == 2
-        assert rank(array([[1,2,3],[4,5,6]])) == 2
-        assert rank(1) == 0
-
-    def test_var(self):
-        from numpypy import array, var
-        a = array([[1,2],[3,4]])
-        assert var(a) == 1.25
-        assert (var(a,0) == array([ 1.,  1.])).all()
-        assert (var(a,1) == array([ 0.25,  0.25])).all()
-
-    def test_std(self):
-        from numpypy import array, std
-        a = array([[1, 2], [3, 4]])
-        assert std(a) ==  1.1180339887498949
-        assert (std(a, axis=0) == array([ 1.,  1.])).all()
-        assert (std(a, axis=1) == array([ 0.5,  0.5])).all()
-
-    def test_mean(self):
-        from numpypy import array, mean, arange
-        assert mean(array(range(5))) == 2.0
-        assert mean(range(5)) == 2.0
-        assert (mean(arange(10).reshape(5, 2), axis=0) == [4, 5]).all()
-        assert (mean(arange(10).reshape(5, 2), axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all()
-
-    def test_reshape(self):
-        from numpypy import arange, array, dtype, reshape
-        a = arange(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = range(12)
-        b = reshape(a, (3, 4))
-        assert b.shape == (3, 4)
-        a = array(range(105)).reshape(3, 5, 7)
-        assert reshape(a, (1, -1)).shape == (1, 105)
-        assert reshape(a, (1, 1, -1)).shape == (1, 1, 105)
-        assert reshape(a, (-1, 1, 1)).shape == (105, 1, 1)
-
-    def test_transpose(self):
-        from numpypy import arange, array, transpose, ones
-        x = arange(4).reshape((2,2))
-        assert (transpose(x) == array([[0, 2],[1, 3]])).all()
-        # Once axes argument is implemented, add more tests
-        import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
-        # x = ones((1, 2, 3))
-        # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-
-    def test_fromnumeric(self):
-        from numpypy import array, swapaxes
-        x = array([[1,2,3]])
-        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
-        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
-        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]],
-                                          [[1, 5], [3, 7]]])).all()
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/numpypy/core/test_numeric.py
+++ /dev/null
@@ -1,250 +0,0 @@
-from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
-
-
-class AppTestBaseRepr(BaseNumpyAppTest):
-    def test_base3(self):
-        from numpypy import base_repr
-        assert base_repr(3**5, 3) == '100000'
-
-    def test_positive(self):
-        from numpypy import base_repr
-        assert base_repr(12, 10) == '12'
-        assert base_repr(12, 10, 4) == '000012'
-        assert base_repr(12, 4) == '30'
-        assert base_repr(3731624803700888, 36) == '10QR0ROFCEW'
-
-    def test_negative(self):
-        from numpypy import base_repr
-        assert base_repr(-12, 10) == '-12'
-        assert base_repr(-12, 10, 4) == '-000012'
-        assert base_repr(-12, 4) == '-30'
-
-
-class AppTestRepr(BaseNumpyAppTest):
-    def test_repr(self):
-        from numpypy import array
-        assert repr(array([1, 2, 3, 4])) == 'array([1, 2, 3, 4])'
-
-    def test_repr_2(self):
-        from numpypy import array, zeros
-        int_size = array(5).dtype.itemsize
-        a = array(range(5), float)
-        assert repr(a) == "array([ 0.,  1.,  2.,  3.,  4.])"
-        a = array([], float)
-        assert repr(a) == "array([], dtype=float64)"
-        a = zeros(1001)
-        assert repr(a) == "array([ 0.,  0.,  0., ...,  0.,  0.,  0.])"
-        a = array(range(5), int)
-        if a.dtype.itemsize == int_size:
-            assert repr(a) == "array([0, 1, 2, 3, 4])"
-        else:
-            assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)"
-        a = array(range(5), 'int32')
-        if a.dtype.itemsize == int_size:
-            assert repr(a) == "array([0, 1, 2, 3, 4])"
-        else:
-            assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)"
-        a = array([], long)
-        assert repr(a) == "array([], dtype=int64)"
-        a = array([True, False, True, False], "?")
-        assert repr(a) == "array([ True, False,  True, False], dtype=bool)"
-        a = zeros([])
-        assert repr(a) == "array(0.0)"
-        a = array(0.2)
-        assert repr(a) == "array(0.2)"
-        a = array([2])
-        assert repr(a) == "array([2])"
-
-    def test_repr_multi(self):
-        from numpypy import arange, zeros, array
-        a = zeros((3, 4))
-        assert repr(a) == '''array([[ 0.,  0.,  0.,  0.],
-       [ 0.,  0.,  0.,  0.],
-       [ 0.,  0.,  0.,  0.]])'''
-        a = zeros((2, 3, 4))
-        assert repr(a) == '''array([[[ 0.,  0.,  0.,  0.],
-        [ 0.,  0.,  0.,  0.],
-        [ 0.,  0.,  0.,  0.]],
-
-       [[ 0.,  0.,  0.,  0.],
-        [ 0.,  0.,  0.,  0.],
-        [ 0.,  0.,  0.,  0.]]])'''
-        a = arange(1002).reshape((2, 501))
-        assert repr(a) == '''array([[   0,    1,    2, ...,  498,  499,  500],
-       [ 501,  502,  503, ...,  999, 1000, 1001]])'''
-        assert repr(a.T) == '''array([[   0,  501],
-       [   1,  502],
-       [   2,  503],
-       ..., 
-       [ 498,  999],
-       [ 499, 1000],
-       [ 500, 1001]])'''
-        a = arange(2).reshape((2,1))
-        assert repr(a) == '''array([[0],
-       [1]])'''
-
-    def test_repr_slice(self):
-        from numpypy import array, zeros
-        a = array(range(5), float)
-        b = a[1::2]
-        assert repr(b) == "array([ 1.,  3.])"
-        a = zeros(2002)
-        b = a[::2]
-        assert repr(b) == "array([ 0.,  0.,  0., ...,  0.,  0.,  0.])"
-        a = array((range(5), range(5, 10)), dtype="int16")
-        b = a[1, 2:]
-        assert repr(b) == "array([7, 8, 9], dtype=int16)"
-        # an empty slice prints its shape
-        b = a[2:1, ]
-        assert repr(b) == "array([], shape=(0, 5), dtype=int16)"
-
-    def test_str(self):
-        from numpypy import array, zeros
-        a = array(range(5), float)
-        assert str(a) == "[ 0.  1.  2.  3.  4.]"
-        assert str((2 * a)[:]) == "[ 0.  2.  4.  6.  8.]"
-        a = zeros(1001)
-        assert str(a) == "[ 0.  0.  0. ...,  0.  0.  0.]"
-
-        a = array(range(5), dtype=long)
-        assert str(a) == "[0 1 2 3 4]"
-        a = array([True, False, True, False], dtype="?")
-        assert str(a) == "[ True False  True False]"
-
-        a = array(range(5), dtype="int8")
-        assert str(a) == "[0 1 2 3 4]"
-
-        a = array(range(5), dtype="int16")
-        assert str(a) == "[0 1 2 3 4]"
-
-        a = array((range(5), range(5, 10)), dtype="int16")
-        assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]"
-
-        a = array(3, dtype=int)
-        assert str(a) == "3"
-
-        a = zeros((400, 400), dtype=int)
-        assert str(a) == '[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n ..., \n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]'
-        a = zeros((2, 2, 2))
-        r = str(a)
-        assert r == '[[[ 0.  0.]\n  [ 0.  0.]]\n\n [[ 0.  0.]\n  [ 0.  0.]]]'
-
-    def test_str_slice(self):
-        from numpypy import array, zeros
-        a = array(range(5), float)
-        b = a[1::2]
-        assert str(b) == "[ 1.  3.]"
-        a = zeros(2002)
-        b = a[::2]
-        assert str(b) == "[ 0.  0.  0. ...,  0.  0.  0.]"
-        a = array((range(5), range(5, 10)), dtype="int16")
-        b = a[1, 2:]
-        assert str(b) == "[7 8 9]"
-        b = a[2:1, ]
-        assert str(b) == "[]"
-
-    def test_equal(self):
-        from numpypy import array, array_equal
-
-        a = [1, 2, 3]
-        b = [1, 2, 3]
-
-        assert array_equal(a, b)
-        assert array_equal(a, array(b))
-        assert array_equal(array(a), b)
-        assert array_equal(array(a), array(b))
-
-    def test_not_equal(self):
-        from numpypy import array, array_equal
-
-        a = [1, 2, 3]
-        b = [1, 2, 4]
-
-        assert not array_equal(a, b)
-        assert not array_equal(a, array(b))
-        assert not array_equal(array(a), b)
-        assert not array_equal(array(a), array(b))
-
-    def test_mismatched_shape(self):
-        from numpypy import array, array_equal
-
-        a = [1, 2, 3]
-        b = [[1, 2, 3], [1, 2, 3]]
-
-        assert not array_equal(a, b)
-        assert not array_equal(a, array(b))
-        assert not array_equal(array(a), b)
-        assert not array_equal(array(a), array(b))
-
-    def test_equiv(self):
-        import numpypy as np
-
-        assert np.array_equiv([1, 2], [1, 2])
-        assert not np.array_equiv([1, 2], [1, 3])
-        assert np.array_equiv([1, 2], [[1, 2], [1, 2]])
-        assert not np.array_equiv([1, 2], [[1, 2, 1, 2], [1, 2, 1, 2]])
-        assert not np.array_equiv([1, 2], [[1, 2], [1, 3]])
-
-
-class AppTestNumeric(BaseNumpyAppTest):
-    def test_argwhere(self):
-        import numpypy as np
-        x = np.arange(6).reshape(2,3)
-        a = np.argwhere(x>1)
-        assert np.array_equal(a,
-            [[0, 2],
-            [1, 0],
-            [1, 1],
-            [1, 2]]
-        )
-
-    def test_flatnonzero(self):
-        import numpypy as np
-        x = np.arange(-2, 3)
-        a = np.flatnonzero(x)
-        assert np.array_equal(a, [0, 1, 3, 4])
-
-    def test_outer(self):
-        from numpypy import array, outer
-        a = [1, 2, 3]
-        b = [4, 5, 6]
-        res = outer(a, b)
-        expected = array([[ 4,  5,  6],
-                          [ 8, 10, 12],
-                          [12, 15, 18]])
-        assert (res == expected).all()
-
-    def test_vdot(self):
-        import numpypy as np
-        a = np.array([1+2j,3+4j])
-        b = np.array([5+6j,7+8j])
-        c = np.vdot(a, b)
-        assert c == (70-8j)
-        c = np.vdot(b, a)
-        assert c == (70+8j)
-
-        a = np.array([[1, 4], [5, 6]])
-        b = np.array([[4, 1], [2, 2]])
-        c = np.vdot(a, b)
-        assert c == 30
-        c = np.vdot(b, a)
-        assert c == 30
-
-    def test_identity(self):
-        from numpypy import array, int32, float64, dtype, identity
-        a = identity(0)
-        assert len(a) == 0
-        assert a.dtype == dtype('float64')
-        assert a.shape == (0, 0)
-        b = identity(1, dtype=int32)
-        assert len(b) == 1
-        assert b[0][0] == 1
-        assert b.shape == (1, 1)
-        assert b.dtype == dtype('int32')
-        c = identity(2)
-        assert c.shape == (2, 2)
-        assert (c == [[1, 0], [0, 1]]).all()
-        d = identity(3, dtype='int32')
-        assert d.shape == (3, 3)
-        assert d.dtype == dtype('int32')
-        assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all()
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_shape_base.py b/pypy/module/test_lib_pypy/numpypy/core/test_shape_base.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/numpypy/core/test_shape_base.py
+++ /dev/null
@@ -1,131 +0,0 @@
-from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
-
-
-class AppTestShapeBase(BaseNumpyAppTest):
-
-    def test_atleast_1d(self):
-        from numpypy import array, array_equal
-        import numpypy as np
-        a = np.atleast_1d(1.0)
-        assert np.array_equal(a, [1.])
-
-        x = np.arange(9.0).reshape(3, 3)
-        a = np.atleast_1d(x)
-        assert np.array_equal(a, [[0.,  1.,  2.],
-                                  [3.,  4.,  5.],
-                                  [6.,  7.,  8.]])
-        assert np.atleast_1d(x) is x
-
-        a = np.atleast_1d(1, [3, 4])
-        assert len(a) == 2
-        assert array_equal(a[0], [1])
-        assert array_equal(a[1], [3, 4])
-
-    def test_atleast_2d(self):
-        import numpypy as np
-        a = np.atleast_2d(3.0)
-        assert np.array_equal(a, [[3.]])
-
-        x = np.arange(3.0)
-        a = np.atleast_2d(x)
-        assert np.array_equal(a, [[0., 1., 2.]])
-
-        a = np.atleast_2d(1, [1, 2], [[1, 2]])
-        assert len(a) == 3
-        assert np.array_equal(a[0], [[1]])
-        assert np.array_equal(a[1], [[1, 2]])
-        assert np.array_equal(a[2], [[1, 2]])
-
-    def test_atleast_3d(self):
-        import numpypy as np
-
-        a = np.atleast_3d(3.0)
-        assert np.array_equal(a, [[[3.]]])
-
-        x = np.arange(3.0)
-        assert np.atleast_3d(x).shape == (1, 3, 1)
-
-        x = np.arange(12.0).reshape(4, 3)
-        assert np.atleast_3d(x).shape == (4, 3, 1)
-
-        a = np.atleast_3d([1, 2])
-        assert np.array_equal(a, [[[1],
-                                   [2]]])
-        assert a.shape == (1, 2, 1)
-
-        a = np.atleast_3d([[1, 2]])
-        assert np.array_equal(a, [[[1],
-                                   [2]]])
-        assert a.shape == (1, 2, 1)
-
-        a = np.atleast_3d([[[1, 2]]])
-        assert np.array_equal(a, [[[1, 2]]])
-        assert a.shape == (1, 1, 2)
-
-    def test_vstack(self):
-        import numpypy as np
-
-        a = np.array([1, 2, 3])
-        b = np.array([2, 3, 4])
-        c = np.vstack((a, b))
-        assert np.array_equal(c, [[1, 2, 3],
-                                  [2, 3, 4]])
-
-        a = np.array([[1], [2], [3]])
-        b = np.array([[2], [3], [4]])
-        c = np.vstack((a, b))
-        assert np.array_equal(c, [[1],
-                                  [2],
-                                  [3],
-                                  [2],
-                                  [3],
-                                  [4]])
-
-        for shape1, shape2 in [[(2, 1), (3, 1)],
-                               [(2, 4), [3, 4]]]:
-            a, b = np.ones(shape1), np.ones(shape2)
-            assert np.all(np.vstack((a, b)) ==
-                          np.ones((a.shape[0] + b.shape[0],
-                                   a.shape[1])))
-
-        #skip("https://bugs.pypy.org/issue1394")
-        for shape1, shape2 in [[(3, 2, 4), (7, 2, 4)],
-                               [(0, 2, 7), (10, 2, 7)],
-                               [(0, 2, 7), (0, 2, 7)]]:
-            a, b = np.ones(shape1), np.ones(shape2)
-            assert np.all(np.vstack((a, b)) ==
-                          np.ones((a.shape[0] + b.shape[0],
-                                   a.shape[1],
-                                   a.shape[2])))
-
-    def test_hstack(self):
-        import numpypy as np
-        a = np.array((1, 2, 3))
-        b = np.array((2, 3, 4))
-        c = np.hstack((a, b))
-        assert np.array_equal(c, [1, 2, 3, 2, 3, 4])
-
-        a = np.array([[1], [2], [3]])
-        b = np.array([[2], [3], [4]])
-        c = np.hstack((a, b))
-        assert np.array_equal(c, [[1, 2],
-                                  [2, 3],
-                                  [3, 4]])
-
-        for shape1, shape2 in [[(1, 2), (1, 3)],
-                               [(4, 2), (4, 3)]]:
-            a, b = np.ones(shape1), np.ones(shape2)
-            assert np.all(np.hstack((a, b)) ==
-                          np.ones((a.shape[0],
-                                   a.shape[1] + b.shape[1])))
-
-        #skip("https://bugs.pypy.org/issue1394")
-        for shape1, shape2 in [[(2, 3, 4), (2, 7, 4)],
-                               [(1, 4, 7), (1, 10, 7)],
-                               [(1, 4, 7), (1, 0, 7)],
-                               [(1, 0, 7), (1, 0, 7)]]:
-            a, b = np.ones(shape1), np.ones(shape2)
-            assert np.all(np.hstack((a, b)) ==
-                          np.ones((a.shape[0],
-                                   a.shape[1] + b.shape[1],
-                                   a.shape[2])))
diff --git a/pypy/module/test_lib_pypy/numpypy/lib/test_function_base.py b/pypy/module/test_lib_pypy/numpypy/lib/test_function_base.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/numpypy/lib/test_function_base.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
-
-class AppTestFunctionBase(BaseNumpyAppTest):
-    def test_average(self):
-        from numpypy import array, average
-        assert average(range(10)) == 4.5
-        assert average(array(range(10))) == 4.5
diff --git a/pypy/module/test_lib_pypy/numpypy/lib/test_shape_base_lib.py b/pypy/module/test_lib_pypy/numpypy/lib/test_shape_base_lib.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/numpypy/lib/test_shape_base_lib.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
-
-class AppTestShapeBase(BaseNumpyAppTest):
-    def test_dstack(self):
-        import numpypy as np
-        a = np.array((1, 2, 3))
-        b = np.array((2, 3, 4))
-        c = np.dstack((a, b))
-        assert np.array_equal(c, [[[1, 2], [2, 3], [3, 4]]])
-
-        a = np.array([[1], [2], [3]])
-        b = np.array([[2], [3], [4]])
-        c = np.dstack((a, b))
-        assert np.array_equal(c, [[[1, 2]], [[2, 3]], [[3, 4]]])
-
-        #skip("https://bugs.pypy.org/issue1394")
-        for shape1, shape2 in [[(4, 2, 3), (4, 2, 7)],
-                               [(7, 2, 0), (7, 2, 10)],
-                               [(7, 2, 0), (7, 2, 0)]]:
-            a, b = np.ones(shape1), np.ones(shape2)
-            assert np.all(np.dstack((a, b)) ==
-                          np.ones((a.shape[0],
-                                   a.shape[1],
-                                   a.shape[2] + b.shape[2])))
-
-        for shape1, shape2 in [[(4, 2, 3, 5), (4, 2, 7, 5)],
-                               [(7, 2, 0, 5), (7, 2, 10, 5)],
-                               [(7, 2, 0, 5), (7, 2, 0, 5)]]:
-            a, b = np.ones(shape1), np.ones(shape2)
-            assert np.all(np.dstack((a, b)) ==
-                          np.ones((a.shape[0],
-                                   a.shape[1],
-                                   a.shape[2] + b.shape[2],
-                                   a.shape[3])))
diff --git a/pypy/module/test_lib_pypy/numpypy/lib/test_twodim_base.py b/pypy/module/test_lib_pypy/numpypy/lib/test_twodim_base.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/numpypy/lib/test_twodim_base.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
-
-class AppTestTwoDimBase(BaseNumpyAppTest):
-    def test_eye(self):
-        from numpypy import eye, int32, dtype
-        a = eye(0)
-        assert len(a) == 0
-        assert a.dtype == dtype('float64')
-        assert a.shape == (0, 0)
-        b = eye(1, dtype=int32)
-        assert len(b) == 1
-        assert b[0][0] == 1
-        assert b.shape == (1, 1)
-        assert b.dtype == dtype('int32')
-        c = eye(2)
-        assert c.shape == (2, 2)
-        assert (c == [[1, 0], [0, 1]]).all()
-        d = eye(3, dtype='int32')
-        assert d.shape == (3, 3)
-        assert d.dtype == dtype('int32')
-        assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all()
-        e = eye(3, 4)
-        assert e.shape == (3, 4)
-        assert (e == [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]]).all()
-        f = eye(2, 4, k=3)
-        assert f.shape == (2, 4)
-        assert (f == [[0, 0, 0, 1], [0, 0, 0, 0]]).all()
-        g = eye(3, 4, k=-1)
-        assert g.shape == (3, 4)
-        assert (g == [[0, 0, 0, 0], [1, 0, 0, 0], [0, 1, 0, 0]]).all()
diff --git a/pypy/module/test_lib_pypy/numpypy/test_base.py b/pypy/module/test_lib_pypy/numpypy/test_base.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/numpypy/test_base.py
+++ /dev/null
@@ -1,12 +0,0 @@
-class BaseNumpyAppTest(object):
-    @classmethod
-    def setup_class(cls):
-        if cls.runappdirect:
-            try:
-                import numpy
-            except ImportError:
-                skip("no numpy found")
-            import sys
-            sys.modules['numpypy'] = numpy
-        else:
-            skip("app-level tests")
diff --git a/pypy/module/test_lib_pypy/numpypy/test_numpy.py b/pypy/module/test_lib_pypy/numpypy/test_numpy.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/numpypy/test_numpy.py
+++ /dev/null
@@ -1,67 +0,0 @@
-from pypy.conftest import option
-import py, sys
-from pypy.module.test_lib_pypy.numpypy.test_base import BaseNumpyAppTest
-
-class AppTestNumpy(BaseNumpyAppTest):
-    def test_min_max_after_import(self):
-        import __builtin__
-        from __builtin__ import *
-
-        from numpypy import *
-        assert min is __builtin__.min
-        assert max is __builtin__.max
-
-        assert min(1, 100) == 1
-        assert min(100, 1) == 1
-
-        assert max(1, 100) == 100
-        assert max(100, 1) == 100
-
-        assert min(4, 3, 2, 1) == 1
-        assert max(1, 2, 3, 4) == 4
-
-        from numpypy import min, max, amin, amax
-        assert min is not __builtin__.min
-        assert max is not __builtin__.max
-        assert min is amin
-        assert max is amax
-
-    def test_builtin_aliases(self):
-        import __builtin__
-        import numpypy
-        from numpypy import *
-
-        for name in ['bool', 'int', 'long', 'float', 'complex', 'object',
-                     'unicode', 'str']:
-            assert name not in locals()
-            assert getattr(numpypy, name) is getattr(__builtin__, name)
-
-    def test_typeinfo(self):
-        import numpypy
-        assert 'typeinfo' not in dir(numpypy)
-        assert 'typeinfo' in dir(numpypy.core.multiarray)
-
-    def test_set_string_function(self):
-        import numpypy
-        assert numpypy.set_string_function is not \
-               numpypy.core.multiarray.set_string_function
-
-    def test_constants(self):
-        import math
-        import numpypy
-        assert numpypy.PZERO == numpypy.NZERO == 0.0
-        assert math.isinf(numpypy.inf)
-        assert math.isnan(numpypy.nan)
-
-    def test___all__(self):
-        import numpypy
-        assert '__all__' in dir(numpypy)
-        assert 'numpypy' not in dir(numpypy)
-
-    def test_get_include(self):
-        import numpypy, os, sys
-        assert 'get_include' in dir(numpypy)
-        path = numpypy.get_include()
-        if not hasattr(sys, 'pypy_translation_info'):
-            skip("pypy white-box test")
-        assert os.path.exists(path + '/numpy/arrayobject.h')

From noreply at buildbot.pypy.org  Fri Dec  5 21:34:27 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 21:34:27 +0100 (CET)
Subject: [pypy-commit] creflect default: ffi.getctype()
Message-ID: <20141205203427.F185C1D28F0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r181:f1168b43ad59
Date: 2014-12-05 21:34 +0100
http://bitbucket.org/cffi/creflect/changeset/f1168b43ad59/

Log:	ffi.getctype()

diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -643,10 +643,53 @@
     }
 }
 
+static PyObject *ffi_getctype(ZefFFIObject *self, PyObject *args)
+{
+    PyObject *cdecl, *res;
+    char *p, *replace_with = "";
+    int add_paren, add_space;
+    CTypeDescrObject *ct;
+    size_t replace_with_len;
+
+    if (!PyArg_ParseTuple(args, "O|s:getctype", &cdecl, &replace_with))
+        return NULL;
+
+    ct = _ffi_type(self, cdecl, ACCEPT_STRING|ACCEPT_CTYPE);
+    if (ct == NULL)
+        return NULL;
+
+    while (replace_with[0] != 0 && isspace(replace_with[0]))
+        replace_with++;
+    replace_with_len = strlen(replace_with);
+    while (replace_with_len > 0 && isspace(replace_with[replace_with_len - 1]))
+        replace_with_len--;
+
+    add_paren = (replace_with[0] == '*' &&
+                 ((ct->ct_flags & (CT_ARRAY | CT_FUNCTION)) != 0));
+    add_space = (!add_paren && replace_with_len > 0 &&
+                 replace_with[0] != '[' && replace_with[0] != '(');
+
+    res = combine_type_name_l(ct, replace_with_len + add_space + 2 * add_paren);
+    if (res == NULL)
+        return NULL;
+
+    p = PyString_AS_STRING(res) + ct->ct_name_position;
+    if (add_paren)
+        *p++ = '(';
+    if (add_space)
+        *p++ = ' ';
+    memcpy(p, replace_with, replace_with_len);
+    if (add_paren)
+        p[replace_with_len] = ')';
+    return res;
+}
+
+
 static PyMethodDef ffi_methods[] = {
     {"addressof",     (PyCFunction)ffi_addressof,METH_VARARGS},
     {"cast",          (PyCFunction)ffi_cast,     METH_VARARGS},
     {"close_library", ffi_close_library,         METH_VARARGS | METH_STATIC},
+    {"getctype",      (PyCFunction)ffi_getctype, METH_VARARGS},
     {"load_library",  (PyCFunction)ffi_load_library,
                                                  METH_VARARGS | METH_KEYWORDS},
     {"offsetof",      (PyCFunction)ffi_offsetof, METH_VARARGS},
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -54,3 +54,21 @@
     p = ffi.new("char[]", "----abcd")
     q = ffi.cast("int *", p)
     assert q[1] in [0x61626364, 0x64636261]
+
+def test_getctype():
+    ffi = support.new_ffi()
+    assert ffi.getctype("int") == "int"
+    assert ffi.getctype("int", 'x') == "int x"
+    assert ffi.getctype("int*") == "int *"
+    assert ffi.getctype("int*", '') == "int *"
+    assert ffi.getctype("int*", 'x') == "int * x"
+    assert ffi.getctype("int", '*') == "int *"
+    assert ffi.getctype("int", ' * x ') == "int * x"
+    assert ffi.getctype(ffi.typeof("int*"), '*') == "int * *"
+    assert ffi.getctype("int", '[5]') == "int[5]"
+    assert ffi.getctype("int[5]", '[6]') == "int[6][5]"
+    assert ffi.getctype("int[5]", '(*)') == "int(*)[5]"
+    # special-case for convenience: automatically put '()' around '*'
+    assert ffi.getctype("int[5]", '*') == "int(*)[5]"
+    assert ffi.getctype("int[5]", '*foo') == "int(*foo)[5]"
+    assert ffi.getctype("int[5]", ' ** foo ') == "int(** foo)[5]"
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -23,3 +23,5 @@
 static int fill_array_type(CTypeDescrObject *ctptr);
 static CTypeDescrObject *fetch_pointer_type(PyObject *types_dict,
                                             CTypeDescrObject *totype);
+static PyObject *combine_type_name_l(CTypeDescrObject *ct,
+                                     size_t extra_text_len);

From noreply at buildbot.pypy.org  Fri Dec  5 21:41:57 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 21:41:57 +0100 (CET)
Subject: [pypy-commit] creflect default: copy cdata_as_number from
	_cffi_backend
Message-ID: <20141205204157.4DA911D2A0C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r182:7f8cb0c71de4
Date: 2014-12-05 21:42 +0100
http://bitbucket.org/cffi/creflect/changeset/7f8cb0c71de4/

Log:	copy cdata_as_number from _cffi_backend

diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1050,6 +1050,85 @@
     }
 }
 
+//1699
+static int cdata_nonzero(CDataObject *cd)
+{
+    return cd->c_data != NULL;
+}
+
+
+//1751
+static PyObject *cdata_float(CDataObject *cd)
+{
+    if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
+        double value;
+        /*READ(cd->c_data, cd->c_type->ct_size)*/
+        if (!(cd->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
+            value = read_raw_float_data(cd->c_data, cd->c_type->ct_size);
+        }
+        else {
+            value = (double)read_raw_longdouble_data(cd->c_data);
+        }
+        return PyFloat_FromDouble(value);
+    }
+    PyErr_Format(PyExc_TypeError, "float() not supported on cdata '%s'",
+                 cd->c_type->ct_name);
+    return NULL;
+}
+
+//1700
+static PyObject *cdata_int(CDataObject *cd)
+{
+    if ((cd->c_type->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_FITS_LONG))
+                             == (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_FITS_LONG)) {
+        /* this case is to handle enums, but also serves as a slight
+           performance improvement for some other primitive types */
+        long value;
+        /*READ(cd->c_data, cd->c_type->ct_size)*/
+        value = (long)read_raw_signed_data(cd->c_data, cd->c_type->ct_size);
+        return PyInt_FromLong(value);
+    }
+    if (cd->c_type->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED)) {
+        return convert_to_object(cd->c_data, cd->c_type);
+    }
+    else if (cd->c_type->ct_flags & CT_PRIMITIVE_CHAR) {
+        /*READ(cd->c_data, cd->c_type->ct_size)*/
+        if (cd->c_type->ct_size == sizeof(char))
+            return PyInt_FromLong((unsigned char)cd->c_data[0]);
+#ifdef HAVE_WCHAR_H
+        else
+            return PyInt_FromLong((long)*(wchar_t *)cd->c_data);
+#endif
+    }
+    else if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
+        PyObject *o = cdata_float(cd);
+#if PY_MAJOR_VERSION < 3
+        PyObject *r = o ? PyNumber_Int(o) : NULL;
+#else
+        PyObject *r = o ? PyNumber_Long(o) : NULL;
+#endif
+        Py_XDECREF(o);
+        return r;
+    }
+    PyErr_Format(PyExc_TypeError, "int() not supported on cdata '%s'",
+                 cd->c_type->ct_name);
+    return NULL;
+}
+
+#if PY_MAJOR_VERSION < 3
+//1739
+static PyObject *cdata_long(CDataObject *cd)
+{
+    PyObject *res = cdata_int(cd);
+    if (res != NULL && PyInt_CheckExact(res)) {
+        PyObject *o = PyLong_FromLong(PyInt_AS_LONG(res));
+        Py_DECREF(res);
+        res = o;
+    }
+    return res;
+}
+#endif
+
 //1769
 static PyObject *cdata_richcompare(PyObject *v, PyObject *w, int op)
 {
@@ -1337,6 +1416,97 @@
     return convert_from_object(c, ctitem, v);
 }
 
+//2081
+static PyObject *_cdata_add_or_sub(PyObject *v, PyObject *w, int sign)
+{
+    Py_ssize_t i, itemsize;
+    CDataObject *cd;
+    CTypeDescrObject *ctptr;
+
+    if (!CData_Check(v)) {
+        PyObject *swap;
+        assert(CData_Check(w));
+        if (sign != 1)
+            goto not_implemented;
+        swap = v;
+        v = w;
+        w = swap;
+    }
+
+    i = PyNumber_AsSsize_t(w, PyExc_OverflowError);
+    if (i == -1 && PyErr_Occurred())
+        return NULL;
+    i *= sign;
+
+    cd = (CDataObject *)v;
+    if (cd->c_type->ct_flags & CT_POINTER)
+        ctptr = cd->c_type;
+    else if (cd->c_type->ct_flags & CT_ARRAY) {
+        ctptr = (CTypeDescrObject *)cd->c_type->ct_stuff;
+    }
+    else {
+        PyErr_Format(PyExc_TypeError, "cannot add a cdata '%s' and a number",
+                     cd->c_type->ct_name);
+        return NULL;
+    }
+    itemsize = ctptr->ct_itemdescr->ct_size;
+    if (itemsize < 0) {
+        if (ctptr->ct_flags & CT_IS_VOID_PTR) {
+            itemsize = 1;
+        }
+        else {
+            PyErr_Format(PyExc_TypeError,
+                         "ctype '%s' points to items of unknown size",
+                         cd->c_type->ct_name);
+            return NULL;
+        }
+    }
+    return new_simple_cdata(cd->c_data + i * itemsize, ctptr);
+
+ not_implemented:
+    Py_INCREF(Py_NotImplemented);
+    return Py_NotImplemented;
+}
+
+//2133
+static PyObject *cdata_add(PyObject *v, PyObject *w)
+{
+    return _cdata_add_or_sub(v, w, +1);
+}
+
+//2139
+static PyObject *cdata_sub(PyObject *v, PyObject *w)
+{
+    if (CData_Check(v) && CData_Check(w)) {
+        CDataObject *cdv = (CDataObject *)v;
+        CDataObject *cdw = (CDataObject *)w;
+        CTypeDescrObject *ct = cdw->c_type;
+        Py_ssize_t diff, itemsize;
+
+        if (ct->ct_flags & CT_ARRAY)     /* ptr_to_T - array_of_T: ok */
+            ct = (CTypeDescrObject *)ct->ct_stuff;
+
+        if (ct != cdv->c_type || !(ct->ct_flags & CT_POINTER) ||
+                (ct->ct_itemdescr->ct_size <= 0 &&
+                 !(ct->ct_flags & CT_IS_VOID_PTR))) {
+            PyErr_Format(PyExc_TypeError,
+                         "cannot subtract cdata '%s' and cdata '%s'",
+                         cdv->c_type->ct_name, ct->ct_name);
+            return NULL;
+        }
+        itemsize = ct->ct_itemdescr->ct_size;
+        if (itemsize <= 0) itemsize = 1;
+        diff = (cdv->c_data - cdw->c_data) / itemsize;
+#if PY_MAJOR_VERSION < 3
+        return PyInt_FromSsize_t(diff);
+#else
+        return PyLong_FromSsize_t(diff);
+#endif
+    }
+
+    return _cdata_add_or_sub(v, w, -1);
+}
+
 //2172
 static PyObject *cdata_getattro(CDataObject *cd, PyObject *attr)
 {
@@ -1390,6 +1560,40 @@
 }
 
 
+static PyNumberMethods CData_as_number = {
+    (binaryfunc)cdata_add,      /*nb_add*/
+    (binaryfunc)cdata_sub,      /*nb_subtract*/
+    0,                          /*nb_multiply*/
+#if PY_MAJOR_VERSION < 3
+    0,                          /*nb_divide*/
+#endif
+    0,                          /*nb_remainder*/
+    0,                          /*nb_divmod*/
+    0,                          /*nb_power*/
+    0,                          /*nb_negative*/
+    0,                          /*nb_positive*/
+    0,                          /*nb_absolute*/
+    (inquiry)cdata_nonzero,     /*nb_nonzero*/
+    0,                          /*nb_invert*/
+    0,                          /*nb_lshift*/
+    0,                          /*nb_rshift*/
+    0,                          /*nb_and*/
+    0,                          /*nb_xor*/
+    0,                          /*nb_or*/
+#if PY_MAJOR_VERSION < 3
+    0,                          /*nb_coerce*/
+#endif
+    (unaryfunc)cdata_int,       /*nb_int*/
+#if PY_MAJOR_VERSION < 3
+    (unaryfunc)cdata_long,      /*nb_long*/
+#else
+    0,
+#endif
+    (unaryfunc)cdata_float,     /*nb_float*/
+    0,                          /*nb_oct*/
+    0,                          /*nb_hex*/
+};
+
 static PyMappingMethods CData_as_mapping = {
     (lenfunc)cdata_length, /*mp_length*/
     (binaryfunc)cdata_subscript, /*mp_subscript*/
@@ -1407,7 +1611,7 @@
     0,                                          /* tp_setattr */
     0,                                          /* tp_compare */
     (reprfunc)cdata_repr,                       /* tp_repr */
-    0,//&CData_as_number,                           /* tp_as_number */
+    &CData_as_number,                           /* tp_as_number */
     0,                                          /* tp_as_sequence */
     &CData_as_mapping,                          /* tp_as_mapping */
     (hashfunc)cdata_hash,                       /* tp_hash */
diff --git a/zeffir/test/test_cdata.py b/zeffir/test/test_cdata.py
--- a/zeffir/test/test_cdata.py
+++ b/zeffir/test/test_cdata.py
@@ -25,3 +25,16 @@
     ffi = support.new_ffi()
     p = ffi.new("int[]", [10, 20, 30])
     assert p[2] == 30
+
+def test_cdata_as_number():
+    ffi = support.new_ffi()
+    p = ffi.new("int[]", [10, 20, 30])
+    assert (p + 2)[0] == 30
+    assert (p + 2) - p == 2
+    assert (p + 2) - (p + 1) == 1
+    assert p
+    assert not ffi.NULL
+    assert int(ffi.cast("short", 40000)) == 40000 - 65536
+    assert long(ffi.cast("short", 40000)) == 40000 - 65536
+    x = float(ffi.cast("float", 1.222))
+    assert 1E-15 < abs(x - 1.222) < 1E-8    # must get rounding errors

From noreply at buildbot.pypy.org  Fri Dec  5 21:44:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 21:44:47 +0100 (CET)
Subject: [pypy-commit] creflect default: cdata iteration (copied from
	_cffi_backend)
Message-ID: <20141205204447.0834F1D2A0C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r183:0290e34a02b3
Date: 2014-12-05 21:45 +0100
http://bitbucket.org/cffi/creflect/changeset/0290e34a02b3/

Log:	cdata iteration (copied from _cffi_backend)

diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1559,6 +1559,7 @@
     return PyObject_GenericSetAttr((PyObject *)cd, attr, value);
 }
 
+static PyObject *cdata_iter(CDataObject *);
 
 static PyNumberMethods CData_as_number = {
     (binaryfunc)cdata_add,      /*nb_add*/
@@ -1626,7 +1627,7 @@
     0,                                          /* tp_clear */
     cdata_richcompare,                          /* tp_richcompare */
     offsetof(CDataObject, c_weakreflist),       /* tp_weaklistoffset */
-    0,//(getiterfunc)cdata_iter,                    /* tp_iter */
+    (getiterfunc)cdata_iter,                    /* tp_iter */
     0,                                          /* tp_iternext */
 };
 
diff --git a/zeffir/citer.c b/zeffir/citer.c
new file mode 100644
--- /dev/null
+++ b/zeffir/citer.c
@@ -0,0 +1,79 @@
+
+typedef struct {
+    PyObject_HEAD
+    char *di_next, *di_stop;
+    CDataObject *di_object;
+    CTypeDescrObject *di_itemtype;
+} CDataIterObject;
+
+static PyObject *
+cdataiter_next(CDataIterObject *it)
+{
+    char *result = it->di_next;
+    if (result != it->di_stop) {
+        it->di_next = result + it->di_itemtype->ct_size;
+        return convert_to_object(result, it->di_itemtype);
+    }
+    return NULL;
+}
+
+static void
+cdataiter_dealloc(CDataIterObject *it)
+{
+    Py_DECREF(it->di_object);
+    PyObject_Del(it);
+}
+
+static PyTypeObject CDataIter_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "_cffi_backend.CDataIter",              /* tp_name */
+    sizeof(CDataIterObject),                /* tp_basicsize */
+    0,                                      /* tp_itemsize */
+    /* methods */
+    (destructor)cdataiter_dealloc,          /* tp_dealloc */
+    0,                                      /* tp_print */
+    0,                                      /* tp_getattr */
+    0,                                      /* tp_setattr */
+    0,                                      /* tp_compare */
+    0,                                      /* tp_repr */
+    0,                                      /* tp_as_number */
+    0,                                      /* tp_as_sequence */
+    0,                                      /* tp_as_mapping */
+    0,                                      /* tp_hash */
+    0,                                      /* tp_call */
+    0,                                      /* tp_str */
+    PyObject_GenericGetAttr,                /* tp_getattro */
+    0,                                      /* tp_setattro */
+    0,                                      /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                     /* tp_flags */
+    0,                                      /* tp_doc */
+    0,                                      /* tp_traverse */
+    0,                                      /* tp_clear */
+    0,                                      /* tp_richcompare */
+    0,                                      /* tp_weaklistoffset */
+    PyObject_SelfIter,                      /* tp_iter */
+    (iternextfunc)cdataiter_next,           /* tp_iternext */
+};
+
+static PyObject *
+cdata_iter(CDataObject *cd)
+{
+    CDataIterObject *it;
+
+    if (!(cd->c_type->ct_flags & CT_ARRAY)) {
+        PyErr_Format(PyExc_TypeError, "cdata '%s' does not support iteration",
+                     cd->c_type->ct_name);
+        return NULL;
+    }
+
+    it = PyObject_New(CDataIterObject, &CDataIter_Type);
+    if (it == NULL)
+        return NULL;
+
+    Py_INCREF(cd);
+    it->di_object = cd;
+    it->di_itemtype = cd->c_type->ct_itemdescr;
+    it->di_next = cd->c_data;
+    it->di_stop = cd->c_data + get_array_length(cd) * it->di_itemtype->ct_size;
+    return (PyObject *)it;
+}
diff --git a/zeffir/test/test_cdata.py b/zeffir/test/test_cdata.py
--- a/zeffir/test/test_cdata.py
+++ b/zeffir/test/test_cdata.py
@@ -38,3 +38,9 @@
     assert long(ffi.cast("short", 40000)) == 40000 - 65536
     x = float(ffi.cast("float", 1.222))
     assert 1E-15 < abs(x - 1.222) < 1E-8    # must get rounding errors
+
+def test_cdata_iter():
+    ffi = support.new_ffi()
+    p = ffi.new("int[]", [10, 20, 30])
+    it = iter(p)
+    assert list(it) == [10, 20, 30]
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -13,6 +13,7 @@
 #include "zeffir.h"
 #include "ctype.c"
 #include "cdata.c"
+#include "citer.c"
 #include "cglob.c"
 #include "lib_obj.c"
 #include "cfunc.c"

From noreply at buildbot.pypy.org  Fri Dec  5 21:56:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 21:56:10 +0100 (CET)
Subject: [pypy-commit] creflect default: ffi.new_handle(), ffi.from_handle()
Message-ID: <20141205205610.C21C21D2CF0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r184:f6141dc25e53
Date: 2014-12-05 21:56 +0100
http://bitbucket.org/cffi/creflect/changeset/f6141dc25e53/

Log:	ffi.new_handle(), ffi.from_handle()

diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1,10 +1,10 @@
 
-typedef struct {
+struct cdataobject_s {
     PyObject_HEAD
     CTypeDescrObject *c_type;
     char *c_data;
     PyObject *c_weakreflist;
-} CDataObject;
+};
 
 typedef union {
     unsigned char m_char;
@@ -884,64 +884,59 @@
 //1491
 static void cdataowninggc_dealloc(CDataObject *cd)
 {
-    abort();
-#if 0
-    assert(!(cd->c_type->ct_flags & (CT_IS_PTR_TO_OWNED |
-                                     CT_PRIMITIVE_ANY |
-                                     CT_STRUCT | CT_UNION)));
+    assert(!(cd->c_type->ct_flags & (CT_PRIMITIVE_ANY | CT_STRUCT | CT_UNION)));
     PyObject_GC_UnTrack(cd);
 
     if (cd->c_type->ct_flags & CT_IS_VOID_PTR) {        /* a handle */
         PyObject *x = (PyObject *)(cd->c_data + 42);
         Py_DECREF(x);
     }
+#if 0
     else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) {   /* a callback */
         ffi_closure *closure = (ffi_closure *)cd->c_data;
         PyObject *args = (PyObject *)(closure->user_data);
         Py_XDECREF(args);
         cffi_closure_free(closure);
     }
+#endif
     cdata_dealloc(cd);
-#endif
 }
 
 //1534
 static int cdataowninggc_traverse(CDataObject *cd, visitproc visit, void *arg)
 {
-    abort();
-#if 0
     if (cd->c_type->ct_flags & CT_IS_VOID_PTR) {        /* a handle */
         PyObject *x = (PyObject *)(cd->c_data + 42);
         Py_VISIT(x);
     }
+#if 0
     else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) {   /* a callback */
         ffi_closure *closure = (ffi_closure *)cd->c_data;
         PyObject *args = (PyObject *)(closure->user_data);
         Py_VISIT(args);
     }
+#endif
     return 0;
-#endif
 }
 
 //1548
 static int cdataowninggc_clear(CDataObject *cd)
 {
-    abort();
-#if 0
     if (cd->c_type->ct_flags & CT_IS_VOID_PTR) {        /* a handle */
         PyObject *x = (PyObject *)(cd->c_data + 42);
         Py_INCREF(Py_None);
         cd->c_data = ((char *)Py_None) - 42;
         Py_DECREF(x);
     }
+#if 0
     else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) {   /* a callback */
         ffi_closure *closure = (ffi_closure *)cd->c_data;
         PyObject *args = (PyObject *)(closure->user_data);
         closure->user_data = NULL;
         Py_XDECREF(args);
     }
+#endif
     return 0;
-#endif
 }
 
 //1599
@@ -1644,7 +1639,7 @@
     (reprfunc)cdataowning_repr,                 /* tp_repr */
     0,                                          /* tp_as_number */
     0,                                          /* tp_as_sequence */
-    0,//&CDataOwn_as_mapping,                       /* tp_as_mapping */
+    0,                                          /* tp_as_mapping */
     0,                                          /* tp_hash */
     0,                                          /* tp_call */
     0,                                          /* tp_str */
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -684,19 +684,64 @@
     return res;
 }
 
+static PyObject *ffi_new_handle(ZefFFIObject *self, PyObject *arg)
+{
+    CTypeDescrObject *ct = ZefNULL->c_type;   // <ctype 'void *'>
+    CDataObject *cd;
+
+    cd = (CDataObject *)PyObject_GC_New(CDataObject, &CDataOwningGC_Type);
+    if (cd == NULL)
+        return NULL;
+    Py_INCREF(ct);
+    cd->c_type = ct;
+    Py_INCREF(arg);
+    cd->c_data = ((char *)arg) - 42;
+    cd->c_weakreflist = NULL;
+    PyObject_GC_Track(cd);
+    return (PyObject *)cd;
+}
+
+static PyObject *ffi_from_handle(PyObject *self, PyObject *arg)
+{
+    CTypeDescrObject *ct;
+    char *raw;
+    PyObject *x;
+    if (!CData_Check(arg)) {
+        PyErr_SetString(PyExc_TypeError, "expected a 'cdata' object");
+        return NULL;
+    }
+    ct = ((CDataObject *)arg)->c_type;
+    raw = ((CDataObject *)arg)->c_data;
+    if (!(ct->ct_flags & CT_CAST_ANYTHING)) {
+        PyErr_Format(PyExc_TypeError,
+                     "expected a 'cdata' object with a 'void *' out of "
+                     "new_handle(), got '%s'", ct->ct_name);
+        return NULL;
+    }
+    if (!raw) {
+        PyErr_SetString(PyExc_RuntimeError,
+                        "cannot use from_handle() on NULL pointer");
+        return NULL;
+    }
+    x = (PyObject *)(raw + 42);
+    Py_INCREF(x);
+    return x;
+}
+
 
 static PyMethodDef ffi_methods[] = {
-    {"addressof",     (PyCFunction)ffi_addressof,METH_VARARGS},
-    {"cast",          (PyCFunction)ffi_cast,     METH_VARARGS},
-    {"close_library", ffi_close_library,         METH_VARARGS | METH_STATIC},
-    {"getctype",      (PyCFunction)ffi_getctype, METH_VARARGS},
-    {"load_library",  (PyCFunction)ffi_load_library,
-                                                 METH_VARARGS | METH_KEYWORDS},
-    {"offsetof",      (PyCFunction)ffi_offsetof, METH_VARARGS},
-    {"new",           (PyCFunction)ffi_new,      METH_VARARGS},
-    {"sizeof",        (PyCFunction)ffi_sizeof,   METH_O},
-    {"string",        (PyCFunction)ffi_string,   METH_VARARGS},
-    {"typeof",        (PyCFunction)ffi_typeof,   METH_O},
+    {"addressof",     (PyCFunction)ffi_addressof, METH_VARARGS},
+    {"cast",          (PyCFunction)ffi_cast,      METH_VARARGS},
+    {"close_library", ffi_close_library,          METH_VARARGS | METH_STATIC},
+    {"from_handle",   (PyCFunction)ffi_from_handle,METH_O},
+    {"getctype",      (PyCFunction)ffi_getctype,  METH_VARARGS},
+    {"load_library",  (PyCFunction)ffi_load_library,METH_VARARGS|METH_KEYWORDS},
+    {"offsetof",      (PyCFunction)ffi_offsetof,  METH_VARARGS},
+    {"new",           (PyCFunction)ffi_new,       METH_VARARGS},
+    {"new_handle",    (PyCFunction)ffi_new_handle,METH_O},
+    {"sizeof",        (PyCFunction)ffi_sizeof,    METH_O},
+    {"string",        (PyCFunction)ffi_string,    METH_VARARGS},
+    {"typeof",        (PyCFunction)ffi_typeof,    METH_O},
     {NULL}
 };
 
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -72,3 +72,12 @@
     assert ffi.getctype("int[5]", '*') == "int(*)[5]"
     assert ffi.getctype("int[5]", '*foo') == "int(*foo)[5]"
     assert ffi.getctype("int[5]", ' ** foo ') == "int(** foo)[5]"
+
+def test_from_handle():
+    ffi = support.new_ffi()
+    x = [5, 6]
+    x.append(7)
+    p = ffi.new_handle(x)
+    assert ffi.from_handle(p) is x
+    del x
+    assert ffi.from_handle(p) == [5, 6, 7]
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -73,16 +73,17 @@
 
     {
         CTypeDescrObject *ctvoidp;
-        PyObject *cdnull;
         PyObject *d1 = PyDict_New();
         if (d1 == NULL)
             return;
 
         ctvoidp = parse_c_decl(d1, "void*");
         if (ctvoidp) {
-            cdnull = new_simple_cdata(NULL, ctvoidp);
-            if (cdnull)
-                PyDict_SetItemString(ZefFFI_Type.tp_dict, "NULL", cdnull);
+            ZefNULL = (CDataObject *)new_simple_cdata(NULL, ctvoidp);
+            if (ZefNULL) {
+                PyDict_SetItemString(ZefFFI_Type.tp_dict, "NULL",
+                                     (PyObject *)ZefNULL);
+            }
         }
         Py_DECREF(d1);
         if (PyErr_Occurred())
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -1,5 +1,6 @@
 
 typedef struct _crx_type_s CTypeDescrObject;
+typedef struct cdataobject_s CDataObject;
 typedef struct cfieldobject_s CFieldObject;
 typedef struct ZefLibObject_s ZefLibObject;
 typedef struct ZefFFIObject_s ZefFFIObject;
@@ -12,6 +13,7 @@
 static PyTypeObject CDataOwningGC_Type;
 
 static PyObject *ZefError;
+static CDataObject *ZefNULL;
 
 static int lib_close(ZefLibObject *);
 static int load_creflect_main(ZefFFIObject *, ZefLibObject *);

From noreply at buildbot.pypy.org  Fri Dec  5 22:15:43 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  5 Dec 2014 22:15:43 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Fix various tests
Message-ID: <20141205211543.4B11F1D29F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: more-rposix
Changeset: r74841:76feb483c6b4
Date: 2014-11-11 21:00 +0100
http://bitbucket.org/pypy/pypy/changeset/76feb483c6b4/

Log:	Fix various tests

diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -10,6 +10,7 @@
 from rpython.flowspace.flowcontext import FlowingError, FlowContext
 from rpython.conftest import option
 from rpython.tool.stdlib_opcode import host_bytecode_spec
+from rpython.rlib import rposix
 
 import os
 import operator
@@ -1261,9 +1262,9 @@
         self.show(x)
         ops = x.startblock.operations
         assert ops[0].opname == 'simple_call'
-        assert ops[0].args[0].value is os.unlink
+        assert ops[0].args[0].value is rposix.unlink
         assert ops[1].opname == 'simple_call'
-        assert ops[1].args[0].value is os.unlink
+        assert ops[1].args[0].value is rposix.unlink
 
     def test_rabspath(self):
         import os.path
diff --git a/rpython/rtyper/module/test/test_posix.py b/rpython/rtyper/module/test/test_posix.py
--- a/rpython/rtyper/module/test/test_posix.py
+++ b/rpython/rtyper/module/test/test_posix.py
@@ -211,7 +211,9 @@
 
     def test_os_wstar(self):
         from rpython.rtyper.module.ll_os import RegisterOs
-        for name in RegisterOs.w_star:
+        for name in ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
+                     'WIFSIGNALED', 'WIFEXITED',
+                     'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']:
             if not hasattr(os, name):
                 continue
             def fun(s):
diff --git a/rpython/rtyper/test/test_rbuiltin.py b/rpython/rtyper/test/test_rbuiltin.py
--- a/rpython/rtyper/test/test_rbuiltin.py
+++ b/rpython/rtyper/test/test_rbuiltin.py
@@ -284,7 +284,7 @@
         count = 0
         for dir_call in enum_direct_calls(test_llinterp.typer.annotator.translator, wr_open):
             cfptr = dir_call.args[0]
-            assert self.get_callable(cfptr.value).__name__.startswith('os_open')
+            assert self.get_callable(cfptr.value).__name__ == 'open'
             count += 1
         assert count == 1
 

From noreply at buildbot.pypy.org  Fri Dec  5 22:15:44 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  5 Dec 2014 22:15:44 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Give a better name to the list of
 os.WEXITSTATUS macros
Message-ID: <20141205211544.AA9091D29F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: more-rposix
Changeset: r74842:0edb78d84f6a
Date: 2014-11-11 21:16 +0100
http://bitbucket.org/pypy/pypy/changeset/0edb78d84f6a/

Log:	Give a better name to the list of os.WEXITSTATUS macros

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.mixedmodule import MixedModule
+from rpython.rlib import rposix
 
 import os
 exec 'import %s as posix' % os.name
@@ -171,9 +172,7 @@
     if hasattr(os, 'chroot'):
         interpleveldefs['chroot'] = 'interp_posix.chroot'
 
-    for name in ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
-                 'WIFSIGNALED', 'WIFEXITED',
-                 'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']:
+    for name in rposix.WAIT_MACROS:
         if hasattr(os, name):
             interpleveldefs[name] = 'interp_posix.' + name
 
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1214,9 +1214,7 @@
     WSTAR.func_name = name
     return WSTAR
 
-for name in ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
-             'WIFSIGNALED', 'WIFEXITED',
-             'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']:
+for name in rposix.WAIT_MACROS:
     if hasattr(os, name):
         func = declare_new_w_star(name)
         globals()[name] = func
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -7,6 +7,7 @@
 from pypy.tool.pytest.objspace import gettestobjspace
 from pypy.conftest import pypydir
 from rpython.translator.c.test.test_extfunc import need_sparse_files
+from rpython.rlib import rposix
 import os
 import py
 import sys
@@ -575,9 +576,7 @@
         raises(TypeError, "os.utime('xxx', 3)")
         raises(OSError, "os.utime('somefilewhichihopewouldneverappearhere', None)")
 
-    for name in ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
-                 'WIFSIGNALED', 'WIFEXITED',
-                 'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']:
+    for name in rposix.WAIT_MACROS:
         if hasattr(os, name):
             values = [0, 1, 127, 128, 255]
             code = py.code.Source("""
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -765,9 +765,10 @@
         else:
             return bool(c_func(status))
 
-for name in ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
-             'WIFSIGNALED', 'WIFEXITED',
-             'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']:
+WAIT_MACROS = ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
+               'WIFSIGNALED', 'WIFEXITED',
+               'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']
+for name in WAIT_MACROS:
     _make_waitmacro(name)
 
 #___________________________________________________________________
diff --git a/rpython/rtyper/module/test/test_posix.py b/rpython/rtyper/module/test/test_posix.py
--- a/rpython/rtyper/module/test/test_posix.py
+++ b/rpython/rtyper/module/test/test_posix.py
@@ -210,10 +210,8 @@
             assert self.interpret(f, []) == 1
 
     def test_os_wstar(self):
-        from rpython.rtyper.module.ll_os import RegisterOs
-        for name in ['WCOREDUMP', 'WIFCONTINUED', 'WIFSTOPPED',
-                     'WIFSIGNALED', 'WIFEXITED',
-                     'WEXITSTATUS', 'WSTOPSIG', 'WTERMSIG']:
+        from rpython.rlib import rposix
+        for name in rposix.WAIT_MACROS:
             if not hasattr(os, name):
                 continue
             def fun(s):

From noreply at buildbot.pypy.org  Fri Dec  5 22:15:45 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  5 Dec 2014 22:15:45 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Found a much better way to register
 a replacement function.
Message-ID: <20141205211545.D64CD1D29F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: more-rposix
Changeset: r74843:9af5b8184a71
Date: 2014-12-05 19:52 +0100
http://bitbucket.org/pypy/pypy/changeset/9af5b8184a71/

Log:	Found a much better way to register a replacement function.

diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -12,10 +12,11 @@
 from rpython.tool.sourcetools import compile2
 from rpython.flowspace.model import (Constant, WrapException, const, Variable,
                                      SpaceOperation)
-from rpython.flowspace.specialcase import register_flow_sc, get_specialcase
+from rpython.flowspace.specialcase import register_flow_sc
 from rpython.annotator.model import (
     SomeTuple, AnnotatorError, read_can_only_throw)
 from rpython.annotator.argument import ArgumentsForTranslation
+from rpython.flowspace.specialcase import SPECIAL_CASES
 
 
 NOT_REALLY_CONST = {
@@ -569,8 +570,11 @@
         w_callable, args_w = self.args[0], self.args[1:]
         if isinstance(w_callable, Constant):
             fn = w_callable.value
-            sc = get_specialcase(fn)
-            if sc:
+            try:
+                sc = SPECIAL_CASES[fn]   # TypeError if 'fn' not hashable
+            except (KeyError, TypeError):
+                pass
+            else:
                 return sc(ctx, *args_w)
         return ctx.do_op(self)
 
@@ -585,8 +589,11 @@
         w_callable = self.args[0]
         if isinstance(w_callable, Constant):
             fn = w_callable.value
-            sc = get_specialcase(fn)
-            if sc:
+            try:
+                sc = SPECIAL_CASES[fn]   # TypeError if 'fn' not hashable
+            except (KeyError, TypeError):
+                pass
+            else:
                 from rpython.flowspace.flowcontext import FlowingError
                 raise FlowingError(
                     "should not call %r with keyword arguments" % (fn,))
diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -54,26 +54,6 @@
         from rpython.flowspace.operation import op
         return op.getattr(w_obj, w_index).eval(ctx)
 
-def get_specialcase(fn):
-    try:
-        return SPECIAL_CASES[fn]   # TypeError if 'fn' not hashable
-    except (KeyError, TypeError):
-        # Try to import modules containing special cases
-        for modname in SPECIAL_MODULES.get(getattr(fn, '__module__', None), []):
-            __import__(modname)
-        try:
-            return SPECIAL_CASES[fn]
-        except (KeyError, TypeError):
-            pass
-    return None
-
-SPECIAL_MODULES = {
-    # Modules with functions registered with @register_flow_sc, and
-    # which cannot be imported when before the flow object space
-    # (because of import loops).
-    'posix': ['rpython.rlib.rposix'],
-}
-
 # _________________________________________________________________________
 
 redirect_function(open,       'rpython.rlib.rfile.create_file')
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -290,15 +290,14 @@
 def sc_we_are_translated(ctx):
     return Constant(True)
 
+def register_replacement_for(replaced_function, sandboxed_name=None):
+    def wrap(func):
+        from rpython.rtyper.extregistry import ExtRegistryEntry
+        class ExtRegistry(ExtRegistryEntry):
+            _about_ = replaced_function
+            def compute_annotation(self):
+                return self.bookkeeper.immutablevalue(func)
 
-def register_replacement_for(replaced_function, sandboxed_name=None):
-    """Decorator that causes RPython to replace the function passed as parameter
-    with the function being defined."""
-    def wrap(func):
-        if replaced_function is not None:
-            @register_flow_sc(replaced_function)
-            def sc_redirected_function(ctx, *args_w):
-                return ctx.appcall(func, *args_w)
         if sandboxed_name:
             func._sandbox_external_name = sandboxed_name
             # XXX THIS IS NOT CORRECT. Only do this when config.sandbox.
@@ -306,7 +305,6 @@
         return func
     return wrap
 
-
 def keepalive_until_here(*values):
     pass
 

From noreply at buildbot.pypy.org  Fri Dec  5 22:15:52 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  5 Dec 2014 22:15:52 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: hg merge default
Message-ID: <20141205211552.4C1E01D29F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: more-rposix
Changeset: r74844:511d0dd384da
Date: 2014-12-05 22:06 +0100
http://bitbucket.org/pypy/pypy/changeset/511d0dd384da/

Log:	hg merge default

diff too long, truncating to 2000 out of 20185 lines

diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -59,7 +59,7 @@
     def __init__(self, basename, core=False, compiler=None, usemodules='',
                  skip=None):
         self.basename = basename
-        self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket']
+        self._usemodules = usemodules.split() + ['signal', 'time', 'itertools', '_socket']
         self._compiler = compiler
         self.core = core
         self.skip = skip
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -29,7 +29,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -40,7 +40,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
     "cStringIO", "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -64,19 +64,15 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
@@ -86,9 +82,10 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")],
 }
+if sys.platform == "win32":
+    module_suggests["cpyext"].append(("translation.shared", True))
 
 module_import_dependencies = {
     # no _rawffi if importing rpython.rlib.clibffi raises ImportError
diff --git a/pypy/doc/config/objspace.usemodules.rctime.txt b/pypy/doc/config/objspace.usemodules.rctime.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules.rctime.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Use the 'rctime' module. 
-
-'rctime' is our `rffi`_ based implementation of the builtin 'time' module.
-It supersedes the less complete :config:`objspace.usemodules.time`,
-at least for C-like targets (the C and LLVM backends).
-
-.. _`rffi`: ../rffi.html
diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt
--- a/pypy/doc/config/objspace.usemodules.time.txt
+++ b/pypy/doc/config/objspace.usemodules.time.txt
@@ -1,5 +1,1 @@
 Use the 'time' module. 
-
-Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version
-of the application-level 'time' module, at least for C-like targets (the C
-and LLVM backends).
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -205,23 +205,28 @@
 The above is true both in CPython and in PyPy.  Differences
 can occur about whether a built-in function or method will
 call an overridden method of *another* object than ``self``.
-In PyPy, they are generally always called, whereas not in
-CPython.  For example, in PyPy, ``dict1.update(dict2)``
-considers that ``dict2`` is just a general mapping object, and
-will thus call overridden ``keys()``  and ``__getitem__()``
-methods on it.  So the following code prints ``42`` on PyPy
-but ``foo`` on CPython::
+In PyPy, they are often called in cases where CPython would not.
+Two examples::
 
-    >>>> class D(dict):
-    ....     def __getitem__(self, key):
-    ....         return 42
-    ....
-    >>>>
-    >>>> d1 = {}
-    >>>> d2 = D(a='foo')
-    >>>> d1.update(d2)
-    >>>> print d1['a']
-    42
+    class D(dict):
+        def __getitem__(self, key):
+            return "%r from D" % (key,)
+
+    class A(object):
+        pass
+
+    a = A()
+    a.__dict__ = D()
+    a.foo = "a's own foo"
+    print a.foo
+    # CPython => a's own foo
+    # PyPy => 'foo' from D
+
+    glob = D(foo="base item")
+    loc = {}
+    exec "print foo" in glob, loc
+    # CPython => base item
+    # PyPy => 'foo' from D
 
 
 Mutating classes of objects which are already used as dictionary keys
@@ -292,6 +297,9 @@
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
+Notably missing from the list above are ``str`` and ``unicode``.  If your
+code relies on comparing strings with ``is``, then it might break in PyPy.
+
 
 Miscellaneous
 -------------
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -43,3 +43,11 @@
 .. branch nditer-external_loop
 
 Implement `external_loop` arguement to numpy's nditer
+
+.. branch kill-rctime
+
+Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
+
+.. branch: ssa-flow
+
+Use SSA form for flow graphs inside build_flow() and part of simplify_graph()
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,15 +1,13 @@
-"""
-Python control flow graph generation and bytecode assembly.
-"""
+"""Python control flow graph generation and bytecode assembly."""
 
-from pypy.interpreter.astcompiler import ast, symtable
-from pypy.interpreter import pycode
+from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import we_are_translated
+
+from pypy.interpreter.astcompiler import ast, misc, symtable
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pycode import PyCode
 from pypy.tool import stdlib_opcode as ops
 
-from pypy.interpreter.error import OperationError
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import rfloat
-
 
 class Instruction(object):
     """Represents a single opcode."""
@@ -21,14 +19,12 @@
         self.has_jump = False
 
     def size(self):
-        """Return the size of bytes of this instruction when it is encoded."""
+        """Return the size of bytes of this instruction when it is
+        encoded.
+        """
         if self.opcode >= ops.HAVE_ARGUMENT:
-            if self.arg > 0xFFFF:
-                return 6
-            else:
-                return 3
-        else:
-            return 1
+            return (6 if self.arg > 0xFFFF else 3)
+        return 1
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -54,9 +50,9 @@
 class Block(object):
     """A basic control flow block.
 
-    It has one entry point and several possible exit points.  Its instructions
-    may be jumps to other blocks, or if control flow reaches the end of the
-    block, it continues to next_block.
+    It has one entry point and several possible exit points.  Its
+    instructions may be jumps to other blocks, or if control flow
+    reaches the end of the block, it continues to next_block.
     """
 
     def __init__(self):
@@ -71,10 +67,10 @@
             stack.append(nextblock)
 
     def post_order(self):
-        """Return this block and its children in post order.
-        This means that the graph of blocks is first cleaned up to
-        ignore back-edges, thus turning it into a DAG.  Then the DAG
-        is linearized.  For example:
+        """Return this block and its children in post order.  This means
+        that the graph of blocks is first cleaned up to ignore
+        back-edges, thus turning it into a DAG.  Then the DAG is
+        linearized.  For example:
 
                    A --> B -\           =>     [A, D, B, C]
                      \-> D ---> C
@@ -105,7 +101,9 @@
         return resultblocks
 
     def code_size(self):
-        """Return the encoded size of all the instructions in this block."""
+        """Return the encoded size of all the instructions in this
+        block.
+        """
         i = 0
         for instr in self.instructions:
             i += instr.size()
@@ -141,6 +139,7 @@
             i += 1
     return result
 
+
 def _list_to_dict(l, offset=0):
     result = {}
     index = offset
@@ -300,11 +299,11 @@
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
         last_extended_arg_count = 0
-        # The reason for this loop is extended jumps.  EXTENDED_ARG extends the
-        # bytecode size, so it might invalidate the offsets we've already given.
-        # Thus we have to loop until the number of extended args is stable.  Any
-        # extended jump at all is extremely rare, so performance is not too
-        # concerning.
+        # The reason for this loop is extended jumps.  EXTENDED_ARG
+        # extends the bytecode size, so it might invalidate the offsets
+        # we've already given.  Thus we have to loop until the number of
+        # extended args is stable.  Any extended jump at all is
+        # extremely rare, so performance is not too concerning.
         while True:
             extended_arg_count = 0
             offset = 0
@@ -330,7 +329,8 @@
                                     instr.opcode = ops.JUMP_ABSOLUTE
                                     absolute = True
                                 elif target_op == ops.RETURN_VALUE:
-                                    # Replace JUMP_* to a RETURN into just a RETURN
+                                    # Replace JUMP_* to a RETURN into
+                                    # just a RETURN
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
@@ -345,7 +345,8 @@
                         instr.arg = jump_arg
                         if jump_arg > 0xFFFF:
                             extended_arg_count += 1
-            if extended_arg_count == last_extended_arg_count and not force_redo:
+            if (extended_arg_count == last_extended_arg_count and
+                not force_redo):
                 break
             else:
                 last_extended_arg_count = extended_arg_count
@@ -360,12 +361,14 @@
         while True:
             try:
                 w_key = space.next(w_iter)
-            except OperationError, e:
+            except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise
                 break
             w_index = space.getitem(w_consts, w_key)
-            consts_w[space.int_w(w_index)] = space.getitem(w_key, first)
+            w_constant = space.getitem(w_key, first)
+            w_constant = misc.intern_if_common_string(space, w_constant)
+            consts_w[space.int_w(w_index)] = w_constant
         return consts_w
 
     def _get_code_flags(self):
@@ -433,15 +436,16 @@
                         continue
                     addr = offset - current_off
                     # Python assumes that lineno always increases with
-                    # increasing bytecode address (lnotab is unsigned char).
-                    # Depending on when SET_LINENO instructions are emitted this
-                    # is not always true.  Consider the code:
+                    # increasing bytecode address (lnotab is unsigned
+                    # char).  Depending on when SET_LINENO instructions
+                    # are emitted this is not always true.  Consider the
+                    # code:
                     #     a = (1,
                     #          b)
-                    # In the bytecode stream, the assignment to "a" occurs after
-                    # the loading of "b".  This works with the C Python compiler
-                    # because it only generates a SET_LINENO instruction for the
-                    # assignment.
+                    # In the bytecode stream, the assignment to "a"
+                    # occurs after the loading of "b".  This works with
+                    # the C Python compiler because it only generates a
+                    # SET_LINENO instruction for the assignment.
                     if line or addr:
                         while addr > 255:
                             push(chr(255))
@@ -484,22 +488,22 @@
         free_names = _list_from_dict(self.free_vars, len(cell_names))
         flags = self._get_code_flags() | self.compile_info.flags
         bytecode = ''.join([block.get_code() for block in blocks])
-        return pycode.PyCode(self.space,
-                             self.argcount,
-                             len(self.var_names),
-                             stack_depth,
-                             flags,
-                             bytecode,
-                             list(consts_w),
-                             names,
-                             var_names,
-                             self.compile_info.filename,
-                             self.name,
-                             self.first_lineno,
-                             lnotab,
-                             free_names,
-                             cell_names,
-                             self.compile_info.hidden_applevel)
+        return PyCode(self.space,
+                      self.argcount,
+                      len(self.var_names),
+                      stack_depth,
+                      flags,
+                      bytecode,
+                      list(consts_w),
+                      names,
+                      var_names,
+                      self.compile_info.filename,
+                      self.name,
+                      self.first_lineno,
+                      lnotab,
+                      free_names,
+                      cell_names,
+                      self.compile_info.hidden_applevel)
 
 
 def _list_from_dict(d, offset=0):
@@ -510,134 +514,134 @@
 
 
 _static_opcode_stack_effects = {
-    ops.NOP : 0,
-    ops.STOP_CODE : 0,
+    ops.NOP: 0,
+    ops.STOP_CODE: 0,
 
-    ops.POP_TOP : -1,
-    ops.ROT_TWO : 0,
-    ops.ROT_THREE : 0,
-    ops.ROT_FOUR : 0,
-    ops.DUP_TOP : 1,
+    ops.POP_TOP: -1,
+    ops.ROT_TWO: 0,
+    ops.ROT_THREE: 0,
+    ops.ROT_FOUR: 0,
+    ops.DUP_TOP: 1,
 
-    ops.UNARY_POSITIVE : 0,
-    ops.UNARY_NEGATIVE : 0,
-    ops.UNARY_NOT : 0,
-    ops.UNARY_CONVERT : 0,
-    ops.UNARY_INVERT : 0,
+    ops.UNARY_POSITIVE: 0,
+    ops.UNARY_NEGATIVE: 0,
+    ops.UNARY_NOT: 0,
+    ops.UNARY_CONVERT: 0,
+    ops.UNARY_INVERT: 0,
 
-    ops.LIST_APPEND : -1,
-    ops.SET_ADD : -1,
-    ops.MAP_ADD : -2,
-    ops.STORE_MAP : -2,
+    ops.LIST_APPEND: -1,
+    ops.SET_ADD: -1,
+    ops.MAP_ADD: -2,
+    ops.STORE_MAP: -2,
 
-    ops.BINARY_POWER : -1,
-    ops.BINARY_MULTIPLY : -1,
-    ops.BINARY_DIVIDE : -1,
-    ops.BINARY_MODULO : -1,
-    ops.BINARY_ADD : -1,
-    ops.BINARY_SUBTRACT : -1,
-    ops.BINARY_SUBSCR : -1,
-    ops.BINARY_FLOOR_DIVIDE : -1,
-    ops.BINARY_TRUE_DIVIDE : -1,
-    ops.BINARY_LSHIFT : -1,
-    ops.BINARY_RSHIFT : -1,
-    ops.BINARY_AND : -1,
-    ops.BINARY_OR : -1,
-    ops.BINARY_XOR : -1,
+    ops.BINARY_POWER: -1,
+    ops.BINARY_MULTIPLY: -1,
+    ops.BINARY_DIVIDE: -1,
+    ops.BINARY_MODULO: -1,
+    ops.BINARY_ADD: -1,
+    ops.BINARY_SUBTRACT: -1,
+    ops.BINARY_SUBSCR: -1,
+    ops.BINARY_FLOOR_DIVIDE: -1,
+    ops.BINARY_TRUE_DIVIDE: -1,
+    ops.BINARY_LSHIFT: -1,
+    ops.BINARY_RSHIFT: -1,
+    ops.BINARY_AND: -1,
+    ops.BINARY_OR: -1,
+    ops.BINARY_XOR: -1,
 
-    ops.INPLACE_FLOOR_DIVIDE : -1,
-    ops.INPLACE_TRUE_DIVIDE : -1,
-    ops.INPLACE_ADD : -1,
-    ops.INPLACE_SUBTRACT : -1,
-    ops.INPLACE_MULTIPLY : -1,
-    ops.INPLACE_DIVIDE : -1,
-    ops.INPLACE_MODULO : -1,
-    ops.INPLACE_POWER : -1,
-    ops.INPLACE_LSHIFT : -1,
-    ops.INPLACE_RSHIFT : -1,
-    ops.INPLACE_AND : -1,
-    ops.INPLACE_OR : -1,
-    ops.INPLACE_XOR : -1,
+    ops.INPLACE_FLOOR_DIVIDE: -1,
+    ops.INPLACE_TRUE_DIVIDE: -1,
+    ops.INPLACE_ADD: -1,
+    ops.INPLACE_SUBTRACT: -1,
+    ops.INPLACE_MULTIPLY: -1,
+    ops.INPLACE_DIVIDE: -1,
+    ops.INPLACE_MODULO: -1,
+    ops.INPLACE_POWER: -1,
+    ops.INPLACE_LSHIFT: -1,
+    ops.INPLACE_RSHIFT: -1,
+    ops.INPLACE_AND: -1,
+    ops.INPLACE_OR: -1,
+    ops.INPLACE_XOR: -1,
 
-    ops.SLICE+0 : 1,
-    ops.SLICE+1 : 0,
-    ops.SLICE+2 : 0,
-    ops.SLICE+3 : -1,
-    ops.STORE_SLICE+0 : -2,
-    ops.STORE_SLICE+1 : -3,
-    ops.STORE_SLICE+2 : -3,
-    ops.STORE_SLICE+3 : -4,
-    ops.DELETE_SLICE+0 : -1,
-    ops.DELETE_SLICE+1 : -2,
-    ops.DELETE_SLICE+2 : -2,
-    ops.DELETE_SLICE+3 : -3,
+    ops.SLICE+0: 1,
+    ops.SLICE+1: 0,
+    ops.SLICE+2: 0,
+    ops.SLICE+3: -1,
+    ops.STORE_SLICE+0: -2,
+    ops.STORE_SLICE+1: -3,
+    ops.STORE_SLICE+2: -3,
+    ops.STORE_SLICE+3: -4,
+    ops.DELETE_SLICE+0: -1,
+    ops.DELETE_SLICE+1: -2,
+    ops.DELETE_SLICE+2: -2,
+    ops.DELETE_SLICE+3: -3,
 
-    ops.STORE_SUBSCR : -2,
-    ops.DELETE_SUBSCR : -2,
+    ops.STORE_SUBSCR: -2,
+    ops.DELETE_SUBSCR: -2,
 
-    ops.GET_ITER : 0,
-    ops.FOR_ITER : 1,
-    ops.BREAK_LOOP : 0,
-    ops.CONTINUE_LOOP : 0,
-    ops.SETUP_LOOP : 0,
+    ops.GET_ITER: 0,
+    ops.FOR_ITER: 1,
+    ops.BREAK_LOOP: 0,
+    ops.CONTINUE_LOOP: 0,
+    ops.SETUP_LOOP: 0,
 
-    ops.PRINT_EXPR : -1,
-    ops.PRINT_ITEM : -1,
-    ops.PRINT_NEWLINE : 0,
-    ops.PRINT_ITEM_TO : -2,
-    ops.PRINT_NEWLINE_TO : -1,
+    ops.PRINT_EXPR: -1,
+    ops.PRINT_ITEM: -1,
+    ops.PRINT_NEWLINE: 0,
+    ops.PRINT_ITEM_TO: -2,
+    ops.PRINT_NEWLINE_TO: -1,
 
-    ops.WITH_CLEANUP : -1,
-    ops.POP_BLOCK : 0,
-    ops.END_FINALLY : -1,
-    ops.SETUP_WITH : 1,
-    ops.SETUP_FINALLY : 0,
-    ops.SETUP_EXCEPT : 0,
+    ops.WITH_CLEANUP: -1,
+    ops.POP_BLOCK: 0,
+    ops.END_FINALLY: -1,
+    ops.SETUP_WITH: 1,
+    ops.SETUP_FINALLY: 0,
+    ops.SETUP_EXCEPT: 0,
 
-    ops.LOAD_LOCALS : 1,
-    ops.RETURN_VALUE : -1,
-    ops.EXEC_STMT : -3,
-    ops.YIELD_VALUE : 0,
-    ops.BUILD_CLASS : -2,
-    ops.BUILD_MAP : 1,
-    ops.BUILD_SET : 1,
-    ops.COMPARE_OP : -1,
+    ops.LOAD_LOCALS: 1,
+    ops.RETURN_VALUE: -1,
+    ops.EXEC_STMT: -3,
+    ops.YIELD_VALUE: 0,
+    ops.BUILD_CLASS: -2,
+    ops.BUILD_MAP: 1,
+    ops.BUILD_SET: 1,
+    ops.COMPARE_OP: -1,
 
-    ops.LOOKUP_METHOD : 1,
+    ops.LOOKUP_METHOD: 1,
 
-    ops.LOAD_NAME : 1,
-    ops.STORE_NAME : -1,
-    ops.DELETE_NAME : 0,
+    ops.LOAD_NAME: 1,
+    ops.STORE_NAME: -1,
+    ops.DELETE_NAME: 0,
 
-    ops.LOAD_FAST : 1,
-    ops.STORE_FAST : -1,
-    ops.DELETE_FAST : 0,
+    ops.LOAD_FAST: 1,
+    ops.STORE_FAST: -1,
+    ops.DELETE_FAST: 0,
 
-    ops.LOAD_ATTR : 0,
-    ops.STORE_ATTR : -2,
-    ops.DELETE_ATTR : -1,
+    ops.LOAD_ATTR: 0,
+    ops.STORE_ATTR: -2,
+    ops.DELETE_ATTR: -1,
 
-    ops.LOAD_GLOBAL : 1,
-    ops.STORE_GLOBAL : -1,
-    ops.DELETE_GLOBAL : 0,
+    ops.LOAD_GLOBAL: 1,
+    ops.STORE_GLOBAL: -1,
+    ops.DELETE_GLOBAL: 0,
 
-    ops.LOAD_CLOSURE : 1,
-    ops.LOAD_DEREF : 1,
-    ops.STORE_DEREF : -1,
+    ops.LOAD_CLOSURE: 1,
+    ops.LOAD_DEREF: 1,
+    ops.STORE_DEREF: -1,
 
-    ops.LOAD_CONST : 1,
+    ops.LOAD_CONST: 1,
 
-    ops.IMPORT_STAR : -1,
-    ops.IMPORT_NAME : -1,
-    ops.IMPORT_FROM : 1,
+    ops.IMPORT_STAR: -1,
+    ops.IMPORT_NAME: -1,
+    ops.IMPORT_FROM: 1,
 
-    ops.JUMP_FORWARD : 0,
-    ops.JUMP_ABSOLUTE : 0,
-    ops.JUMP_IF_TRUE_OR_POP : 0,
-    ops.JUMP_IF_FALSE_OR_POP : 0,
-    ops.POP_JUMP_IF_TRUE : -1,
-    ops.POP_JUMP_IF_FALSE : -1,
-    ops.JUMP_IF_NOT_DEBUG : 0,
+    ops.JUMP_FORWARD: 0,
+    ops.JUMP_ABSOLUTE: 0,
+    ops.JUMP_IF_TRUE_OR_POP: 0,
+    ops.JUMP_IF_FALSE_OR_POP: 0,
+    ops.POP_JUMP_IF_TRUE: -1,
+    ops.POP_JUMP_IF_FALSE: -1,
+    ops.JUMP_IF_NOT_DEBUG: 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -106,3 +106,13 @@
     except IndexError:
         return name
     return "_%s%s" % (klass[i:], name)
+
+
+def intern_if_common_string(space, w_const):
+    # only intern identifier-like strings
+    if not space.is_w(space.type(w_const), space.w_str):
+        return w_const
+    for c in space.str_w(w_const):
+        if not (c.isalnum() or c == '_'):
+            return w_const
+    return space.new_interned_w_str(w_const)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -272,6 +272,11 @@
                 if w_const is None:
                     return tup
                 consts_w[i] = w_const
+            # intern the string constants packed into the tuple here,
+            # because assemble.py will see the result as just a tuple constant
+            for i in range(len(consts_w)):
+                consts_w[i] = misc.intern_if_common_string(
+                    self.space, consts_w[i])
         else:
             consts_w = []
         w_consts = self.space.newtuple(consts_w)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -14,7 +14,7 @@
     UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
-from pypy.interpreter.miscutils import ThreadLocals
+from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
 
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
@@ -384,7 +384,7 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        self.interned_strings = {}
+        self.interned_strings = make_weak_value_dictionary(self, str, W_Root)
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
@@ -522,11 +522,6 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: rctime take precedence over time
-        # XXX this could probably be done as a "requires" in the config
-        if 'rctime' in modules and 'time' in modules:
-            modules.remove('time')
-
         self._builtinmodule_list = modules
         return self._builtinmodule_list
 
@@ -782,25 +777,30 @@
             return self.w_False
 
     def new_interned_w_str(self, w_s):
+        assert isinstance(w_s, W_Root)   # and is not None
         s = self.str_w(w_s)
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        self.interned_strings[s] = w_s
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = w_s
+            self.interned_strings.set(s, w_s1)
+        return w_s1
 
     def new_interned_str(self, s):
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        w_s = self.interned_strings[s] = self.wrap(s)
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = self.wrap(s)
+            self.interned_strings.set(s, w_s1)
+        return w_s1
+
+    def is_interned_str(self, s):
+        # interface for marshal_impl
+        if not we_are_translated():
+            assert type(s) is str
+        return self.interned_strings.get(s) is not None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -31,3 +31,19 @@
 
     def getallvalues(self):
         return {0: self._value}
+
+
+def make_weak_value_dictionary(space, keytype, valuetype):
+    "NOT_RPYTHON"
+    if space.config.translation.rweakref:
+        from rpython.rlib.rweakref import RWeakValueDictionary
+        return RWeakValueDictionary(keytype, valuetype)
+    else:
+        class FakeWeakValueDict(object):
+            def __init__(self):
+                self._dict = {}
+            def get(self, key):
+                return self._dict.get(key, None)
+            def set(self, key, value):
+                self._dict[key] = value
+        return FakeWeakValueDict()
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -131,7 +131,6 @@
         # class bodies only have CO_NEWLOCALS.
         # CO_NEWLOCALS: make a locals dict unless optimized is also set
         # CO_OPTIMIZED: no locals dict needed at all
-        # NB: this method is overridden in nestedscope.py
         flags = code.co_flags
         if not (flags & pycode.CO_OPTIMIZED):
             if flags & pycode.CO_NEWLOCALS:
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -970,7 +970,12 @@
             sys.stdout = out
         output = s.getvalue()
         assert "CALL_METHOD" in output
-            
+
+    def test_interned_strings(self):
+        source = """x = ('foo_bar42', 5); y = 'foo_bar42'; z = x[0]"""
+        exec source
+        assert y is z
+
 
 class AppTestExceptions:
     def test_indentation_error(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -378,3 +378,41 @@
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+
+    def test_interned_strings_are_weak(self):
+        import weakref, gc, random
+        space = self.space
+        assert space.config.translation.rweakref
+        w1 = space.new_interned_str("abcdef")
+        w2 = space.new_interned_str("abcdef")
+        assert w2 is w1
+        #
+        # check that 'w1' goes away if we don't hold a reference to it
+        rw1 = weakref.ref(w1)
+        del w1, w2
+        i = 10
+        while rw1() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
+        #
+        s = "foobar%r" % random.random()
+        w0 = space.wrap(s)
+        w1 = space.new_interned_w_str(w0)
+        assert w1 is w0
+        w2 = space.new_interned_w_str(w0)
+        assert w2 is w0
+        w3 = space.wrap(s)
+        assert w3 is not w0
+        w4 = space.new_interned_w_str(w3)
+        assert w4 is w0
+        #
+        # check that 'w0' goes away if we don't hold a reference to it
+        # (even if we hold a reference to 'w3')
+        rw0 = weakref.ref(w0)
+        del w0, w1, w2, w4
+        i = 10
+        while rw0() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -158,21 +158,14 @@
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
-    _attrs_            = ['value_fits_long', 'vmin', 'vrangemax']
-    _immutable_fields_ = ['value_fits_long', 'vmin', 'vrangemax']
+    _attrs_            = ['value_fits_long', 'value_smaller_than_long']
+    _immutable_fields_ = ['value_fits_long', 'value_smaller_than_long']
     is_primitive_integer = True
 
     def __init__(self, *args):
         W_CTypePrimitive.__init__(self, *args)
         self.value_fits_long = self.size <= rffi.sizeof(lltype.Signed)
-        if self.size < rffi.sizeof(lltype.Signed):
-            assert self.value_fits_long
-            sh = self.size * 8
-            self.vmin = r_uint(-1) << (sh - 1)
-            self.vrangemax = (r_uint(1) << sh) - 1
-        else:
-            self.vmin = r_uint(0)
-            self.vrangemax = r_uint(-1)
+        self.value_smaller_than_long = self.size < rffi.sizeof(lltype.Signed)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -192,8 +185,17 @@
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
             value = misc.as_long(self.space, w_ob)
-            if self.size < rffi.sizeof(lltype.Signed):
-                if r_uint(value) - self.vmin > self.vrangemax:
+            if self.value_smaller_than_long:
+                size = self.size
+                if size == 1:
+                    signextended = misc.signext(value, 1)
+                elif size == 2:
+                    signextended = misc.signext(value, 2)
+                elif size == 4:
+                    signextended = misc.signext(value, 4)
+                else:
+                    raise AssertionError("unsupported size")
+                if value != signextended:
                     self._overflow(w_ob)
             misc.write_raw_signed_data(cdata, value, self.size)
         else:
@@ -221,7 +223,7 @@
             length = w_cdata.get_array_length()
             populate_list_from_raw_array(res, buf, length)
             return res
-        elif self.value_fits_long:
+        elif self.value_smaller_than_long:
             res = [0] * w_cdata.get_array_length()
             misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
             return res
@@ -235,8 +237,8 @@
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
-                overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                overflowed = misc.pack_list_to_raw_array_bounds_signed(
+                    int_list, cdata, self.size)
                 if overflowed != 0:
                     self._overflow(self.space.wrap(overflowed))
             return True
@@ -314,8 +316,8 @@
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
-            overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
+                int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.wrap(overflowed))
             return True
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -216,6 +216,19 @@
 neg_msg = "can't convert negative number to unsigned"
 ovf_msg = "long too big to convert"
 
+ at specialize.arg(1)
+def signext(value, size):
+    # 'value' is sign-extended from 'size' bytes to a full integer.
+    # 'size' should be a constant smaller than a full integer size.
+    if size == rffi.sizeof(rffi.SIGNEDCHAR):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SIGNEDCHAR, value))
+    elif size == rffi.sizeof(rffi.SHORT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SHORT, value))
+    elif size == rffi.sizeof(rffi.INT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+    else:
+        raise AssertionError("unsupported size")
+
 # ____________________________________________________________
 
 class _NotStandardObject(Exception):
@@ -334,13 +347,26 @@
 
 # ____________________________________________________________
 
-def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+def pack_list_to_raw_array_bounds_signed(int_list, target, size):
     for TP, TPP in _prim_signed_types:
         if size == rffi.sizeof(TP):
             ptr = rffi.cast(TPP, target)
             for i in range(len(int_list)):
                 x = int_list[i]
-                if r_uint(x) - vmin > vrangemax:
+                y = rffi.cast(TP, x)
+                if x != rffi.cast(lltype.Signed, y):
+                    return x      # overflow
+                ptr[i] = y
+            return 0
+    raise NotImplementedError("bad integer size")
+
+def pack_list_to_raw_array_bounds_unsigned(int_list, target, size, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) > vrangemax:
                     return x      # overflow
                 ptr[i] = rffi.cast(TP, x)
             return 0
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -304,7 +304,7 @@
             py.test.skip("works with internals of _file impl on py.py")
         state = [0]
         def read(fd, n=None):
-            if fd != 42:
+            if fd != 424242:
                 return cls.old_read(fd, n)
             if state[0] == 0:
                 state[0] += 1
@@ -315,7 +315,7 @@
             return ''
         os.read = read
         stdin = W_File(cls.space)
-        stdin.file_fdopen(42, 'rb', 1)
+        stdin.file_fdopen(424242, 'rb', 1)
         stdin.name = '<stdin>'
         cls.w_stream = stdin
 
diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py
--- a/pypy/module/_file/test/test_file_extra.py
+++ b/pypy/module/_file/test/test_file_extra.py
@@ -221,7 +221,7 @@
     expected_filename = str(udir.join('sample'))
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -281,7 +281,7 @@
     expected_filename = '<fdopen>'
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -359,7 +359,7 @@
 #  A few extra tests
 
 class AppTestAFewExtra:
-    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'rctime',
+    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'time',
                                   'struct']}
 
     def setup_method(self, method):
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -1,6 +1,6 @@
 class AppTestCProfile(object):
     spaceconfig = {
-        "usemodules": ['_lsprof', 'rctime'],
+        "usemodules": ['_lsprof', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -5,7 +5,7 @@
 
 class AppTestMD5(object):
     spaceconfig = {
-        'usemodules': ['_md5', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_md5', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
@@ -97,24 +97,24 @@
   Py_UNICODE *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_dec_s *pypy_cjk_dec_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_init(struct pypy_cjk_dec_s *d,
                              char *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_dec_free(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_chunk(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_UNICODE *pypy_cjk_dec_outbuf(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_outlen(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_remaining(struct pypy_cjk_dec_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_consumed(struct pypy_cjk_dec_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_replace_on_error(struct pypy_cjk_dec_s* d,
                                          Py_UNICODE *, Py_ssize_t, Py_ssize_t);
 
@@ -125,35 +125,35 @@
   unsigned char *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_enc_s *pypy_cjk_enc_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_init(struct pypy_cjk_enc_s *d,
                              Py_UNICODE *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_enc_free(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_chunk(struct pypy_cjk_enc_s *, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_reset(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char *pypy_cjk_enc_outbuf(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_outlen(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_remaining(struct pypy_cjk_enc_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_consumed(struct pypy_cjk_enc_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_replace_on_error(struct pypy_cjk_enc_s* d,
                                          char *, Py_ssize_t, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 const MultibyteCodec *pypy_cjk_enc_getcodec(struct pypy_cjk_enc_s *);
 
 /* list of codecs defined in the .c files */
 
 #define DEFINE_CODEC(name)                              \
-    RPY_EXPORTED_FOR_TESTS MultibyteCodec *pypy_cjkcodec_##name(void);
+    RPY_EXTERN MultibyteCodec *pypy_cjkcodec_##name(void);
 
 // _codecs_cn
 DEFINE_CODEC(gb2312)
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -254,7 +254,7 @@
         start = _GetTickCount()
 
         while True:
-            from pypy.module.rctime.interp_time import State
+            from pypy.module.time.interp_time import State
             interrupt_event = space.fromcache(State).get_interrupt_event()
             handles = [self.handle, interrupt_event]
 
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -1,6 +1,6 @@
 class AppTestRandom:
     spaceconfig = {
-        "usemodules": ['_random', 'rctime'],
+        "usemodules": ['_random', 'time'],
     }
 
     def test_dict(self):
diff --git a/pypy/module/_sha/test/test_sha.py b/pypy/module/_sha/test/test_sha.py
--- a/pypy/module/_sha/test/test_sha.py
+++ b/pypy/module/_sha/test/test_sha.py
@@ -5,7 +5,7 @@
 
 class AppTestSHA(object):
     spaceconfig = {
-        'usemodules': ['_sha', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_sha', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -24,12 +24,19 @@
 
 separate_module_source = """
 #include <openssl/crypto.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 
 static unsigned int _ssl_locks_count = 0;
 static struct RPyOpaque_ThreadLock *_ssl_locks;
 
 static unsigned long _ssl_thread_id_function(void) {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (unsigned long)GetCurrentThreadId();
+#else
+    return (unsigned long)pthread_self();
+#endif
 }
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
@@ -65,7 +72,7 @@
 eci = rthread.eci.merge(ExternalCompilationInfo(
     separate_module_sources=[separate_module_source],
     post_include_bits=[
-        "RPY_EXPORTED_FOR_TESTS int _PyPy_SSL_SetupThreads(void);"],
+        "RPY_EXTERN int _PyPy_SSL_SetupThreads(void);"],
     libraries = libraries,
 ))
 
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -53,7 +53,7 @@
 
 class AppTestBZ2File(CheckAllocation):
     spaceconfig = {
-        'usemodules': ['bz2', 'binascii', 'rctime', 'struct']
+        'usemodules': ['bz2', 'binascii', 'time', 'struct']
     }
 
     def setup_class(cls):
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -2,6 +2,7 @@
 #define CPPYY_CAPI
 
 #include <stddef.h>
+#include "src/precommondefs.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -15,102 +16,167 @@
     typedef void* (*cppyy_methptrgetter_t)(cppyy_object_t);
 
     /* name to opaque C++ scope representation -------------------------------- */
+    RPY_EXTERN
     int cppyy_num_scopes(cppyy_scope_t parent);
+    RPY_EXTERN
     char* cppyy_scope_name(cppyy_scope_t parent, int iscope);
 
+    RPY_EXTERN
     char* cppyy_resolve_name(const char* cppitem_name);
+    RPY_EXTERN
     cppyy_scope_t cppyy_get_scope(const char* scope_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_get_template(const char* template_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj);
 
     /* memory management ------------------------------------------------------ */
+    RPY_EXTERN
     cppyy_object_t cppyy_allocate(cppyy_type_t type);
+    RPY_EXTERN
     void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self);
+    RPY_EXTERN
     void cppyy_destruct(cppyy_type_t type, cppyy_object_t self);
 
     /* method/function dispatching -------------------------------------------- */
+    RPY_EXTERN
     void   cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char   cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     short  cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     int    cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long   cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     float  cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args);
+    RPY_EXTERN
     cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type);
 
+    RPY_EXTERN
     cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx);
 
     /* handling of function argument buffer ----------------------------------- */
+    RPY_EXTERN
     void*  cppyy_allocate_function_args(int nargs);
+    RPY_EXTERN
     void   cppyy_deallocate_function_args(void* args);
+    RPY_EXTERN
     size_t cppyy_function_arg_sizeof();
+    RPY_EXTERN
     size_t cppyy_function_arg_typeoffset();
 
     /* scope reflection information ------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_namespace(cppyy_scope_t scope);
+    RPY_EXTERN
     int cppyy_is_enum(const char* type_name);
 
     /* class reflection information ------------------------------------------- */
+    RPY_EXTERN
     char* cppyy_final_name(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_scoped_final_name(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_has_complex_hierarchy(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_num_bases(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_base_name(cppyy_type_t type, int base_index);
+    RPY_EXTERN
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
 
     /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */
+    RPY_EXTERN
     ptrdiff_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
 
     /* method/function reflection information --------------------------------- */
+    RPY_EXTERN
     int cppyy_num_methods(cppyy_scope_t scope);
+    RPY_EXTERN
     cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth);
+    RPY_EXTERN
     cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name);
 
+    RPY_EXTERN
     char* cppyy_method_name(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_result_type(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_req_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_arg_type(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_arg_default(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx);
 
+    RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_template_arg_name(cppyy_scope_t scope, cppyy_index_t idx, cppyy_index_t iarg);
 
+    RPY_EXTERN
     cppyy_method_t cppyy_get_method(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     cppyy_index_t cppyy_get_global_operator(
         cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op);
 
     /* method properties ------------------------------------------------------ */
+    RPY_EXTERN
     int cppyy_is_constructor(cppyy_type_t type, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_is_staticmethod(cppyy_type_t type, cppyy_index_t idx);
 
     /* data member reflection information ------------------------------------- */
+    RPY_EXTERN
     int cppyy_num_datamembers(cppyy_scope_t scope);
+    RPY_EXTERN
     char* cppyy_datamember_name(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
 
+    RPY_EXTERN
     int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 
     /* data member properties ------------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_publicdata(cppyy_type_t type, int datamember_index);
+    RPY_EXTERN
     int cppyy_is_staticdata(cppyy_type_t type, int datamember_index);
 
     /* misc helpers ----------------------------------------------------------- */
+    RPY_EXTERN
     long long cppyy_strtoll(const char* str);
+    RPY_EXTERN
     unsigned long long cppyy_strtoull(const char* str);
+    RPY_EXTERN
     void cppyy_free(void* ptr);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_charp2stdstring(const char* str);
+    RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
 #ifdef __cplusplus
diff --git a/pypy/module/cppyy/src/dummy_backend.cxx b/pypy/module/cppyy/src/dummy_backend.cxx
--- a/pypy/module/cppyy/src/dummy_backend.cxx
+++ b/pypy/module/cppyy/src/dummy_backend.cxx
@@ -1,4 +1,3 @@
-#include "src/precommondefs.h"
 #include "cppyy.h"
 #include "capi.h"
 
@@ -349,29 +348,24 @@
 
 
 /* name to opaque C++ scope representation -------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_scopes(cppyy_scope_t handle) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_resolve_name(const char* cppitem_name) {
     return cppstring_to_cstring(cppitem_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_scope_t cppyy_get_scope(const char* scope_name) {
     return s_handles[scope_name];  // lookup failure will return 0 (== error)
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t /* obj */) {
     return klass;
 }
 
 
 /* memory management ------------------------------------------------------ */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
     if (handle == s_handles["example01"])
        delete (dummy::example01*)self;
@@ -379,7 +373,6 @@
 
 
 /* method/function dispatching -------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long idx = (long)method;
     if (idx == s_methods["static_example01::staticSetPayload_payload*_double"]) {
@@ -469,7 +462,6 @@
     }
 }
 
-RPY_EXPORTED_FOR_TESTS
 unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     unsigned char result = 0;
     const long idx = (long)method;
@@ -482,7 +474,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char result = 0;
     const long idx = (long)method;
@@ -498,7 +489,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     short result = 0;
     const long idx = (long)method; 
@@ -514,7 +504,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     int result = 0;
     const long idx = (long)method;
@@ -547,7 +536,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long result = 0;
     const long idx = (long)method;
@@ -689,7 +677,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long long result = 0;
     const long idx = (long)method;
@@ -705,7 +692,6 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     float result = 0;
     const long idx = (long)method;
@@ -718,7 +704,6 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     double result = 0.;
     const long idx = (long)method;
@@ -740,7 +725,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char* result = 0;
     const long idx = (long)method;
@@ -753,7 +737,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
     void* result = 0;
     const long idx = (long)method;
@@ -776,14 +759,12 @@
     return (cppyy_object_t)result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) {
     return (cppyy_methptrgetter_t)0;
 }
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void* cppyy_allocate_function_args(int nargs) {
     CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
     for (int i = 0; i < nargs; ++i)
@@ -793,36 +774,30 @@
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_deallocate_function_args(void* args) {
     free(args);
 }
 
-RPY_EXPORTED_FOR_TESTS
 size_t cppyy_function_arg_sizeof() {
     return sizeof(CPPYY_G__value);
 }
 
-RPY_EXPORTED_FOR_TESTS
 size_t cppyy_function_arg_typeoffset() {
     return offsetof(CPPYY_G__value, type);
 }
 
 
 /* scope reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_namespace(cppyy_scope_t /* handle */) {
     return 0;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_enum(const char* /* type_name */) {
     return 0;
 }
     
     
 /* class reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_final_name(cppyy_type_t handle) {
     for (Handles_t::iterator isp = s_handles.begin(); isp != s_handles.end(); ++isp) {
         if (isp->second == handle)
@@ -831,75 +806,61 @@
     return cppstring_to_cstring("<unknown>");
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_scoped_final_name(cppyy_type_t handle) {
     return cppyy_final_name(handle);
 }   
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_has_complex_hierarchy(cppyy_type_t /* handle */) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_bases(cppyy_type_t /*handle*/) {
    return 0;
 }
 
 
 /* method/function reflection information --------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_methods(cppyy_scope_t handle) {
     return s_scopes[handle].m_methods.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) {
     return (cppyy_index_t)imeth;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_returntype);
 }
     
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return s_scopes[handle].m_methods[method_index].m_argtypes.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppyy_method_num_args(handle, method_index);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_argtypes[arg_index]);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_arg_default(
         cppyy_scope_t /* handle */, cppyy_index_t /* method_index */, int /* arg_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_signature(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_is_template(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return 0;
 }
     
-RPY_EXPORTED_FOR_TESTS
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end()) {
         long id = s_scopes[handle].m_method_offset + (long)method_index;
@@ -911,7 +872,6 @@
 
 
 /* method properties -----------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kConstructor;
@@ -919,7 +879,6 @@
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kStatic;
@@ -929,34 +888,28 @@
 
 
 /* data member reflection information ------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_datamembers(cppyy_scope_t handle) {
     return s_scopes[handle].m_datambrs.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_datamember_name(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_datamember_type(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_type);
 }
 
-RPY_EXPORTED_FOR_TESTS
 ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_offset;
 }
 
 
 /* data member properties ------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) {
     return 1;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
@@ -964,44 +917,37 @@
 
 /* misc helpers ----------------------------------------------------------- */
 #if defined(_MSC_VER)
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_strtoll(const char* str) {
     return _strtoi64(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
 unsigned long long cppyy_strtoull(const char* str) {
     return _strtoui64(str, NULL, 0);
 }
 }
 #else
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_strtoll(const char* str) {
     return strtoll(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
 unsigned long long cppyy_strtoull(const char* str) {
     return strtoull(str, NULL, 0);
 }
 }
 #endif
 
-RPY_EXPORTED_FOR_TESTS
 void cppyy_free(void* ptr) {
     free(ptr);
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_charp2stdstring(const char* str) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(str);
     return (cppyy_object_t)arena;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(*(std::string*)ptr);
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -50,7 +50,7 @@
             eci = ExternalCompilationInfo(
                 separate_module_files=[srcpath.join('dummy_backend.cxx')],
                 include_dirs=[incpath, tstpath, cdir],
-                compile_extra=['-DRPY_EXPORTED_FOR_TESTS=RPY_EXPORTED'],
+                compile_extra=['-DRPY_EXTERN=RPY_EXPORTED'],
                 use_cpp_linker=True,
             )
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -775,8 +775,7 @@
     struct PyPyAPI {
     %(members)s
     } _pypyAPI;
-    RPY_EXPORTED_FOR_TESTS
-    struct PyPyAPI* pypyAPI = &_pypyAPI;
+    RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
     functions = generate_decls_and_callbacks(db, export_symbols)
@@ -947,7 +946,7 @@
         name_no_star = process_va_name(name)
         header = ('%s pypy_va_get_%s(va_list* vp)' %
                   (name, name_no_star))
-        pypy_decls.append('RPY_EXPORTED_FOR_TESTS ' + header + ';')
+        pypy_decls.append('RPY_EXTERN ' + header + ';')
         functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
 
     for name, (typ, expr) in GLOBALS.iteritems():
@@ -1007,7 +1006,7 @@
     if sys.platform == 'win32':
         get_pythonapi_source = '''
         #include <windows.h>
-        RPY_EXPORTED_FOR_TESTS
+        RPY_EXTERN
         HANDLE pypy_get_pythonapi_handle() {
             MEMORY_BASIC_INFORMATION  mi;
             memset(&mi, 0, sizeof(mi));
diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h
--- a/pypy/module/cpyext/include/modsupport.h
+++ b/pypy/module/cpyext/include/modsupport.h
@@ -78,11 +78,20 @@
 /*
  * This is from pyport.h.  Perhaps it belongs elsewhere.
  */
+#ifdef _WIN32
+/* explicitly export since PyAPI_FUNC is usually dllimport */
+#ifdef __cplusplus
+#define PyMODINIT_FUNC extern "C" __declspec(dllexport) void
+#else
+#define PyMODINIT_FUNC __declspec(dllexport) void
+#endif
+#else
 #ifdef __cplusplus
 #define PyMODINIT_FUNC extern "C" PyAPI_FUNC(void)
 #else
 #define PyMODINIT_FUNC PyAPI_FUNC(void)
 #endif
+#endif /* WIN32 */
 
 PyAPI_DATA(char *) _Py_PackageContext;
 
diff --git a/pypy/module/cpyext/src/pythread.c b/pypy/module/cpyext/src/pythread.c
--- a/pypy/module/cpyext/src/pythread.c
+++ b/pypy/module/cpyext/src/pythread.c
@@ -1,11 +1,18 @@
 #include <Python.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 #include "pythread.h"
 #include "src/thread.h"
 
 long
 PyThread_get_thread_ident(void)
 {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (long)GetCurrentThreadId();
+#else
+    return (long)pthread_self();
+#endif
 }
 
 PyThread_type_lock
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -7,7 +7,7 @@
     # it's necessary to run "import time" at least once before any
     # other cpyext test, otherwise the same statement will fail in
     # test_datetime.py.
-    space = gettestobjspace(usemodules=['rctime'])
+    space = gettestobjspace(usemodules=['time'])
     space.getbuiltinmodule("time")
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -102,7 +102,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
+                                   'itertools', 'time', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -12,7 +12,7 @@
 
 class AppTestFcntl:
     spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios',
-                                   'select', 'rctime'))
+                                   'select', 'time'))
 
     def setup_class(cls):
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -4,7 +4,7 @@
 
 class AppTestImpModule:
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'rctime', 'struct'],
+        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -149,7 +149,7 @@
 
 class AppTestImport:
     spaceconfig = {
-        "usemodules": ['_md5', 'rctime'],
+        "usemodules": ['_md5', 'time'],
     }
 
     def setup_class(cls):
@@ -1044,7 +1044,7 @@
 
 class AppTestImportHooks(object):
     spaceconfig = {
-        "usemodules": ['struct', 'itertools', 'rctime'],
+        "usemodules": ['struct', 'itertools', 'time'],
     }
 
     def setup_class(cls):
@@ -1304,7 +1304,7 @@
 
 
 class AppTestMultithreadedImp(object):
-    spaceconfig = dict(usemodules=['thread', 'rctime'])
+    spaceconfig = dict(usemodules=['thread', 'time'])
 
     def setup_class(cls):
         #if not conftest.option.runappdirect:
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -144,7 +144,6 @@
     atom_int(tc, int)           puts code and int
     atom_int64(tc, int64)       puts code and int64
     atom_str(tc, str)           puts code, len and string
-    atom_strlist(tc, strlist)   puts code, len and list of strings
 
     building blocks for compound types:
 
@@ -198,15 +197,6 @@
         self.atom_int(typecode, len(x))
         self.put(x)
 
-    def atom_strlist(self, typecode, tc2, x):
-        self.atom_int(typecode, len(x))
-        atom_str = self.atom_str
-        for item in x:
-            # type(str) seems to be forbidden
-            #if type(item) is not str:
-            #    self.raise_exc('object with wrong type in strlist')
-            atom_str(tc2, item)
-
     def start(self, typecode):
         # type(char) not supported
         self.put(typecode)
@@ -379,16 +369,6 @@
         self.start(typecode)
         return self.get_lng()
 
-    def atom_strlist(self, typecode, tc2):
-        self.start(typecode)
-        lng = self.get_lng()
-        res = [None] * lng
-        idx = 0
-        while idx < lng:
-            res[idx] = self.atom_str(tc2)
-            idx += 1
-        return res
-
     def start(self, typecode):
         tc = self.get1()
         if tc != typecode:
@@ -436,7 +416,6 @@
 
     def get_w_obj(self, allow_null=False):
         space = self.space
-        w_ret = space.w_None # something not None
         tc = self.get1()
         w_ret = self._dispatch[ord(tc)](space, self, tc)
         if w_ret is None and not allow_null:
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -7,7 +7,7 @@
 
 class AppTestMath:
     spaceconfig = {
-        "usemodules": ['math', 'struct', 'itertools', 'rctime', 'binascii'],
+        "usemodules": ['math', 'struct', 'itertools', 'time', 'binascii'],
     }

From noreply at buildbot.pypy.org  Fri Dec  5 22:15:53 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  5 Dec 2014 22:15:53 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Restore the previous version to fix
	the test
Message-ID: <20141205211553.7A4231D29F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: more-rposix
Changeset: r74845:34acd72615a6
Date: 2014-12-05 22:15 +0100
http://bitbucket.org/pypy/pypy/changeset/34acd72615a6/

Log:	Restore the previous version to fix the test

diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -10,7 +10,6 @@
 from rpython.flowspace.flowcontext import FlowingError, FlowContext
 from rpython.conftest import option
 from rpython.tool.stdlib_opcode import host_bytecode_spec
-from rpython.rlib import rposix
 
 import os
 import operator
@@ -1263,9 +1262,9 @@
         self.show(x)
         ops = x.startblock.operations
         assert ops[0].opname == 'simple_call'
-        assert ops[0].args[0].value is rposix.unlink
+        assert ops[0].args[0].value is os.unlink
         assert ops[1].opname == 'simple_call'
-        assert ops[1].args[0].value is rposix.unlink
+        assert ops[1].args[0].value is os.unlink
 
     def test_rabspath(self):
         import os.path

From noreply at buildbot.pypy.org  Fri Dec  5 22:37:25 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 22:37:25 +0100 (CET)
Subject: [pypy-commit] creflect default: move some code around and translate
 gc_weakrefs.py into C.
Message-ID: <20141205213725.A528D1D2E26@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r185:771ed4577187
Date: 2014-12-05 22:37 +0100
http://bitbucket.org/cffi/creflect/changeset/771ed4577187/

Log:	move some code around and translate gc_weakrefs.py into C.

diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1708,3 +1708,197 @@
     cd->c_weakreflist = NULL;
     return cd;
 }
+
+static int _my_PyObject_AsBool(PyObject *ob)
+{
+    /* convert and cast a Python object to a boolean.  Accept an integer
+       or a float object, up to a CData 'long double'. */
+    PyObject *io;
+    PyNumberMethods *nb;
+    int res;
+
+#if PY_MAJOR_VERSION < 3
+    if (PyInt_Check(ob)) {
+        return PyInt_AS_LONG(ob) != 0;
+    }
+    else
+#endif
+    if (PyLong_Check(ob)) {
+        return _PyLong_Sign(ob) != 0;
+    }
+    else if (PyFloat_Check(ob)) {
+        return PyFloat_AS_DOUBLE(ob) != 0.0;
+    }
+    else if (CData_Check(ob)) {
+        CDataObject *cd = (CDataObject *)ob;
+        if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
+            /*READ(cd->c_data, cd->c_type->ct_size)*/
+            if (cd->c_type->ct_flags & CT_IS_LONGDOUBLE) {
+                /* 'long double' objects: return the answer directly */
+                return read_raw_longdouble_data(cd->c_data) != 0.0;
+            }
+            else {
+                /* 'float'/'double' objects: return the answer directly */
+                return read_raw_float_data(cd->c_data,
+                                           cd->c_type->ct_size) != 0.0;
+            }
+        }
+    }
+    nb = ob->ob_type->tp_as_number;
+    if (nb == NULL || (nb->nb_float == NULL && nb->nb_int == NULL)) {
+        PyErr_SetString(PyExc_TypeError, "integer/float expected");
+        return -1;
+    }
+    if (nb->nb_float && !CData_Check(ob))
+        io = (*nb->nb_float) (ob);
+    else
+        io = (*nb->nb_int) (ob);
+    if (io == NULL)
+        return -1;
+
+    if (PyIntOrLong_Check(io) || PyFloat_Check(io)) {
+        res = _my_PyObject_AsBool(io);
+    }
+    else {
+        PyErr_SetString(PyExc_TypeError, "integer/float conversion failed");
+        res = -1;
+    }
+    Py_DECREF(io);
+    return res;
+}
+
+static CDataObject *cast_to_integer_or_char(CTypeDescrObject *ct, PyObject *ob)
+{
+    unsigned PY_LONG_LONG value;
+    CDataObject *cd;
+
+    if (CData_Check(ob) &&
+            ((CDataObject *)ob)->c_type->ct_flags & (CT_POINTER | CT_ARRAY)) {
+        value = (Py_intptr_t)((CDataObject *)ob)->c_data;
+    }
+    else if (PyString_Check(ob)) {
+        if (PyString_GET_SIZE(ob) != 1) {
+            PyErr_Format(PyExc_TypeError,
+                         "cannot cast string of length %zd to ctype '%s'",
+                         PyString_GET_SIZE(ob), ct->ct_name);
+            return NULL;
+        }
+        value = (unsigned char)PyString_AS_STRING(ob)[0];
+    }
+    else if (ct->ct_flags & CT_IS_BOOL) {
+        int res = _my_PyObject_AsBool(ob);
+        if (res < 0)
+            return NULL;
+        value = res;
+    }
+    else {
+        value = _my_PyLong_AsUnsignedLongLong(ob, 0);
+        if (value == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())
+            return NULL;
+    }
+    if (ct->ct_flags & CT_IS_BOOL)
+        value = !!value;
+    cd = _new_casted_primitive(ct);
+    if (cd != NULL)
+        write_raw_integer_data(cd->c_data, value, ct->ct_size);
+    return cd;
+}
+
+static PyObject *do_cast(CTypeDescrObject *ct, PyObject *ob)
+{
+    CDataObject *cd;
+
+    if (ct->ct_flags & (CT_POINTER | CT_ARRAY) &&
+        ct->ct_size >= 0) {
+        /* cast to a pointer or to an array.
+           Note that casting to an array is an extension to the C language,
+           which seems to be necessary in order to sanely get a
+           <cdata 'int[3]'> at some address. */
+        unsigned PY_LONG_LONG value;
+
+        if (CData_Check(ob)) {
+            CDataObject *cdsrc = (CDataObject *)ob;
+            if (cdsrc->c_type->ct_flags & (CT_POINTER | CT_ARRAY)) {
+                return new_simple_cdata(cdsrc->c_data, ct);
+            }
+        }
+        value = _my_PyLong_AsUnsignedLongLong(ob, 0);
+        if (value == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())
+            return NULL;
+        return new_simple_cdata((char *)(Py_intptr_t)value, ct);
+    }
+    else if (ct->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED
+                             |CT_PRIMITIVE_CHAR)) {
+        /* cast to an integer type or a char */
+        return (PyObject *)cast_to_integer_or_char(ct, ob);
+    }
+    else if (ct->ct_flags & CT_PRIMITIVE_FLOAT) {
+        /* cast to a float */
+        double value;
+        PyObject *io;
+
+        if (CData_Check(ob)) {
+            CDataObject *cdsrc = (CDataObject *)ob;
+
+            if (!(cdsrc->c_type->ct_flags & CT_PRIMITIVE_ANY))
+                goto cannot_cast;
+            io = convert_to_object(cdsrc->c_data, cdsrc->c_type);
+            if (io == NULL)
+                return NULL;
+        }
+        else {
+            io = ob;
+            Py_INCREF(io);
+        }
+
+        if (PyBytes_Check(io)) {
+            if (PyBytes_GET_SIZE(io) != 1) {
+                Py_DECREF(io);
+                goto cannot_cast;
+            }
+            value = (unsigned char)PyBytes_AS_STRING(io)[0];
+        }
+        else if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
+                 CData_Check(io) &&
+                 (((CDataObject *)io)->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
+            long double lvalue;
+            char *data = ((CDataObject *)io)->c_data;
+            /*READ(data, sizeof(long double)*/
+            lvalue = read_raw_longdouble_data(data);
+            cd = _new_casted_primitive(ct);
+            if (cd != NULL)
+                write_raw_longdouble_data(cd->c_data, lvalue);
+            return (PyObject *)cd;
+        }
+        else {
+            value = PyFloat_AsDouble(io);
+        }
+        Py_DECREF(io);
+        if (value == -1.0 && PyErr_Occurred())
+            return NULL;
+
+        cd = _new_casted_primitive(ct);
+        if (cd != NULL) {
+            if (!(ct->ct_flags & CT_IS_LONGDOUBLE))
+                write_raw_float_data(cd->c_data, value, ct->ct_size);
+            else
+                write_raw_longdouble_data(cd->c_data, (long double)value);
+        }
+        return (PyObject *)cd;
+    }
+    else {
+        PyErr_Format(PyExc_TypeError, "cannot cast to ctype '%s'",
+                     ct->ct_name);
+        return NULL;
+    }
+
+ cannot_cast:
+    if (CData_Check(ob))
+        PyErr_Format(PyExc_TypeError, "cannot cast ctype '%s' to ctype '%s'",
+                     ((CDataObject *)ob)->c_type->ct_name, ct->ct_name);
+    else
+        PyErr_Format(PyExc_TypeError,
+                     "cannot cast %.200s object to ctype '%s'",
+                     Py_TYPE(ob)->tp_name, ct->ct_name);
+    return NULL;
+}
diff --git a/zeffir/cgc.c b/zeffir/cgc.c
new file mode 100644
--- /dev/null
+++ b/zeffir/cgc.c
@@ -0,0 +1,86 @@
+
+/* translated to C from cffi/gc_weakref.py */
+
+
+#define GCWREF_DATA(ffi)       ((ffi)->gc_wrefs[0])
+#define GCWREF_CALLBACK(ffi)   ((ffi)->gc_wrefs[1])
+
+
+static PyObject *zef_name_pop;
+
+static PyObject *gc_wref_remove(ZefFFIObject *ffi, PyObject *arg)
+{
+    PyObject *destructor, *cdata, *x;
+    PyObject *res = PyObject_CallMethodObjArgs(GCWREF_DATA(ffi),
+                                               zef_name_pop, arg, NULL);
+    if (res == NULL)
+        return NULL;
+
+    assert(PyTuple_Check(res));
+    destructor = PyTuple_GET_ITEM(res, 0);
+    cdata = PyTuple_GET_ITEM(res, 1);
+    x = PyObject_CallFunctionObjArgs(destructor, cdata, NULL);
+    Py_DECREF(res);
+    if (x == NULL)
+        return NULL;
+    Py_DECREF(x);
+
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+static PyMethodDef remove_callback = {
+    "remove", (PyCFunction)gc_wref_remove, METH_O
+};
+
+static PyObject *gc_weakrefs_build(ZefFFIObject *ffi, CDataObject *cd,
+                                   PyObject *destructor)
+{
+    PyObject *new_cdata, *ref = NULL, *tup = NULL;
+
+    if (GCWREF_DATA(ffi) == NULL) {
+        /* initialize */
+        PyObject *remove, *data;
+
+        if (zef_name_pop == NULL) {
+            zef_name_pop = PyString_InternFromString("pop");
+            if (zef_name_pop == NULL)
+                return NULL;
+        }
+        remove = PyCFunction_New(&remove_callback, (PyObject *)ffi);
+        if (remove == NULL)
+            return NULL;
+        data = PyDict_New();
+        if (data == NULL) {
+            Py_DECREF(remove);
+            return NULL;
+        }
+        GCWREF_DATA(ffi)     = data;
+        GCWREF_CALLBACK(ffi) = remove;
+    }
+
+    new_cdata = do_cast(cd->c_type, (PyObject *)cd);
+    if (new_cdata == NULL)
+        goto error;
+
+    ref = PyWeakref_NewRef(new_cdata, GCWREF_CALLBACK(ffi));
+    if (ref == NULL)
+        goto error;
+
+    tup = PyTuple_Pack(2, destructor, cd);
+    if (tup == NULL)
+        goto error;
+
+    if (PyDict_SetItem(GCWREF_DATA(ffi), ref, tup) < 0)
+        goto error;
+
+    Py_DECREF(tup);
+    Py_DECREF(ref);
+    return new_cdata;
+
+ error:
+    Py_XDECREF(new_cdata);
+    Py_XDECREF(ref);
+    Py_XDECREF(tup);
+    return NULL;
+}
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -17,18 +17,23 @@
 struct ZefFFIObject_s {
     PyObject_HEAD
     PyObject *types_dict;
+    PyObject *gc_wrefs[2];
 };
 
 static void ffi_dealloc(ZefFFIObject *ffi)
 {
     PyObject_GC_UnTrack(ffi);
     Py_DECREF(ffi->types_dict);
+    Py_XDECREF(ffi->gc_wrefs[0]);
+    Py_XDECREF(ffi->gc_wrefs[1]);
     PyObject_GC_Del(ffi);
 }
 
 static int ffi_traverse(ZefFFIObject *ffi, visitproc visit, void *arg)
 {
     Py_VISIT(ffi->types_dict);
+    Py_VISIT(ffi->gc_wrefs[0]);
+    Py_VISIT(ffi->gc_wrefs[1]);
     return 0;
 }
 
@@ -48,6 +53,8 @@
         return NULL;
     }
     ffi->types_dict = dict;
+    ffi->gc_wrefs[0] = NULL;
+    ffi->gc_wrefs[1] = NULL;
     PyObject_GC_Track(ffi);
 
     return (PyObject *)ffi;
@@ -255,105 +262,9 @@
     return (PyObject *)cd;
 }
 
-static int _my_PyObject_AsBool(PyObject *ob)
-{
-    /* convert and cast a Python object to a boolean.  Accept an integer
-       or a float object, up to a CData 'long double'. */
-    PyObject *io;
-    PyNumberMethods *nb;
-    int res;
-
-#if PY_MAJOR_VERSION < 3
-    if (PyInt_Check(ob)) {
-        return PyInt_AS_LONG(ob) != 0;
-    }
-    else
-#endif
-    if (PyLong_Check(ob)) {
-        return _PyLong_Sign(ob) != 0;
-    }
-    else if (PyFloat_Check(ob)) {
-        return PyFloat_AS_DOUBLE(ob) != 0.0;
-    }
-    else if (CData_Check(ob)) {
-        CDataObject *cd = (CDataObject *)ob;
-        if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
-            /*READ(cd->c_data, cd->c_type->ct_size)*/
-            if (cd->c_type->ct_flags & CT_IS_LONGDOUBLE) {
-                /* 'long double' objects: return the answer directly */
-                return read_raw_longdouble_data(cd->c_data) != 0.0;
-            }
-            else {
-                /* 'float'/'double' objects: return the answer directly */
-                return read_raw_float_data(cd->c_data,
-                                           cd->c_type->ct_size) != 0.0;
-            }
-        }
-    }
-    nb = ob->ob_type->tp_as_number;
-    if (nb == NULL || (nb->nb_float == NULL && nb->nb_int == NULL)) {
-        PyErr_SetString(PyExc_TypeError, "integer/float expected");
-        return -1;
-    }
-    if (nb->nb_float && !CData_Check(ob))
-        io = (*nb->nb_float) (ob);
-    else
-        io = (*nb->nb_int) (ob);
-    if (io == NULL)
-        return -1;
-
-    if (PyIntOrLong_Check(io) || PyFloat_Check(io)) {
-        res = _my_PyObject_AsBool(io);
-    }
-    else {
-        PyErr_SetString(PyExc_TypeError, "integer/float conversion failed");
-        res = -1;
-    }
-    Py_DECREF(io);
-    return res;
-}
-
-static CDataObject *cast_to_integer_or_char(CTypeDescrObject *ct, PyObject *ob)
-{
-    unsigned PY_LONG_LONG value;
-    CDataObject *cd;
-
-    if (CData_Check(ob) &&
-            ((CDataObject *)ob)->c_type->ct_flags & (CT_POINTER | CT_ARRAY)) {
-        value = (Py_intptr_t)((CDataObject *)ob)->c_data;
-    }
-    else if (PyString_Check(ob)) {
-        if (PyString_GET_SIZE(ob) != 1) {
-            PyErr_Format(PyExc_TypeError,
-                         "cannot cast string of length %zd to ctype '%s'",
-                         PyString_GET_SIZE(ob), ct->ct_name);
-            return NULL;
-        }
-        value = (unsigned char)PyString_AS_STRING(ob)[0];
-    }
-    else if (ct->ct_flags & CT_IS_BOOL) {
-        int res = _my_PyObject_AsBool(ob);
-        if (res < 0)
-            return NULL;
-        value = res;
-    }
-    else {
-        value = _my_PyLong_AsUnsignedLongLong(ob, 0);
-        if (value == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())
-            return NULL;
-    }
-    if (ct->ct_flags & CT_IS_BOOL)
-        value = !!value;
-    cd = _new_casted_primitive(ct);
-    if (cd != NULL)
-        write_raw_integer_data(cd->c_data, value, ct->ct_size);
-    return cd;
-}
-
 static PyObject *ffi_cast(ZefFFIObject *self, PyObject *args)
 {
     CTypeDescrObject *ct;
-    CDataObject *cd;
     PyObject *ob, *arg;
     if (!PyArg_ParseTuple(args, "OO:cast", &arg, &ob))
         return NULL;
@@ -362,99 +273,7 @@
     if (ct == NULL)
         return NULL;
 
-    if (ct->ct_flags & (CT_POINTER | CT_ARRAY) &&
-        ct->ct_size >= 0) {
-        /* cast to a pointer or to an array.
-           Note that casting to an array is an extension to the C language,
-           which seems to be necessary in order to sanely get a
-           <cdata 'int[3]'> at some address. */
-        unsigned PY_LONG_LONG value;
-
-        if (CData_Check(ob)) {
-            CDataObject *cdsrc = (CDataObject *)ob;
-            if (cdsrc->c_type->ct_flags & (CT_POINTER | CT_ARRAY)) {
-                return new_simple_cdata(cdsrc->c_data, ct);
-            }
-        }
-        value = _my_PyLong_AsUnsignedLongLong(ob, 0);
-        if (value == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())
-            return NULL;
-        return new_simple_cdata((char *)(Py_intptr_t)value, ct);
-    }
-    else if (ct->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED
-                             |CT_PRIMITIVE_CHAR)) {
-        /* cast to an integer type or a char */
-        return (PyObject *)cast_to_integer_or_char(ct, ob);
-    }
-    else if (ct->ct_flags & CT_PRIMITIVE_FLOAT) {
-        /* cast to a float */
-        double value;
-        PyObject *io;
-
-        if (CData_Check(ob)) {
-            CDataObject *cdsrc = (CDataObject *)ob;
-
-            if (!(cdsrc->c_type->ct_flags & CT_PRIMITIVE_ANY))
-                goto cannot_cast;
-            io = convert_to_object(cdsrc->c_data, cdsrc->c_type);
-            if (io == NULL)
-                return NULL;
-        }
-        else {
-            io = ob;
-            Py_INCREF(io);
-        }
-
-        if (PyBytes_Check(io)) {
-            if (PyBytes_GET_SIZE(io) != 1) {
-                Py_DECREF(io);
-                goto cannot_cast;
-            }
-            value = (unsigned char)PyBytes_AS_STRING(io)[0];
-        }
-        else if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
-                 CData_Check(io) &&
-                 (((CDataObject *)io)->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
-            long double lvalue;
-            char *data = ((CDataObject *)io)->c_data;
-            /*READ(data, sizeof(long double)*/
-            lvalue = read_raw_longdouble_data(data);
-            cd = _new_casted_primitive(ct);
-            if (cd != NULL)
-                write_raw_longdouble_data(cd->c_data, lvalue);
-            return (PyObject *)cd;
-        }
-        else {
-            value = PyFloat_AsDouble(io);
-        }
-        Py_DECREF(io);
-        if (value == -1.0 && PyErr_Occurred())
-            return NULL;
-
-        cd = _new_casted_primitive(ct);
-        if (cd != NULL) {
-            if (!(ct->ct_flags & CT_IS_LONGDOUBLE))
-                write_raw_float_data(cd->c_data, value, ct->ct_size);
-            else
-                write_raw_longdouble_data(cd->c_data, (long double)value);
-        }
-        return (PyObject *)cd;
-    }
-    else {
-        PyErr_Format(PyExc_TypeError, "cannot cast to ctype '%s'",
-                     ct->ct_name);
-        return NULL;
-    }
-
- cannot_cast:
-    if (CData_Check(ob))
-        PyErr_Format(PyExc_TypeError, "cannot cast ctype '%s' to ctype '%s'",
-                     ((CDataObject *)ob)->c_type->ct_name, ct->ct_name);
-    else
-        PyErr_Format(PyExc_TypeError,
-                     "cannot cast %.200s object to ctype '%s'",
-                     Py_TYPE(ob)->tp_name, ct->ct_name);
-    return NULL;
+    return do_cast(ct, ob);
 }
 
 static PyObject *ffi_string(ZefFFIObject *self, PyObject *args)
@@ -728,12 +547,24 @@
     return x;
 }
 
+static PyObject *ffi_gc(ZefFFIObject *self, PyObject *args)
+{
+    CDataObject *cd;
+    PyObject *destructor;
+
+    if (!PyArg_ParseTuple(args, "O!O:gc", &CData_Type, &cd, &destructor))
+        return NULL;
+
+    return gc_weakrefs_build(self, cd, destructor);
+}
+
 
 static PyMethodDef ffi_methods[] = {
     {"addressof",     (PyCFunction)ffi_addressof, METH_VARARGS},
     {"cast",          (PyCFunction)ffi_cast,      METH_VARARGS},
     {"close_library", ffi_close_library,          METH_VARARGS | METH_STATIC},
     {"from_handle",   (PyCFunction)ffi_from_handle,METH_O},
+    {"gc",            (PyCFunction)ffi_gc,        METH_VARARGS},
     {"getctype",      (PyCFunction)ffi_getctype,  METH_VARARGS},
     {"load_library",  (PyCFunction)ffi_load_library,METH_VARARGS|METH_KEYWORDS},
     {"offsetof",      (PyCFunction)ffi_offsetof,  METH_VARARGS},
diff --git a/zeffir/test/test_cgc.py b/zeffir/test/test_cgc.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_cgc.py
@@ -0,0 +1,18 @@
+import gc
+import support
+
+
+def test_simple_gc():
+    ffi = support.new_ffi()
+    p1 = ffi.cast("int *", 0x12345)
+    #
+    seen = []
+    q1 = ffi.gc(p1, seen.append)
+    del p1
+    for i in range(3):
+        gc.collect()
+    assert seen == []
+    del q1
+    for i in range(3):
+        gc.collect()
+    assert seen == [ffi.cast("int *", 0x12345)]
diff --git a/zeffir/zeffir.c b/zeffir/zeffir.c
--- a/zeffir/zeffir.c
+++ b/zeffir/zeffir.c
@@ -18,6 +18,7 @@
 #include "lib_obj.c"
 #include "cfunc.c"
 #include "ffi_obj.c"
+#include "cgc.c"
 #include "builder.c"
 #include "../creflect/creflect_cdecl.c"
 
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -27,3 +27,5 @@
                                             CTypeDescrObject *totype);
 static PyObject *combine_type_name_l(CTypeDescrObject *ct,
                                      size_t extra_text_len);
+static PyObject *gc_weakrefs_build(ZefFFIObject *ffi, CDataObject *cd,
+                                   PyObject *destructor);

From noreply at buildbot.pypy.org  Fri Dec  5 22:42:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  5 Dec 2014 22:42:52 +0100 (CET)
Subject: [pypy-commit] creflect default: simplify
Message-ID: <20141205214252.04CE41D257D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r186:8bcc78765083
Date: 2014-12-05 22:43 +0100
http://bitbucket.org/cffi/creflect/changeset/8bcc78765083/

Log:	simplify

diff --git a/zeffir/cgc.c b/zeffir/cgc.c
--- a/zeffir/cgc.c
+++ b/zeffir/cgc.c
@@ -2,16 +2,12 @@
 /* translated to C from cffi/gc_weakref.py */
 
 
-#define GCWREF_DATA(ffi)       ((ffi)->gc_wrefs[0])
-#define GCWREF_CALLBACK(ffi)   ((ffi)->gc_wrefs[1])
-
-
 static PyObject *zef_name_pop;
 
-static PyObject *gc_wref_remove(ZefFFIObject *ffi, PyObject *arg)
+static PyObject *gc_wref_remove(PyObject *ffi_wref_data, PyObject *arg)
 {
     PyObject *destructor, *cdata, *x;
-    PyObject *res = PyObject_CallMethodObjArgs(GCWREF_DATA(ffi),
+    PyObject *res = PyObject_CallMethodObjArgs(ffi_wref_data,
                                                zef_name_pop, arg, NULL);
     if (res == NULL)
         return NULL;
@@ -30,7 +26,7 @@
 }
 
 static PyMethodDef remove_callback = {
-    "remove", (PyCFunction)gc_wref_remove, METH_O
+    "zef_remove", (PyCFunction)gc_wref_remove, METH_O
 };
 
 static PyObject *gc_weakrefs_build(ZefFFIObject *ffi, CDataObject *cd,
@@ -38,32 +34,29 @@
 {
     PyObject *new_cdata, *ref = NULL, *tup = NULL;
 
-    if (GCWREF_DATA(ffi) == NULL) {
+    if (ffi->gc_wrefs == NULL) {
         /* initialize */
-        PyObject *remove, *data;
+        PyObject *data;
 
         if (zef_name_pop == NULL) {
             zef_name_pop = PyString_InternFromString("pop");
             if (zef_name_pop == NULL)
                 return NULL;
         }
-        remove = PyCFunction_New(&remove_callback, (PyObject *)ffi);
-        if (remove == NULL)
+        data = PyDict_New();
+        if (data == NULL)
             return NULL;
-        data = PyDict_New();
-        if (data == NULL) {
-            Py_DECREF(remove);
+        ffi->gc_wrefs = PyCFunction_New(&remove_callback, data);
+        Py_DECREF(data);
+        if (ffi->gc_wrefs == NULL)
             return NULL;
-        }
-        GCWREF_DATA(ffi)     = data;
-        GCWREF_CALLBACK(ffi) = remove;
     }
 
     new_cdata = do_cast(cd->c_type, (PyObject *)cd);
     if (new_cdata == NULL)
         goto error;
 
-    ref = PyWeakref_NewRef(new_cdata, GCWREF_CALLBACK(ffi));
+    ref = PyWeakref_NewRef(new_cdata, ffi->gc_wrefs);
     if (ref == NULL)
         goto error;
 
@@ -71,7 +64,8 @@
     if (tup == NULL)
         goto error;
 
-    if (PyDict_SetItem(GCWREF_DATA(ffi), ref, tup) < 0)
+    /* the 'self' of the function 'gc_wrefs' is actually the data dict */
+    if (PyDict_SetItem(PyCFunction_GET_SELF(ffi->gc_wrefs), ref, tup) < 0)
         goto error;
 
     Py_DECREF(tup);
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -17,23 +17,21 @@
 struct ZefFFIObject_s {
     PyObject_HEAD
     PyObject *types_dict;
-    PyObject *gc_wrefs[2];
+    PyObject *gc_wrefs;
 };
 
 static void ffi_dealloc(ZefFFIObject *ffi)
 {
     PyObject_GC_UnTrack(ffi);
     Py_DECREF(ffi->types_dict);
-    Py_XDECREF(ffi->gc_wrefs[0]);
-    Py_XDECREF(ffi->gc_wrefs[1]);
+    Py_XDECREF(ffi->gc_wrefs);
     PyObject_GC_Del(ffi);
 }
 
 static int ffi_traverse(ZefFFIObject *ffi, visitproc visit, void *arg)
 {
     Py_VISIT(ffi->types_dict);
-    Py_VISIT(ffi->gc_wrefs[0]);
-    Py_VISIT(ffi->gc_wrefs[1]);
+    Py_VISIT(ffi->gc_wrefs);
     return 0;
 }
 
@@ -53,8 +51,7 @@
         return NULL;
     }
     ffi->types_dict = dict;
-    ffi->gc_wrefs[0] = NULL;
-    ffi->gc_wrefs[1] = NULL;
+    ffi->gc_wrefs = NULL;
     PyObject_GC_Track(ffi);
 
     return (PyObject *)ffi;

From noreply at buildbot.pypy.org  Fri Dec  5 23:01:38 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Fri,  5 Dec 2014 23:01:38 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Progress
Message-ID: <20141205220138.6CAEB1D2CF0@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: extradoc
Changeset: r5478:dd38c1bea82c
Date: 2014-12-06 03:31 +0530
http://bitbucket.org/pypy/extradoc/changeset/dd38c1bea82c/

Log:	Progress

diff --git a/talk/scipyindia2014/talk.rst b/talk/scipyindia2014/talk.rst
--- a/talk/scipyindia2014/talk.rst
+++ b/talk/scipyindia2014/talk.rst
@@ -1,3 +1,5 @@
+.. include:: beamerdefs.txt
+
 =============================
 PyPy and the scientific stack
 =============================
@@ -40,7 +42,16 @@
 Speed
 -----
 
-* XXX : Insert speed.pypy.org screenshot
+.. image:: speed.png
+   :scale: 50%
+   :align: center
+
+How ?
+-----
+
+* Tracing Just-In-Time compiler
+
+* Removes overhead
 
 Demo
 ----
@@ -103,8 +114,23 @@
 NumPyPy performance
 -------------------
 
+* Vectorized operations should be as fast as Numpy
+
+* Using ndarrays as you would use arrays in C or Java should be as fast
+
+* Lazy evaluation ?
+
 PyMetabiosis
 ------------
 
+* Work in progress
+
 JitPy
 -----
+
+* Work in progress
+
+Thank You
+---------
+
+Questions ?

From noreply at buildbot.pypy.org  Fri Dec  5 23:17:35 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Fri,  5 Dec 2014 23:17:35 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Add image
Message-ID: <20141205221735.BA3FA1D2CF0@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: extradoc
Changeset: r5479:9883ef8b82cc
Date: 2014-12-06 03:47 +0530
http://bitbucket.org/pypy/extradoc/changeset/9883ef8b82cc/

Log:	Add image

diff --git a/talk/scipyindia2014/speed.png b/talk/scipyindia2014/speed.png
new file mode 100644
index 0000000000000000000000000000000000000000..63b771ce59358bbcb28efbba84a43f03328b4554
GIT binary patch

[cut]


From noreply at buildbot.pypy.org  Fri Dec  5 23:27:52 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 23:27:52 +0100 (CET)
Subject: [pypy-commit] pypy default: add test_zjit for reduce_cumulative
Message-ID: <20141205222752.733741D2E26@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74846:f907c020762a
Date: 2014-12-05 16:48 -0500
http://bitbucket.org/pypy/pypy/changeset/f907c020762a/

Log:	add test_zjit for reduce_cumulative

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -34,8 +34,8 @@
 
 
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
-                        "unegative", "flat", "tostring","count_nonzero",
-                        "argsort"]
+                        "unegative", "flat", "tostring", "count_nonzero",
+                        "argsort", "cumsum"]
 TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted']
 TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype']
 THREE_ARG_FUNCTIONS = ['where']
@@ -559,6 +559,8 @@
                 w_res = arr.descr_any(interp.space)
             elif self.name == "all":
                 w_res = arr.descr_all(interp.space)
+            elif self.name == "cumsum":
+                w_res = arr.descr_cumsum(interp.space)
             elif self.name == "unegative":
                 neg = ufuncs.get(interp.space).negative
                 w_res = neg.call(interp.space, [arr])
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -183,6 +183,28 @@
             'raw_load': 1,
         })
 
+    def define_cumsum():
+        return """
+        a = |30|
+        b = cumsum(a)
+        b -> 5
+        """
+
+    def test_cumsum(self):
+        result = self.run("cumsum")
+        assert result == 15
+        self.check_trace_count(1)
+        self.check_simple_loop({
+            'float_add': 1,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'int_add': 4,
+            'int_ge': 1,
+            'jump': 1,
+            'raw_load': 1,
+            'raw_store': 1,
+        })
+
     def define_axissum():
         return """
         a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
@@ -218,7 +240,7 @@
             'getfield_gc': 5,
             'getfield_gc_pure': 51,
             'guard_class': 3,
-            'guard_false': 12,
+            'guard_false': 13,
             'guard_nonnull': 11,
             'guard_nonnull_class': 3,
             'guard_not_invalidated': 2,

From noreply at buildbot.pypy.org  Fri Dec  5 23:27:53 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 23:27:53 +0100 (CET)
Subject: [pypy-commit] pypy default: avoid tracking an index in
	reduce_cumulative
Message-ID: <20141205222753.B83E91D2E26@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74847:f28d82b7078c
Date: 2014-12-05 17:03 -0500
http://bitbucket.org/pypy/pypy/changeset/f28d82b7078c/

Log:	avoid tracking an index in reduce_cumulative

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -166,6 +166,7 @@
 def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity):
     obj_iter, obj_state = obj.create_iter()
     out_iter, out_state = out.create_iter()
+    out_iter.track_index = False
     if identity is None:
         cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
         out_iter.setitem(out_state, cur_value)
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -198,7 +198,7 @@
             'float_add': 1,
             'guard_false': 1,
             'guard_not_invalidated': 1,
-            'int_add': 4,
+            'int_add': 3,
             'int_ge': 1,
             'jump': 1,
             'raw_load': 1,

From noreply at buildbot.pypy.org  Fri Dec  5 23:27:54 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 23:27:54 +0100 (CET)
Subject: [pypy-commit] pypy default: add test_zjit for logical_xor reduce
Message-ID: <20141205222754.D64691D2E26@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74848:dba4524de78e
Date: 2014-12-05 16:17 -0500
http://bitbucket.org/pypy/pypy/changeset/dba4524de78e/

Log:	add test_zjit for logical_xor reduce

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -33,7 +33,7 @@
     pass
 
 
-SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
+SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "xor",
                         "unegative", "flat", "tostring", "count_nonzero",
                         "argsort", "cumsum"]
 TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted']
@@ -561,6 +561,9 @@
                 w_res = arr.descr_all(interp.space)
             elif self.name == "cumsum":
                 w_res = arr.descr_cumsum(interp.space)
+            elif self.name == "xor":
+                xor = ufuncs.get(interp.space).logical_xor
+                w_res = xor.reduce(interp.space, arr, None)
             elif self.name == "unegative":
                 neg = ufuncs.get(interp.space).negative
                 w_res = neg.call(interp.space, [arr])
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -400,6 +400,7 @@
     def test_all(self):
         result = self.run("all")
         assert result == 1
+        self.check_trace_count(1)
         self.check_simple_loop({
             'cast_float_to_int': 1,
             'guard_false': 1,
@@ -412,6 +413,36 @@
             'raw_load': 1,
         })
 
+    def define_logical_xor_reduce():
+        return """
+        a = [1,1,1,1,1,1,1,1]
+        xor(a)
+        """
+
+    def test_logical_xor_reduce(self):
+        result = self.run("logical_xor_reduce")
+        assert result == 0
+        self.check_trace_count(2)
+        # XXX fix this
+        self.check_simple_loop({
+            'cast_float_to_int': 1,
+            'getfield_gc': 2,
+            'getfield_gc_pure': 11,
+            'guard_class': 1,
+            'guard_false': 2,
+            'guard_not_invalidated': 1,
+            'guard_true': 4,
+            'int_add': 2,
+            'int_and': 1,
+            'int_ge': 1,
+            'int_is_true': 3,
+            'int_xor': 1,
+            'jump': 1,
+            'new_with_vtable': 1,
+            'raw_load': 1,
+            'setfield_gc': 4,
+        })
+
     def define_already_forced():
         return """
         a = |30|

From noreply at buildbot.pypy.org  Fri Dec  5 23:27:56 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri,  5 Dec 2014 23:27:56 +0100 (CET)
Subject: [pypy-commit] pypy default: eliminate some ops in logical_xor
Message-ID: <20141205222756.038B71D2E26@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74849:794a98c020bf
Date: 2014-12-05 16:39 -0500
http://bitbucket.org/pypy/pypy/changeset/794a98c020bf/

Log:	eliminate some ops in logical_xor

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -429,14 +429,13 @@
             'getfield_gc': 2,
             'getfield_gc_pure': 11,
             'guard_class': 1,
-            'guard_false': 2,
+            'guard_false': 1,
             'guard_not_invalidated': 1,
-            'guard_true': 4,
+            'guard_true': 5,
             'int_add': 2,
             'int_and': 1,
             'int_ge': 1,
-            'int_is_true': 3,
-            'int_xor': 1,
+            'int_is_true': 2,
             'jump': 1,
             'new_with_vtable': 1,
             'raw_load': 1,
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -285,7 +285,9 @@
 
     @raw_binary_op
     def logical_xor(self, v1, v2):
-        return bool(v1) ^ bool(v2)
+        a = bool(v1)
+        b = bool(v2)
+        return (not b and a) or (not a and b)
 
     @raw_unary_op
     def bool(self, v):
@@ -1258,7 +1260,9 @@
 
     @raw_binary_op
     def logical_xor(self, v1, v2):
-        return self._bool(v1) ^ self._bool(v2)
+        a = self._bool(v1)
+        b = self._bool(v2)
+        return (not b and a) or (not a and b)
 
     def min(self, v1, v2):
         if self.le(v1, v2) or self.isnan(v1):
@@ -1733,7 +1737,9 @@
 
     @str_binary_op
     def logical_xor(self, v1, v2):
-        return bool(v1) ^ bool(v2)
+        a = bool(v1)
+        b = bool(v2)
+        return (not b and a) or (not a and b)
 
     def bool(self, v):
         return bool(self.to_str(v))
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -30,9 +30,7 @@
             guard_true(i14, descr=...)
             i15 = getfield_gc_pure(p1, descr=<FieldU pypy.module.micronumpy.boxes.W_BoolBox.inst_value \d+>)
             i16 = int_is_true(i15)
-            i18 = int_xor(i16, 1)
-            i19 = int_is_true(i18)
-            guard_true(i19, descr=...)
+            guard_false(i16, descr=...)
             i20 = getfield_gc(p2, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index \d+>)
             i21 = getfield_gc_pure(p0, descr=<FieldU pypy.module.micronumpy.iterators.ArrayIter.inst_track_index \d+>)
             guard_true(i21, descr=...)

From noreply at buildbot.pypy.org  Sat Dec  6 06:56:55 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sat,  6 Dec 2014 06:56:55 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Progress
Message-ID: <20141206055655.AD09E1C07E8@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: extradoc
Changeset: r5480:61ba4f722878
Date: 2014-12-06 11:25 +0530
http://bitbucket.org/pypy/extradoc/changeset/61ba4f722878/

Log:	Progress

diff --git a/talk/scipyindia2014/talk.rst b/talk/scipyindia2014/talk.rst
--- a/talk/scipyindia2014/talk.rst
+++ b/talk/scipyindia2014/talk.rst
@@ -17,8 +17,6 @@
 
 * Software consultant
 
-* Feel free to interrupt me
-
 PyPy
 ----
 
@@ -125,11 +123,71 @@
 
 * Work in progress
 
+* Allows you to use any CPython module on PyPy
+
+* Embeds CPython into PyPy with CFFI
+
+* Numpy arrays can be shared between PyPy and CPython
+
+PyMetabiosis
+------------
+
+|scriptsize|
+
+.. sourcecode:: python
+
+    from pymetabiosis import import_module
+
+    cpython_virtualenv_path = 
+        "/home/rguillebert/.virtualenvs/venv/bin/activate_this.py"
+
+    builtin = import_module("__builtin__")
+
+    # Activate a virtualenv for the cpython interpreter
+    builtin.execfile(cpython_virtualenv_path,
+        {"__file__" : cpython_virtualenv_path}
+    )
+
+    pylab = import_module("matplotlib.pylab")
+
+    pylab.plot([1, 2, 3, 4])
+    pylab.show()
+
+|end_scriptsize|
+
 JitPy
 -----
 
 * Work in progress
 
+* Embeds PyPy into CPython
+
+* Provides a decorator that allows you to run specific functions on PyPy
+
+* Is used the same way as numba, but different performance caracteristics
+
+JitPy
+-----
+
+|scriptsize|
+
+.. sourcecode:: python
+
+    from jitpy import setup
+    setup('<path-to-pypy-home>')
+    from jitpy.wrapper import jittify
+
+    @jittify([int, float], float)
+    def func(count, no):
+        s = 0
+        for i in range(count):
+           s += no
+        return s
+
+    func(100000, 1.2)
+
+|end_scriptsize|
+
 Thank You
 ---------
 

From noreply at buildbot.pypy.org  Sat Dec  6 09:32:51 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Sat,  6 Dec 2014 09:32:51 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Nearly done ?
Message-ID: <20141206083251.DE7B21D3616@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: extradoc
Changeset: r5481:89e8eadeff5e
Date: 2014-12-06 14:02 +0530
http://bitbucket.org/pypy/extradoc/changeset/89e8eadeff5e/

Log:	Nearly done ?

diff --git a/talk/scipyindia2014/talk.pdf b/talk/scipyindia2014/talk.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..4be849fc047dc21919a478732a2521894f5b01fc
GIT binary patch

[cut]

diff --git a/talk/scipyindia2014/talk.rst b/talk/scipyindia2014/talk.rst
--- a/talk/scipyindia2014/talk.rst
+++ b/talk/scipyindia2014/talk.rst
@@ -107,14 +107,14 @@
 
 * Most of numpy is there
 
-* XXX is missing
+* linalg and the object dtype are the two biggest features that we haven't implemented yet
 
 NumPyPy performance
 -------------------
 
 * Vectorized operations should be as fast as Numpy
 
-* Using ndarrays as you would use arrays in C or Java should be as fast
+* Using ndarrays as you would use arrays in C or Java should be as fast as the vectorized way
 
 * Lazy evaluation ?
 
@@ -123,7 +123,7 @@
 
 * Work in progress
 
-* Allows you to use any CPython module on PyPy
+* Allows you to use any CPython module on PyPy (scipy for example)
 
 * Embeds CPython into PyPy with CFFI
 
@@ -164,7 +164,7 @@
 
 * Provides a decorator that allows you to run specific functions on PyPy
 
-* Is used the same way as numba, but different performance caracteristics
+* Is used the same way as numba, but different performance characteristics
 
 JitPy
 -----
@@ -188,6 +188,17 @@
 
 |end_scriptsize|
 
+Future
+------
+
+* Full numpy support
+
+* Improved C extension compatibility
+
+* I would like to see Cython work with PyPy
+
+* No more Global Interpreter Lock
+
 Thank You
 ---------
 

From noreply at buildbot.pypy.org  Sat Dec  6 11:41:59 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat,  6 Dec 2014 11:41:59 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Better placement for sandbox
	properties
Message-ID: <20141206104159.4CE871C07E8@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: more-rposix
Changeset: r74850:3b5d1aeee6d5
Date: 2014-12-06 11:37 +0100
http://bitbucket.org/pypy/pypy/changeset/3b5d1aeee6d5/

Log:	Better placement for sandbox properties

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -296,12 +296,12 @@
         class ExtRegistry(ExtRegistryEntry):
             _about_ = replaced_function
             def compute_annotation(self):
+                if sandboxed_name:
+                    config = self.bookkeeper.annotator.translator.config
+                    if config.translation.sandbox:
+                        func._sandbox_external_name = sandboxed_name
+                        func._dont_inline_ = True
                 return self.bookkeeper.immutablevalue(func)
-
-        if sandboxed_name:
-            func._sandbox_external_name = sandboxed_name
-            # XXX THIS IS NOT CORRECT. Only do this when config.sandbox.
-            func._dont_inline_ = True
         return func
     return wrap
 

From noreply at buildbot.pypy.org  Sat Dec  6 11:42:00 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat,  6 Dec 2014 11:42:00 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: Don't use os.write in gc helper
	functions.
Message-ID: <20141206104200.D6F8B1C07E8@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: more-rposix
Changeset: r74851:c722fad1b5bf
Date: 2014-12-06 11:41 +0100
http://bitbucket.org/pypy/pypy/changeset/c722fad1b5bf/

Log:	Don't use os.write in gc helper functions. This fixes a test where
	os.write is not used in translated function, and is annotated too
	late.

	extfunc.py uses a "mixed-level delayed object" but I don't know how
	to do this here.

diff --git a/rpython/memory/gctransform/support.py b/rpython/memory/gctransform/support.py
--- a/rpython/memory/gctransform/support.py
+++ b/rpython/memory/gctransform/support.py
@@ -73,15 +73,19 @@
         hop.exception_cannot_occur()
         return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const)
 
+def write(fd, string):
+    from rpython.rlib.rposix import c_write
+    return c_write(fd, string, len(string))
+
 def ll_call_destructor(destrptr, destr_v, typename):
     try:
         destrptr(destr_v)
     except Exception, e:
         try:
-            os.write(2, "a destructor of type ")
-            os.write(2, typename)
-            os.write(2, " raised an exception ")
-            os.write(2, str(e))
-            os.write(2, " ignoring it\n")
+            write(2, "a destructor of type ")
+            write(2, typename)
+            write(2, " raised an exception ")
+            write(2, str(e))
+            write(2, " ignoring it\n")
         except:
             pass

From noreply at buildbot.pypy.org  Sat Dec  6 18:02:42 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  6 Dec 2014 18:02:42 +0100 (CET)
Subject: [pypy-commit] creflect default: tweak the regexp to recognize also
 "#define FOO" with no value specified.
Message-ID: <20141206170242.D3C811D2324@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r187:31b44d824886
Date: 2014-12-06 18:03 +0100
http://bitbucket.org/cffi/creflect/changeset/31b44d824886/

Log:	tweak the regexp to recognize also "#define FOO" with no value
	specified.

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -31,7 +31,7 @@
 r_empty_braces = re.compile(r"{(\s*)}")     # after comments have been removed
 r_typedef_dotdodot = re.compile(r"\btypedef(\s*)[.][.][.]")
 r_define_var = re.compile(
-    r"^[ \t]*#[ \t]*define[ \t]+([A-Za-z_][A-Za-z0-9_]+)[ \t][^\n]+",
+    r"^[ \t]*#[ \t]*define[ \t]+([A-Za-z_][A-Za-z0-9_]+)([ \t].*)?$",
     re.MULTILINE)
 
 def remove_comments(csource):
diff --git a/creflect/test/codegen/macro-001.c b/creflect/test/codegen/macro-001.c
--- a/creflect/test/codegen/macro-001.c
+++ b/creflect/test/codegen/macro-001.c
@@ -1,5 +1,10 @@
+#define FOO
+#define BAR  (40U + 2)
+
+# ____________________________________________________________
+
+#undef FOO
 #define FOO  42
-#define BAR  (40U + 2)
 
 # ____________________________________________________________
 

From noreply at buildbot.pypy.org  Sat Dec  6 19:34:38 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  6 Dec 2014 19:34:38 +0100 (CET)
Subject: [pypy-commit] creflect default: Calling function pointer cdata
 objects. Works only for types that have
Message-ID: <20141206183438.72CEF1C13BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r188:13d44f67603a
Date: 2014-12-06 19:34 +0100
http://bitbucket.org/cffi/creflect/changeset/13d44f67603a/

Log:	Calling function pointer cdata objects. Works only for types that
	have been declared in the .crx, but without any libffi dependency.

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -160,6 +160,7 @@
 
 static _crx_type_t *_zef_function(_crx_builder_t *cb, _crx_type_t *ret,
                                   _crx_qual_type args[], int nargs,
+                                  _crx_trampoline1_fn trampl,
                                   int dotdotdot)
 {
     if (PyErr_Occurred())
@@ -223,12 +224,16 @@
 
     ct->ct_size = -1;
     ct->ct_flags = CT_FUNCTION;
-    /* XXX more... */
 
     put_cached_type(get_types_dict(cb), name_obj, ct);
 
  done:
     Py_DECREF(name_obj);
+
+    if (ct->ct_stuff == NULL && trampl != NULL && !PyErr_Occurred()) {
+        assert(!dotdotdot);   /* should have 'trampl == NULL' in this case */
+        ct->ct_stuff = make_func_support(ret, args, nargs, trampl, 0);
+    }
     return ct;
 }
 
@@ -236,7 +241,7 @@
                                           _crx_qual_type args[], int nargs,
                                           _crx_trampoline1_fn trampl)
 {
-    return _zef_function(cb, ret, args, nargs, 0);
+    return _zef_function(cb, ret, args, nargs, trampl, 0);
 }
 
 static _crx_type_t *zef_get_ellipsis_function_type(_crx_builder_t *cb,
@@ -244,7 +249,7 @@
                                                    _crx_qual_type args[],
                                                    int nargs)
 {
-    return _zef_function(cb, ret, args, nargs, 1);
+    return _zef_function(cb, ret, args, nargs, NULL, 1);
 }
 
 static CTypeDescrObject *fetch_pointer_type(PyObject *types_dict,
@@ -517,7 +522,8 @@
     if (interned_fields == NULL)
         return;
 
-    previous = (CFieldObject **)&ct->ct_extra;
+    assert(ct->ct_fields == NULL);
+    previous = &ct->ct_fields;
 
     for (i = 0; i < nfields; i++) {
         _crx_field_t *f = &fields[i];
diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -638,7 +638,7 @@
     if (PyList_Check(init) || PyTuple_Check(init)) {
         PyObject **items = PySequence_Fast_ITEMS(init);
         Py_ssize_t i, n = PySequence_Fast_GET_SIZE(init);
-        CFieldObject *cf = (CFieldObject *)ct->ct_extra;
+        CFieldObject *cf = ct->ct_fields;
 
         for (i=0; i<n; i++) {
             if (cf == NULL) {
@@ -1611,7 +1611,7 @@
     0,                                          /* tp_as_sequence */
     &CData_as_mapping,                          /* tp_as_mapping */
     (hashfunc)cdata_hash,                       /* tp_hash */
-    0,//(ternaryfunc)cdata_call,                    /* tp_call */
+    (ternaryfunc)cdata_call,                    /* tp_call */
     0,                                          /* tp_str */
     (getattrofunc)cdata_getattro,               /* tp_getattro */
     (setattrofunc)cdata_setattro,               /* tp_setattro */
diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -3,7 +3,7 @@
     PyObject_HEAD
 
     int                 zfs_nargs;
-    _crx_trampoline0_fn zfs_trampl;
+    void               *zfs_trampl;
     PyMethodDef         zfs_md;
     size_t              zfs_size_args;
     CTypeDescrObject   *zfs_ret;
@@ -109,9 +109,9 @@
     return convert_from_object((char *)output_data, ctptr, init);
 }
 
-static PyObject *zfs_call(PyObject *self, PyObject *args)
+static PyObject *common_call(ZefFuncSupportObject *zfs, PyObject *args,
+                             void *func, const char *funcname)
 {
-    ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)self;
     int i, nargs = zfs->zfs_nargs;
     Py_ssize_t actualnargs;
     void *llargs[nargs];
@@ -123,8 +123,8 @@
     if (actualnargs != nargs) {
         if (!PyErr_Occurred())
             PyErr_Format(PyExc_TypeError,
-                         "'%s()' expected %d arguments, but got %zd",
-                         zfs->zfs_md.ml_name, nargs, actualnargs);
+                         "'%s' expected %d arguments, but got %zd",
+                         funcname, nargs, actualnargs);
         return NULL;
     }
 
@@ -165,7 +165,14 @@
         lloffset += ct->ct_size;
     }
 
-    zfs->zfs_trampl(llargs, llbuffer);
+    if (func == NULL) {
+        assert(zfs->zfs_md.ml_name != NULL);
+        ((_crx_trampoline0_fn)zfs->zfs_trampl)(llargs, llbuffer);
+    }
+    else {
+        assert(zfs->zfs_md.ml_name == NULL);
+        ((_crx_trampoline1_fn)zfs->zfs_trampl)(func, llargs, llbuffer);
+    }
 
     if (returns_void) {
         Py_INCREF(Py_None);
@@ -174,21 +181,61 @@
     return convert_to_object(llbuffer, zfs->zfs_ret);
 }
 
-static PyObject *make_builtin_func(PyObject *libname_obj,
-                                   const char *funcname, _crx_type_t *ret,
+static PyObject *zfs_call(PyObject *self, PyObject *args)
+{
+    ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)self;
+    return common_call(zfs, args, NULL, zfs->zfs_md.ml_name);
+}
+
+static PyObject*cdata_call(CDataObject *cd, PyObject *args, PyObject *kwds)
+{
+    CTypeDescrObject *ct = cd->c_type->ct_itemdescr;
+    ZefFuncSupportObject *zfs;
+
+    if (!(cd->c_type->ct_flags & CT_POINTER) || !(ct->ct_flags & CT_FUNCTION)) {
+        PyErr_Format(PyExc_TypeError, "cdata '%s' is not callable",
+                     cd->c_type->ct_name);
+        return NULL;
+    }
+
+    zfs = (ZefFuncSupportObject *)ct->ct_stuff;
+    if (zfs == NULL) {
+        PyErr_SetString(ZefError, "cdata '%s' cannot be called, because no "
+                        "wrapper was generated for functions of this type ");
+        return NULL;
+    }
+
+    if (kwds != NULL && PyDict_Size(kwds) != 0) {
+        PyErr_SetString(PyExc_TypeError,
+            "cdata function pointers cannot be called with keyword arguments");
+        return NULL;
+    }
+    if (cd->c_data == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "cannot call NULL cdata");
+        return NULL;
+    }
+
+    return common_call(zfs, args, cd->c_data, cd->c_type->ct_name);
+}
+
+static PyObject *make_func_support(_crx_type_t *ret,
                                    _crx_qual_type args[], int nargs,
-                                   _crx_trampoline0_fn trampl)
+                                   void *trampl, size_t extra_alloc)
 {
+    /* note that 'trampl' is either a '_crx_trampoline0_fn' or a
+       '_crx_trampoline1_fn', depending on how the ZefFuncSupportObject
+       is going to be used.
+    */
     int i;
-    char *p;
     size_t size = (sizeof(ZefFuncSupportObject)
                    + (nargs - 1) * sizeof(CTypeDescrObject *)
-                   + strlen(funcname) + 1);
+                   + extra_alloc);
     ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)PyObject_Malloc(size);
     if (zfs == NULL)
         return PyErr_NoMemory();
     PyObject_Init((PyObject *)zfs, &ZefFuncSupport_Type);
 
+    memset(&zfs->zfs_md, 0, sizeof(PyMethodDef));
     zfs->zfs_nargs = nargs;
     zfs->zfs_trampl = trampl;
     zfs->zfs_ret = ret;
@@ -207,9 +254,23 @@
         size = ROUND_UP(size) + ct->ct_size;
     }
     zfs->zfs_size_args = size;
+
+    return (PyObject *)zfs;
+}
+
+static PyObject *make_builtin_func(PyObject *libname_obj,
+                                   const char *funcname, _crx_type_t *ret,
+                                   _crx_qual_type args[], int nargs,
+                                   _crx_trampoline0_fn trampl)
+{
+    char *p;
+    ZefFuncSupportObject *zfs;
+    zfs = (ZefFuncSupportObject *)make_func_support(ret, args, nargs, trampl,
+                                                    strlen(funcname) + 1);
+    if (zfs == NULL)
+        return NULL;
+
     p = (char *)(zfs->zfs_args + nargs);
-
-    memset(&zfs->zfs_md, 0, sizeof(PyMethodDef));
     zfs->zfs_md.ml_name = strcpy(p, funcname);
     zfs->zfs_md.ml_meth = &zfs_call;
     zfs->zfs_md.ml_flags = METH_VARARGS;
diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -31,11 +31,10 @@
     CTypeDescrObject *ct_itemdescr;    /* ptrs and arrays: the item type */
     PyObject *ct_stuff;                /* structs: dict of the fields
                                           arrays: ctypedescr of the ptr type
-                                          function: tuple(abi, ctres, ctargs..)
+                                          functions: ZefFuncSupportObject
                                           enum: pair {"name":x},{x:"name"}
                                           ptrs: lazily, ctypedescr of array */
-    void *ct_extra;                    /* structs: first field (not a ref!)
-                                          function types: cif_description */
+    CFieldObject *ct_fields;           /* structs: first field (not a ref!) */
 
     PyObject *ct_weakreflist;    /* weakref support */
 
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -177,7 +177,14 @@
 static PyObject *ffi_typeof(ZefFFIObject *self, PyObject *arg)
 {
     PyObject *x = (PyObject *)_ffi_type(self, arg, ACCEPT_STRING|ACCEPT_CDATA);
-    Py_XINCREF(x);
+    if (x != NULL) {
+        Py_INCREF(x);
+    }
+    else if (PyCFunction_Check(arg)) {
+        PyErr_SetString(PyExc_TypeError, "typeof(lib.func) not supported: the "
+                        "exact type of functions is unknown (declare that "
+                        "function as a function pointer instead)");
+    }
     return x;
 }
 
@@ -446,7 +453,7 @@
         else
             reason = "numeric constants don't have addresses";
 
-        PyErr_Format(PyExc_NotImplementedError,
+        PyErr_Format(PyExc_TypeError,
                      "cannot take the address of '%s' (%s)",
                      fieldname, reason);
         return NULL;
diff --git a/zeffir/test/funcptr.crx b/zeffir/test/funcptr.crx
new file mode 100644
--- /dev/null
+++ b/zeffir/test/funcptr.crx
@@ -0,0 +1,24 @@
+int add_1(int x)
+{
+    return x + 1;
+}
+
+int add_42(int x)
+{
+    return x + 42;
+}
+
+int (*get_fnptr(int target))(int)
+{
+    if (target == 42)
+        return add_42;
+    else
+        return add_1;
+}
+
+
+// CREFLECT: start
+
+int (*get_fnptr(int target))(int);
+
+// CREFLECT: end
diff --git a/zeffir/test/test_funcptr.py b/zeffir/test/test_funcptr.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_funcptr.py
@@ -0,0 +1,31 @@
+import py
+import support
+
+
+def test_cannot_call_nonfunc():
+    ffi = support.new_ffi()
+    x = ffi.new("int *", 42)
+    py.test.raises(TypeError, x)
+
+def test_cannot_call_unprepared_func_type():
+    ffi = support.new_ffi()
+    x = ffi.cast("int(*)(int)", 42)
+    py.test.raises(ffi.error, x)
+
+def test_get_fnptr():
+    ffi, lib = support.compile_and_open('funcptr')
+    fn = lib.get_fnptr(42)
+    assert fn(100) == 142
+    assert ffi.typeof(fn) == ffi.typeof("int(*)(int)")
+    #
+    e = py.test.raises(TypeError, fn, 100, 200)
+    assert str(e.value) == "'int(*)(int)' expected 1 arguments, but got 2"
+    e = py.test.raises(TypeError, fn, foobar=42)
+    assert str(e.value) == ("cdata function pointers cannot be called "
+                            "with keyword arguments")
+
+def test_cannot_call_null():
+    ffi, lib = support.compile_and_open('funcptr')
+    fn = lib.get_fnptr(42)
+    fn = ffi.cast(ffi.typeof(fn), 0)
+    py.test.raises(RuntimeError, fn, 100)
diff --git a/zeffir/test/test_function.py b/zeffir/test/test_function.py
--- a/zeffir/test/test_function.py
+++ b/zeffir/test/test_function.py
@@ -7,6 +7,9 @@
     res = lib.simple_function(42)
     assert type(res) is int
     assert res == 43
+    #
+    e = py.test.raises(TypeError, lib.simple_function, foobar=42)
+    assert str(e.value) == "simple_function() takes no keyword arguments"
 
 def test_function_with_pointer_arg():
     ffi, lib = support.compile_and_open('function')
@@ -29,4 +32,11 @@
 
 def test_addressof_function():
     ffi, lib = support.compile_and_open('function')
-    py.test.raises(NotImplementedError, ffi.addressof, lib, 'simple_function')
+    py.test.raises(TypeError, ffi.addressof, lib, 'simple_function')
+
+def test_typeof_function():
+    ffi, lib = support.compile_and_open('function')
+    e = py.test.raises(TypeError, ffi.typeof, lib.simple_function)
+    assert str(e.value) == ("typeof(lib.func) not supported: the "
+                            "exact type of functions is unknown (declare that "
+                            "function as a function pointer instead)")
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -29,3 +29,4 @@
                                      size_t extra_text_len);
 static PyObject *gc_weakrefs_build(ZefFFIObject *ffi, CDataObject *cd,
                                    PyObject *destructor);
+static PyObject*cdata_call(CDataObject *cd, PyObject *args, PyObject *kwds);

From noreply at buildbot.pypy.org  Sat Dec  6 19:42:31 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  6 Dec 2014 19:42:31 +0100 (CET)
Subject: [pypy-commit] creflect default: fix asserts
Message-ID: <20141206184231.448A11C13BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r189:08d9551d66e1
Date: 2014-12-06 19:42 +0100
http://bitbucket.org/cffi/creflect/changeset/08d9551d66e1/

Log:	fix asserts

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -522,7 +522,6 @@
     if (interned_fields == NULL)
         return;
 
-    assert(ct->ct_fields == NULL);
     previous = &ct->ct_fields;
 
     for (i = 0; i < nfields; i++) {
diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -781,6 +781,7 @@
         return 0;
     }
     if (ct->ct_flags & (CT_STRUCT|CT_UNION)) {
+        assert(ct->ct_size >= 0);
 
         if (CData_Check(init)) {
             if (((CDataObject *)init)->c_type == ct && ct->ct_size >= 0) {

From noreply at buildbot.pypy.org  Sat Dec  6 20:11:06 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  6 Dec 2014 20:11:06 +0100 (CET)
Subject: [pypy-commit] creflect default: port more from _cffi_backend
Message-ID: <20141206191106.9B6B21C07E8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r190:4f6dbe2e4bdc
Date: 2014-12-06 20:11 +0100
http://bitbucket.org/cffi/creflect/changeset/4f6dbe2e4bdc/

Log:	port more from _cffi_backend

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -224,15 +224,21 @@
 
     ct->ct_size = -1;
     ct->ct_flags = CT_FUNCTION;
+    ct->ct_stuff = make_func_support(ret, args, nargs, dotdotdot, 0);
+    if (ct->ct_stuff == NULL) {
+        Py_DECREF(ct);
+        ct = NULL;
+        goto done;
+    }
 
     put_cached_type(get_types_dict(cb), name_obj, ct);
 
  done:
     Py_DECREF(name_obj);
 
-    if (ct->ct_stuff == NULL && trampl != NULL && !PyErr_Occurred()) {
+    if (!PyErr_Occurred() && trampl != NULL) {
         assert(!dotdotdot);   /* should have 'trampl == NULL' in this case */
-        ct->ct_stuff = make_func_support(ret, args, nargs, trampl, 0);
+        provide_trampoline(ct->ct_stuff, trampl);
     }
     return ct;
 }
diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1903,3 +1903,40 @@
                      Py_TYPE(ob)->tp_name, ct->ct_name);
     return NULL;
 }
+
+static PyObject *get_field_name(CTypeDescrObject *ct, CFieldObject *cf)
+{
+    Py_ssize_t i = 0;
+    PyObject *d_key, *d_value;
+    while (PyDict_Next(ct->ct_stuff, &i, &d_key, &d_value)) {
+        if (d_value == (PyObject *)cf)
+            return d_key;
+    }
+    Py_FatalError("_cffi_backend: get_field_name()");
+    return NULL;
+}
+
+static PyObject *cstructtype_getfields(CTypeDescrObject *ct)
+{
+    if (ct->ct_size >= 0) {
+        CFieldObject *cf;
+        PyObject *res = PyList_New(0);
+        if (res == NULL)
+            return NULL;
+        for (cf = ct->ct_fields; cf != NULL; cf = cf->cf_next) {
+            PyObject *o = PyTuple_Pack(2, get_field_name(ct, cf),
+                                       (PyObject *)cf);
+            int err = (o != NULL) ? PyList_Append(res, o) : -1;
+            Py_XDECREF(o);
+            if (err < 0) {
+                Py_DECREF(res);
+                return NULL;
+            }
+        }
+        return res;
+    }
+    else {
+        Py_INCREF(Py_None);
+        return Py_None;
+    }
+}
diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -2,7 +2,12 @@
 typedef struct {
     PyObject_HEAD
 
+    /* note that 'zfs_trampl' is either a '_crx_trampoline0_fn' or a
+       '_crx_trampoline1_fn', depending on how the ZefFuncSupportObject
+       is going to be used.
+    */
     int                 zfs_nargs;
+    int                 zfs_dotdotdot;
     void               *zfs_trampl;
     PyMethodDef         zfs_md;
     size_t              zfs_size_args;
@@ -199,7 +204,7 @@
     }
 
     zfs = (ZefFuncSupportObject *)ct->ct_stuff;
-    if (zfs == NULL) {
+    if (zfs->zfs_trampl == NULL) {
         PyErr_SetString(ZefError, "cdata '%s' cannot be called, because no "
                         "wrapper was generated for functions of this type ");
         return NULL;
@@ -220,12 +225,8 @@
 
 static PyObject *make_func_support(_crx_type_t *ret,
                                    _crx_qual_type args[], int nargs,
-                                   void *trampl, size_t extra_alloc)
+                                   int dotdotdot, size_t extra_alloc)
 {
-    /* note that 'trampl' is either a '_crx_trampoline0_fn' or a
-       '_crx_trampoline1_fn', depending on how the ZefFuncSupportObject
-       is going to be used.
-    */
     int i;
     size_t size = (sizeof(ZefFuncSupportObject)
                    + (nargs - 1) * sizeof(CTypeDescrObject *)
@@ -236,8 +237,9 @@
     PyObject_Init((PyObject *)zfs, &ZefFuncSupport_Type);
 
     memset(&zfs->zfs_md, 0, sizeof(PyMethodDef));
+    zfs->zfs_trampl = NULL;
     zfs->zfs_nargs = nargs;
-    zfs->zfs_trampl = trampl;
+    zfs->zfs_dotdotdot = dotdotdot;
     zfs->zfs_ret = ret;
     Py_INCREF(ret);
 
@@ -258,6 +260,13 @@
     return (PyObject *)zfs;
 }
 
+static void provide_trampoline(PyObject *zfs, void *trampl)
+{
+    if (((ZefFuncSupportObject *)zfs)->zfs_trampl == NULL) {
+        ((ZefFuncSupportObject *)zfs)->zfs_trampl = trampl;
+    }
+}
+
 static PyObject *make_builtin_func(PyObject *libname_obj,
                                    const char *funcname, _crx_type_t *ret,
                                    _crx_qual_type args[], int nargs,
@@ -265,11 +274,13 @@
 {
     char *p;
     ZefFuncSupportObject *zfs;
-    zfs = (ZefFuncSupportObject *)make_func_support(ret, args, nargs, trampl,
+    zfs = (ZefFuncSupportObject *)make_func_support(ret, args, nargs, 0,
                                                     strlen(funcname) + 1);
     if (zfs == NULL)
         return NULL;
 
+    zfs->zfs_trampl = trampl;
+
     p = (char *)(zfs->zfs_args + nargs);
     zfs->zfs_md.ml_name = strcpy(p, funcname);
     zfs->zfs_md.ml_meth = &zfs_call;
@@ -281,3 +292,33 @@
     Py_DECREF(zfs);
     return res;
 }
+
+static PyObject *cfunctype_getargs(CTypeDescrObject *ct)
+{
+    ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)ct->ct_stuff;
+    int i;
+    PyObject *tup = PyTuple_New(zfs->zfs_nargs);
+    if (tup == NULL)
+        return NULL;
+
+    for (i = 0; i < zfs->zfs_nargs; i++) {
+        PyObject *obj = (PyObject *)zfs->zfs_args[i];
+        Py_INCREF(obj);
+        PyTuple_SET_ITEM(tup, i, obj);
+    }
+    return tup;
+}
+
+static PyObject *cfunctype_getresult(CTypeDescrObject *ct)
+{
+    ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)ct->ct_stuff;
+    PyObject *res = (PyObject *)zfs->zfs_ret;
+    Py_INCREF(res);
+    return res;
+}
+
+static PyObject *cfunctype_getellipsis(CTypeDescrObject *ct)
+{
+    ZefFuncSupportObject *zfs = (ZefFuncSupportObject *)ct->ct_stuff;
+    return PyBool_FromLong(zfs->zfs_dotdotdot);
+}
diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -102,6 +102,176 @@
     return 0;
 }
 
+
+static PyObject *nosuchattr(const char *attr)
+{
+    PyErr_SetString(PyExc_AttributeError, attr);
+    return NULL;
+}
+
+static PyObject *ctypeget_kind(CTypeDescrObject *ct, void *context)
+{
+    char *result;
+    if (ct->ct_flags & CT_PRIMITIVE_ANY) {
+        if (ct->ct_flags & CT_IS_ENUM)
+            result = "enum";
+        else
+            result = "primitive";
+    }
+    else if (ct->ct_flags & CT_POINTER) {
+        result = "pointer";
+    }
+    else if (ct->ct_flags & CT_ARRAY) {
+        result = "array";
+    }
+    else if (ct->ct_flags & CT_VOID) {
+        result = "void";
+    }
+    else if (ct->ct_flags & CT_STRUCT) {
+        result = "struct";
+    }
+    else if (ct->ct_flags & CT_UNION) {
+        result = "union";
+    }
+    else if (ct->ct_flags & CT_FUNCTION) {
+        result = "function";
+    }
+    else if (ct->ct_flags & CT_UNKNOWN) {
+        result = "opaque";
+    }
+    else
+        result = "?";
+
+    return PyText_FromString(result);
+}
+
+static PyObject *ctypeget_cname(CTypeDescrObject *ct, void *context)
+{
+    return PyText_FromString(ct->ct_name);
+}
+
+static PyObject *ctypeget_item(CTypeDescrObject *ct, void *context)
+{
+    if (ct->ct_flags & (CT_POINTER | CT_ARRAY)) {
+        Py_INCREF(ct->ct_itemdescr);
+        return (PyObject *)ct->ct_itemdescr;
+    }
+    return nosuchattr("item");
+}
+
+static PyObject *ctypeget_length(CTypeDescrObject *ct, void *context)
+{
+    if (ct->ct_flags & CT_ARRAY) {
+        if (ct->ct_length >= 0) {
+            return PyInt_FromSsize_t(ct->ct_length);
+        }
+        else {
+            Py_INCREF(Py_None);
+            return Py_None;
+        }
+    }
+    return nosuchattr("length");
+}
+
+static PyObject *ctypeget_fields(CTypeDescrObject *ct, void *context)
+{
+    if (ct->ct_flags & (CT_STRUCT | CT_UNION)) {
+        return cstructtype_getfields(ct);
+    }
+    return nosuchattr("fields");
+}
+
+static PyObject *ctypeget_args(CTypeDescrObject *ct, void *context)
+{
+    if (ct->ct_flags & CT_FUNCTION) {
+        return cfunctype_getargs(ct);
+    }
+    return nosuchattr("args");
+}
+
+static PyObject *ctypeget_result(CTypeDescrObject *ct, void *context)
+{
+    if (ct->ct_flags & CT_FUNCTION) {
+        return cfunctype_getresult(ct);
+    }
+    return nosuchattr("result");
+}
+
+static PyObject *ctypeget_ellipsis(CTypeDescrObject *ct, void *context)
+{
+    if (ct->ct_flags & CT_FUNCTION) {
+        return cfunctype_getellipsis(ct);
+    }
+    return nosuchattr("ellipsis");
+}
+
+static PyObject *ctypeget_elements(CTypeDescrObject *ct, void *context)
+{
+    if (ct->ct_flags & CT_IS_ENUM) {
+        PyObject *res = PyTuple_GetItem(ct->ct_stuff, 1);
+        if (res) res = PyDict_Copy(res);
+        return res;
+    }
+    return nosuchattr("elements");
+}
+
+static PyObject *ctypeget_relements(CTypeDescrObject *ct, void *context)
+{
+    if (ct->ct_flags & CT_IS_ENUM) {
+        PyObject *res = PyTuple_GetItem(ct->ct_stuff, 0);
+        if (res) res = PyDict_Copy(res);
+        return res;
+    }
+    return nosuchattr("relements");
+}
+
+static PyGetSetDef ctypedescr_getsets[] = {
+    {"kind", (getter)ctypeget_kind, NULL, "kind"},
+    {"cname", (getter)ctypeget_cname, NULL, "C name"},
+    {"item", (getter)ctypeget_item, NULL, "pointer to, or array of"},
+    {"length", (getter)ctypeget_length, NULL, "array length or None"},
+    {"fields", (getter)ctypeget_fields, NULL, "struct or union fields"},
+    {"args", (getter)ctypeget_args, NULL, "function argument types"},
+    {"result", (getter)ctypeget_result, NULL, "function result type"},
+    {"ellipsis", (getter)ctypeget_ellipsis, NULL, "function has '...'"},
+    {"elements", (getter)ctypeget_elements, NULL, "enum elements"},
+    {"relements", (getter)ctypeget_relements, NULL, "enum elements, reverse"},
+    {NULL}                        /* sentinel */
+};
+
+static PyObject *
+ctypedescr_dir(PyObject *ct, PyObject *noarg)
+{
+    int err;
+    struct PyGetSetDef *gsdef;
+    PyObject *res = PyList_New(0);
+    if (res == NULL)
+        return NULL;
+
+    for (gsdef = ctypedescr_getsets; gsdef->name; gsdef++) {
+        PyObject *x = PyObject_GetAttrString(ct, gsdef->name);
+        if (x == NULL) {
+            PyErr_Clear();
+        }
+        else {
+            Py_DECREF(x);
+            x = PyText_FromString(gsdef->name);
+            err = (x != NULL) ? PyList_Append(res, x) : -1;
+            Py_XDECREF(x);
+            if (err < 0) {
+                Py_DECREF(res);
+                return NULL;
+            }
+        }
+    }
+    return res;
+}
+
+static PyMethodDef ctypedescr_methods[] = {
+    {"__dir__",   ctypedescr_dir,  METH_NOARGS},
+    {NULL,        NULL}           /* sentinel */
+};
+
 static PyTypeObject CTypeDescr_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "zeffir.CType",
@@ -130,7 +300,7 @@
     offsetof(CTypeDescrObject, ct_weakreflist), /* tp_weaklistoffset */
     0,                                          /* tp_iter */
     0,                                          /* tp_iternext */
-    0,//ctypedescr_methods,                         /* tp_methods */
+    ctypedescr_methods,                         /* tp_methods */
     0,                                          /* tp_members */
-    0,//ctypedescr_getsets,                         /* tp_getset */
+    ctypedescr_getsets,                         /* tp_getset */
 };
diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_c.py
@@ -0,0 +1,57 @@
+import py
+import support
+
+import sys
+if sys.version_info < (3,):
+    type_or_class = "type"
+    mandatory_b_prefix = ''
+    mandatory_u_prefix = 'u'
+    bytechr = chr
+    bitem2bchr = lambda x: x
+    class U(object):
+        def __add__(self, other):
+            return eval('u'+repr(other).replace(r'\\u', r'\u')
+                                       .replace(r'\\U', r'\U'))
+    u = U()
+    str2bytes = str
+else:
+    type_or_class = "class"
+    long = int
+    unicode = str
+    unichr = chr
+    mandatory_b_prefix = 'b'
+    mandatory_u_prefix = ''
+    bytechr = lambda n: bytes([n])
+    bitem2bchr = bytechr
+    u = ""
+    str2bytes = lambda s: bytes(s, "ascii")
+
+def size_of_int():
+    ffi = support.new_ffi()
+    return ffi.sizeof("int")
+
+def size_of_long():
+    ffi = support.new_ffi()
+    return ffi.sizeof("long")
+
+def size_of_ptr():
+    ffi = support.new_ffi()
+    return ffi.sizeof("void*")
+
+
+def test_new_primitive_type():
+    ffi = support.new_ffi()
+    py.test.raises(ffi.error, ffi.typeof, "foo")
+    p = ffi.typeof("signed char")
+    assert repr(p) == "<ctype 'signed char'>"
+
+def check_dir(p, expected):
+    got = set(name for name in dir(p) if not name.startswith('_'))
+    assert got == set(expected)
+
+def test_inspect_primitive_type():
+    ffi = support.new_ffi()
+    p = ffi.typeof("signed char")
+    assert p.kind == "primitive"
+    assert p.cname == "signed char"
+    check_dir(p, ['cname', 'kind'])
diff --git a/zeffir/zeffir.h b/zeffir/zeffir.h
--- a/zeffir/zeffir.h
+++ b/zeffir/zeffir.h
@@ -29,4 +29,8 @@
                                      size_t extra_text_len);
 static PyObject *gc_weakrefs_build(ZefFFIObject *ffi, CDataObject *cd,
                                    PyObject *destructor);
-static PyObject*cdata_call(CDataObject *cd, PyObject *args, PyObject *kwds);
+static PyObject *cdata_call(CDataObject *cd, PyObject *args, PyObject *kwds);
+static PyObject *cfunctype_getargs(CTypeDescrObject *ct);
+static PyObject *cfunctype_getresult(CTypeDescrObject *ct);
+static PyObject *cfunctype_getellipsis(CTypeDescrObject *ct);
+static PyObject *cstructtype_getfields(CTypeDescrObject *ct);

From noreply at buildbot.pypy.org  Sat Dec  6 21:03:24 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sat,  6 Dec 2014 21:03:24 +0100 (CET)
Subject: [pypy-commit] pypy default: adjust test_zjit func name for clarity
Message-ID: <20141206200324.7644E1D2324@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74852:b797c990f775
Date: 2014-12-06 15:03 -0500
http://bitbucket.org/pypy/pypy/changeset/b797c990f775/

Log:	adjust test_zjit func name for clarity

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -33,9 +33,9 @@
     pass
 
 
-SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "xor",
+SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
                         "unegative", "flat", "tostring", "count_nonzero",
-                        "argsort", "cumsum"]
+                        "argsort", "cumsum", "logical_xor_reduce"]
 TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted']
 TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype']
 THREE_ARG_FUNCTIONS = ['where']
@@ -561,9 +561,9 @@
                 w_res = arr.descr_all(interp.space)
             elif self.name == "cumsum":
                 w_res = arr.descr_cumsum(interp.space)
-            elif self.name == "xor":
-                xor = ufuncs.get(interp.space).logical_xor
-                w_res = xor.reduce(interp.space, arr, None)
+            elif self.name == "logical_xor_reduce":
+                logical_xor = ufuncs.get(interp.space).logical_xor
+                w_res = logical_xor.reduce(interp.space, arr, None)
             elif self.name == "unegative":
                 neg = ufuncs.get(interp.space).negative
                 w_res = neg.call(interp.space, [arr])
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -416,7 +416,7 @@
     def define_logical_xor_reduce():
         return """
         a = [1,1,1,1,1,1,1,1]
-        xor(a)
+        logical_xor_reduce(a)
         """
 
     def test_logical_xor_reduce(self):

From noreply at buildbot.pypy.org  Sat Dec  6 21:40:21 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  6 Dec 2014 21:40:21 +0100 (CET)
Subject: [pypy-commit] creflect default: enums
Message-ID: <20141206204021.938991C13BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r192:d43293d10be7
Date: 2014-12-06 21:40 +0100
http://bitbucket.org/cffi/creflect/changeset/d43293d10be7/

Log:	enums

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -256,7 +256,7 @@
 
     def get_struct_union_enum_type(self, kind, type, approx_name=None):
         name = type.name or approx_name
-        if not name or name.startswith('$$$'):
+        if kind != 'enum' and (not name or name.startswith('$$$')):
             self.parse_error("not implemented: anonymous 'struct' elsewhere "
                              "than in 'typedef struct { ... } typename;' or "
                              "'typedef struct { ... } *typename;'", type)
@@ -270,7 +270,7 @@
             if kind == 'struct' or kind == 'union':
                 typedecl = model.StructOrUnionDecl(result)
             elif kind == 'enum':
-                typedecl = XXX
+                typedecl = model.EnumDecl(result)
             else:
                 raise AssertionError("kind = %r" % (kind,))
             self.struct_union_enum_decl[key] = typedecl
@@ -280,8 +280,12 @@
         #
         # is there a 'type.decls'?  If yes, then this is the place in the
         # C sources that declare the fields.
-        if type.decls is not None:
-            self.add_fields_declaration(typedecl, type.decls)
+        if kind == 'struct' or kind == 'union':
+            if type.decls is not None:
+                self.add_fields_declaration(typedecl, type.decls)
+        if kind == 'enum':
+            if type.values is not None:
+                self.add_enum_values_declaration(typedecl, type.values)
         if must_add:
             self.declarations.append(typedecl)
         return result
@@ -312,6 +316,13 @@
                 raise NotImplementedError("%s: using both bitfields and '...;'"
                                           % (tp,))
 
+    def add_enum_values_declaration(self, typedecl, values):
+        for enumerator in values.enumerators:
+            self.declarations.append(
+                model.ConstDecl(enumerator.name,
+                                model.QualType(model.int_type)))
+        typedecl.complete = True
+
     def parse_constant(self, constant):
         return int(constant.value)   # xxx
 
diff --git a/creflect/creflect_debug_print.c b/creflect/creflect_debug_print.c
--- a/creflect/creflect_debug_print.c
+++ b/creflect/creflect_debug_print.c
@@ -202,7 +202,8 @@
 static void tst_complete_enum(_crx_builder_t *cb, _crx_type_t *t,
                               _crx_type_t *inttype)
 {
-    abort();
+    assert(memcmp(t->text, "ENUM ", 5) == 0);
+    printf("%s = %s\n", t->text, inttype->text);
 }
 
 static void tst_define_type(_crx_builder_t *cb, const char *name,
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -249,6 +249,8 @@
         block.writeline('cb->define_num_const(cb, "%s", %s, &v);' % (
             varname, t1))
 
+int_type = PrimitiveType('int')
+
 
 class FunctionType(BaseType):
     _attrs_ = ('args', 'result', 'ellipsis')
@@ -730,6 +732,40 @@
         funcblock.writeline('             %s, %d);' % (d1, len(self.fldnames)))
 
 
+class EnumDecl(object):
+
+    def __init__(self, type):
+        self.type = type
+        self.complete = False
+
+    def write_declaration(self, funcblock):
+        if not self.complete or self.type.name is None:
+            return     # opaque
+        tp = "%s %s" % (self.type.kind, self.type.name)
+        block = CodeBlock(funcblock)
+        if not self.type.name.startswith('$'):
+            realtp = tp
+        else:
+            realtp = self.type.name[1:]
+        if not realtp.startswith('$'):
+            expr = '(%s)-1' % realtp
+        else:
+            # xxx obscure case: "typedef enum e *t;"
+            ptrtp = realtp[1:]
+            assert not ptrtp.startswith('$')
+            block.writedecl("%s p1;" % (ptrtp,))
+            block.writedecl("char b[sizeof(*p1)];")
+            block.writeline("memset(b, -1, sizeof(b));")
+            block.writeline("p1 = (%s)b;" % (ptrtp,))
+            expr = '*p1'
+        #
+        t1 = self.type.get_type_var(block)
+        t2 = block.write_crx_type_var('_CRX_INT_TYPE(cb, %s, _crx_sc_int)' %
+                                          (expr,))
+        block.writeline('cb->complete_enum(cb, %s, %s);' % (t1, t2))
+        funcblock.write_subblock(block)
+
+
 class TypeDefDecl(object):
     def __init__(self, name, qualtype):
         self.name = name
diff --git a/creflect/test/codegen/enum-001.c b/creflect/test/codegen/enum-001.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/enum-001.c
@@ -0,0 +1,13 @@
+typedef enum myname_e foo_t;    // unusual, but valid C
+
+# ____________________________________________________________
+
+void testenum_001(_crx_builder_t *cb)
+{
+    _crx_type_t *t1;
+    {
+        t1 = cb->get_enum_type(cb, "myname_e");
+        cb->define_type(cb, "foo_t", t1, 0);
+#expect TYPEDEF foo_t = ENUM myname_e
+    }
+}
diff --git a/creflect/test/codegen/enum-002.c b/creflect/test/codegen/enum-002.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/enum-002.c
@@ -0,0 +1,22 @@
+enum { AA, BB };     /* anonymous */
+
+# ____________________________________________________________
+
+void testenum_002(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2;
+    {
+        _crx_num_const_t v;
+        (void)((AA) << 1);  /* check that 'AA' is an integer */
+        t1 = _CRX_INT_CONST(cb, AA, &v, 1);
+        cb->define_num_const(cb, "AA", t1, &v);
+#expect NUMCONST AA = int 0
+    }
+    {
+        _crx_num_const_t v;
+        (void)((BB) << 1);  /* check that 'BB' is an integer */
+        t2 = _CRX_INT_CONST(cb, BB, &v, 1);
+        cb->define_num_const(cb, "BB", t2, &v);
+#expect NUMCONST BB = int 1
+    }
+}
diff --git a/creflect/test/codegen/enum-002b.c b/creflect/test/codegen/enum-002b.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/enum-002b.c
@@ -0,0 +1,28 @@
+
+enum { AA=(10+10), BB=-4 };
+
+# ____________________________________________________________
+
+#define AA   100
+#define BB   200
+
+# ____________________________________________________________
+
+void testenum_002b(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2;
+    {
+        _crx_num_const_t v;
+        (void)((AA) << 1);  /* check that 'AA' is an integer */
+        t1 = _CRX_INT_CONST(cb, AA, &v, 1);
+        cb->define_num_const(cb, "AA", t1, &v);
+#expect NUMCONST AA = int 100
+    }
+    {
+        _crx_num_const_t v;
+        (void)((BB) << 1);  /* check that 'BB' is an integer */
+        t2 = _CRX_INT_CONST(cb, BB, &v, 1);
+        cb->define_num_const(cb, "BB", t2, &v);
+#expect NUMCONST BB = int 200
+    }
+}
diff --git a/creflect/test/codegen/enum-003.c b/creflect/test/codegen/enum-003.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/enum-003.c
@@ -0,0 +1,23 @@
+enum myname_e { AA };
+/* note that with GCC, "enum myname_e" is equivalent to "unsigned int",
+   but AA itself is a constant of type "int", obscurely */
+
+# ____________________________________________________________
+
+void testenum_003(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2, *t3;
+    {
+        _crx_num_const_t v;
+        (void)((AA) << 1);  /* check that 'AA' is an integer */
+        t1 = _CRX_INT_CONST(cb, AA, &v, 1);
+        cb->define_num_const(cb, "AA", t1, &v);
+#expect NUMCONST AA = int 0
+    }
+    {
+        t2 = cb->get_enum_type(cb, "myname_e");
+        t3 = _CRX_INT_TYPE(cb, (enum myname_e)-1, _crx_sc_int);
+        cb->complete_enum(cb, t2, t3);
+#expect ENUM myname_e = unsigned int
+    }
+}
diff --git a/creflect/test/codegen/enum-003b.c b/creflect/test/codegen/enum-003b.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/enum-003b.c
@@ -0,0 +1,28 @@
+enum myname_e { AA=0x80000000U, BB=3U };
+
+# ____________________________________________________________
+
+void testenum_003b(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2, *t3, *t4;
+    {
+        _crx_num_const_t v;
+        (void)((AA) << 1);  /* check that 'AA' is an integer */
+        t1 = _CRX_INT_CONST(cb, AA, &v, 1);
+        cb->define_num_const(cb, "AA", t1, &v);
+#expect NUMCONST AA = unsigned int 2147483648
+    }
+    {
+        _crx_num_const_t v;
+        (void)((BB) << 1);  /* check that 'BB' is an integer */
+        t2 = _CRX_INT_CONST(cb, BB, &v, 1);
+        cb->define_num_const(cb, "BB", t2, &v);
+#expect NUMCONST BB = int 3
+    }
+    {
+        t3 = cb->get_enum_type(cb, "myname_e");
+        t4 = _CRX_INT_TYPE(cb, (enum myname_e)-1, _crx_sc_int);
+        cb->complete_enum(cb, t3, t4);
+#expect ENUM myname_e = unsigned int
+    }
+}
diff --git a/creflect/test/codegen/enum-003c.c b/creflect/test/codegen/enum-003c.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/enum-003c.c
@@ -0,0 +1,21 @@
+enum myname_e { AA=-3L };
+
+# ____________________________________________________________
+
+void testenum_003c(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2, *t3;
+    {
+        _crx_num_const_t v;
+        (void)((AA) << 1);  /* check that 'AA' is an integer */
+        t1 = _CRX_INT_CONST(cb, AA, &v, 1);
+        cb->define_num_const(cb, "AA", t1, &v);
+#expect NUMCONST AA = int -3
+    }
+    {
+        t2 = cb->get_enum_type(cb, "myname_e");
+        t3 = _CRX_INT_TYPE(cb, (enum myname_e)-1, _crx_sc_int);
+        cb->complete_enum(cb, t2, t3);
+#expect ENUM myname_e = int
+    }
+}
diff --git a/creflect/test/codegen/enum-004.c b/creflect/test/codegen/enum-004.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/enum-004.c
@@ -0,0 +1,34 @@
+typedef enum { AA } *foo_t;
+
+# ____________________________________________________________
+
+void testenum_004(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2, *t3, *t4;
+    {
+        _crx_num_const_t v;
+        (void)((AA) << 1);  /* check that 'AA' is an integer */
+        t1 = _CRX_INT_CONST(cb, AA, &v, 1);
+        cb->define_num_const(cb, "AA", t1, &v);
+#expect NUMCONST AA = int 0
+    }
+    {
+        foo_t p1;
+        char b[sizeof(*p1)];
+        memset(b, -1, sizeof(b));
+        p1 = (foo_t)b;
+        t2 = cb->get_enum_type(cb, "$$foo_t");
+        t3 = _CRX_INT_TYPE(cb, *p1, _crx_sc_int);
+        cb->complete_enum(cb, t2, t3);
+#expect ENUM $$foo_t = unsigned int
+    }
+    {
+        foo_t *p1;
+        char *p2;
+        p1 = (void *)&p2;
+        *p1 = (void *)0;    /* check that 'foo_t' is a pointer type */
+        t4 = cb->get_pointer_type(cb, t2, 0);
+        cb->define_type(cb, "foo_t", t4, 0);
+#expect TYPEDEF foo_t = PTR ENUM $$foo_t
+    }
+}

From noreply at buildbot.pypy.org  Sat Dec  6 21:40:20 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  6 Dec 2014 21:40:20 +0100 (CET)
Subject: [pypy-commit] creflect default: Declare opaque structs
Message-ID: <20141206204020.82D441C13BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r191:1471bfadbc0f
Date: 2014-12-06 20:25 +0100
http://bitbucket.org/cffi/creflect/changeset/1471bfadbc0f/

Log:	Declare opaque structs

diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -659,6 +659,8 @@
         self.fldbitsize = None
 
     def write_declaration(self, funcblock):
+        if self.fldnames is None:
+            return     # opaque
         tp = "%s %s" % (self.type.kind, self.type.name)
         if not self.type.name.startswith('$'):
             realtp = tp
diff --git a/creflect/test/codegen/struct-007.c b/creflect/test/codegen/struct-007.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/struct-007.c
@@ -0,0 +1,13 @@
+typedef struct mystruct_s foo_t;
+
+# ____________________________________________________________
+
+void teststruct_007(_crx_builder_t *cb)
+{
+    _crx_type_t *t1;
+    {
+        t1 = cb->get_struct_type(cb, "mystruct_s");
+        cb->define_type(cb, "foo_t", t1, 0);
+#expect TYPEDEF foo_t = STRUCT mystruct_s
+    }
+}

From noreply at buildbot.pypy.org  Sun Dec  7 05:10:46 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun,  7 Dec 2014 05:10:46 +0100 (CET)
Subject: [pypy-commit] pypy default: Support building on systems with no ssl3
Message-ID: <20141207041046.2ADE71C13F5@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r74853:de9938702e0c
Date: 2014-12-06 20:10 -0800
http://bitbucket.org/pypy/pypy/changeset/de9938702e0c/

Log:	Support building on systems with no ssl3

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -52,7 +52,8 @@
 
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
-constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
+if not OPENSSL_NO_SSL3:
+    constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 
@@ -656,7 +657,7 @@
     # set up context
     if protocol == PY_SSL_VERSION_TLS1:
         method = libssl_TLSv1_method()
-    elif protocol == PY_SSL_VERSION_SSL3:
+    elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
         method = libssl_SSLv3_method()
     elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
         method = libssl_SSLv2_method()
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -68,6 +68,7 @@
     SSLEAY_VERSION = rffi_platform.DefinedConstantString(
         "SSLEAY_VERSION", "SSLeay_version(SSLEAY_VERSION)")
     OPENSSL_NO_SSL2 = rffi_platform.Defined("OPENSSL_NO_SSL2")
+    OPENSSL_NO_SSL3 = rffi_platform.Defined("OPENSSL_NO_SSL3")
     SSL_FILETYPE_PEM = rffi_platform.ConstantInteger("SSL_FILETYPE_PEM")
     SSL_OP_ALL = rffi_platform.ConstantInteger("SSL_OP_ALL")
     SSL_OP_NO_SSLv2 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv2")

From noreply at buildbot.pypy.org  Sun Dec  7 11:18:16 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  7 Dec 2014 11:18:16 +0100 (CET)
Subject: [pypy-commit] creflect default: avoid generating tons of 'a%d[n]'
 local variables if we declare tons
Message-ID: <20141207101816.E00121C33E2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r193:94b9f2a44589
Date: 2014-12-07 11:18 +0100
http://bitbucket.org/cffi/creflect/changeset/94b9f2a44589/

Log:	avoid generating tons of 'a%d[n]' local variables if we declare tons
	of functions with the same argument types

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -9,12 +9,14 @@
             self.crx_type_cache = {}
             self.crx_qualtype_vars = []
             self.crx_field_vars = []
+            self.crx_args_cache = {}
             self.crx_top_level = self
         else:
             self.crx_type_vars = parent.crx_type_vars
             self.crx_type_cache = parent.crx_type_cache
             self.crx_qualtype_vars = parent.crx_qualtype_vars
             self.crx_field_vars = parent.crx_field_vars
+            self.crx_args_cache = parent.crx_args_cache
             self.crx_top_level = parent.crx_top_level
 
     def writedecl(self, line):
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -271,11 +271,16 @@
     def _get_arg_ret_types(self, block):
         inspect = MissingInspector()
         t1, q1_ignored = self.result.inspect_type(block, inspect, 0)
-        a2 = block.write_crx_qualtype_var(len(self.args))
-        for i, arg in enumerate(self.args):
-            t3, q3 = arg.inspect_qualtype(block, inspect)
-            block.writeline('%s[%d].type = %s;' % (a2, i, t3))
-            block.writeline('%s[%d].qualifiers = %s;' % (a2, i, q3))
+        all_args = [arg.inspect_qualtype(block, inspect) for arg in self.args]
+        key = tuple(all_args)
+        try:
+            a2 = block.crx_args_cache[key]
+        except KeyError:
+            a2 = block.write_crx_qualtype_var(len(self.args))
+            block.crx_args_cache[key] = a2
+            for i, (t3, q3) in enumerate(all_args):
+                block.writeline('%s[%d].type = %s;' % (a2, i, t3))
+                block.writeline('%s[%d].qualifiers = %s;' % (a2, i, q3))
         return t1, a2
 
     def _get_c_call_sequence(self, varname):
diff --git a/creflect/test/codegen/func-001b.c b/creflect/test/codegen/func-001b.c
--- a/creflect/test/codegen/func-001b.c
+++ b/creflect/test/codegen/func-001b.c
@@ -39,7 +39,6 @@
     _crx_type_t *t1, *t2, *t3;
     _crx_qual_type a1[1];
     _crx_qual_type a2[2];
-    _crx_qual_type a3[2];
     {
         t1 = cb->get_unsigned_type(cb, sizeof(unsigned int), "int");
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
@@ -58,11 +57,7 @@
 #expect FUNC g: int -> int -> long
     }
     {
-        a3[0].type = t3;
-        a3[0].qualifiers = 0;
-        a3[1].type = t3;
-        a3[1].qualifiers = 0;
-        cb->define_func(cb, "h", t2, a3, 2, &testfunc_001b__c_h, &testfunc_001b__d_h);
+        cb->define_func(cb, "h", t2, a2, 2, &testfunc_001b__c_h, &testfunc_001b__d_h);
 #expect FUNC h: int -> int -> long
     }
 }
diff --git a/creflect/test/codegen/func-005.c b/creflect/test/codegen/func-005.c
--- a/creflect/test/codegen/func-005.c
+++ b/creflect/test/codegen/func-005.c
@@ -28,19 +28,19 @@
         t1 = cb->get_void_type(cb);
         t2 = cb->get_signed_type(cb, sizeof(int), "int");
         t3 = cb->get_incomplete_array_type(cb, t2);
-        a1[0].type = t3;
-        a1[0].qualifiers = _CRX_CONST;
         t4 = cb->get_signed_type(cb, sizeof(long), "long");
         t5 = cb->get_char_type(cb);
-        a2[0].type = t5;
-        a2[0].qualifiers = 0;
         t6 = cb->get_signed_type(cb, sizeof(short), "short");
-        a2[1].type = t6;
+        a1[0].type = t5;
+        a1[0].qualifiers = 0;
+        a1[1].type = t6;
+        a1[1].qualifiers = 0;
+        t7 = cb->get_function_type(cb, t4, a1, 2, &testfunc_005__f4);
+        a2[0].type = t3;
+        a2[0].qualifiers = _CRX_CONST;
+        a2[1].type = t7;
         a2[1].qualifiers = 0;
-        t7 = cb->get_function_type(cb, t4, a2, 2, &testfunc_005__f4);
-        a1[1].type = t7;
-        a1[1].qualifiers = 0;
-        cb->define_func(cb, "f", t1, a1, 2, &testfunc_005__c_f, &testfunc_005__d_f);
+        cb->define_func(cb, "f", t1, a2, 2, &testfunc_005__c_f, &testfunc_005__d_f);
 #expect FUNC f: CONST ARRAY[] int -> FUNC( char -> short -> long ) -> void
     }
 }

From noreply at buildbot.pypy.org  Sun Dec  7 20:56:42 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  7 Dec 2014 20:56:42 +0100 (CET)
Subject: [pypy-commit] pypy default: incliude a note
Message-ID: <20141207195642.4EF261C0907@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74854:f680308d914e
Date: 2014-12-07 21:56 +0200
http://bitbucket.org/pypy/pypy/changeset/f680308d914e/

Log:	incliude a note

diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,6 +6,10 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
+**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
+on linux, linux64 and windows. We will make it the default on all platforms
+by the time of the next release.
+
 The first thing that you need is to compile PyPy yourself with the option
 ``--shared``. We plan to make ``--shared`` the default in the future. Consult
 the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``

From noreply at buildbot.pypy.org  Mon Dec  8 05:34:23 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon,  8 Dec 2014 05:34:23 +0100 (CET)
Subject: [pypy-commit] pypy py3k: fix breakage from default: marshal
 co_names as str again instead of as
Message-ID: <20141208043423.305261D2DFB@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r74855:9314eec351d7
Date: 2014-12-07 20:32 -0800
http://bitbucket.org/pypy/pypy/changeset/9314eec351d7/

Log:	fix breakage from default: marshal co_names as str again instead of
	as unicode. though marshaling as unicode is more correct per
	cpython3 (applies to various other code object fields too), we'll
	come back to that later..

diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -329,7 +329,7 @@
     m.put_int(x.co_flags)
     m.atom_str(TYPE_STRING, x.co_code)
     m.put_tuple_w(TYPE_TUPLE, x.co_consts_w)
-    m.put_tuple_w(TYPE_TUPLE, x.co_names_w)
+    _put_str_list(space, m, [space.str_w(w_name) for w_name in x.co_names_w])
     _put_str_list(space, m, x.co_varnames)
     _put_str_list(space, m, x.co_freevars)
     _put_str_list(space, m, x.co_cellvars)

From noreply at buildbot.pypy.org  Mon Dec  8 05:34:24 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon,  8 Dec 2014 05:34:24 +0100 (CET)
Subject: [pypy-commit] pypy py3k: adapt to py3
Message-ID: <20141208043424.89AAA1D2DFB@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r74856:f91fe709f1b0
Date: 2014-12-07 20:32 -0800
http://bitbucket.org/pypy/pypy/changeset/f91fe709f1b0/

Log:	adapt to py3

diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -110,12 +110,11 @@
 
 def intern_if_common_string(space, w_const):
     # only intern identifier-like strings
-    if not space.is_w(space.type(w_const), space.w_str):
-        return w_const
-    for c in space.str_w(w_const):
-        if not (c.isalnum() or c == '_'):
-            return w_const
-    return space.new_interned_w_str(w_const)
+    from pypy.objspace.std.unicodeobject import _isidentifier
+    if (space.is_w(space.type(w_const), space.w_unicode) and
+        _isidentifier(space.unicode_w(w_const))):
+        return space.new_interned_w_str(w_const)
+    return w_const
 
 
 def new_identifier(space, name):

From noreply at buildbot.pypy.org  Mon Dec  8 05:34:25 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon,  8 Dec 2014 05:34:25 +0100 (CET)
Subject: [pypy-commit] pypy py3k: store unicode for intern'd strings,
 fixes a long standing bug where interning a
Message-ID: <20141208043425.E62FB1D2DFB@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r74857:001db61e74f0
Date: 2014-12-07 20:33 -0800
http://bitbucket.org/pypy/pypy/changeset/001db61e74f0/

Log:	store unicode for intern'd strings, fixes a long standing bug where
	interning a string w/ surrogates would fail

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -362,7 +362,7 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        self.interned_strings = make_weak_value_dictionary(self, str, W_Root)
+        self.interned_strings = make_weak_value_dictionary(self, unicode, W_Root)
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
@@ -765,31 +765,35 @@
         else:
             return self.w_False
 
-    def new_interned_w_str(self, w_s):
-        assert isinstance(w_s, W_Root)   # and is not None
-        s = self.str_w(w_s)
+    def new_interned_w_str(self, w_u):
+        assert isinstance(w_u, W_Root)   # and is not None
+        u = self.unicode_w(w_u)
+        if not we_are_translated():
+            assert type(u) is unicode
+        w_u1 = self.interned_strings.get(u)
+        if w_u1 is None:
+            w_u1 = w_u
+            self.interned_strings.set(u, w_u1)
+        return w_u1
+
+    def new_interned_str(self, s):
+        """Assumes an identifier (utf-8 encoded str)"""
         if not we_are_translated():
             assert type(s) is str
-        w_s1 = self.interned_strings.get(s)
+        u = s.decode('utf-8')
+        w_s1 = self.interned_strings.get(u)
         if w_s1 is None:
-            w_s1 = w_s
-            self.interned_strings.set(s, w_s1)
-        return w_s1
-
-    def new_interned_str(self, s):
-        if not we_are_translated():
-            assert type(s) is str
-        w_s1 = self.interned_strings.get(s)
-        if w_s1 is None:
-            w_s1 = self.wrap(s)
-            self.interned_strings.set(s, w_s1)
+            w_s1 = self.wrap(u)
+            self.interned_strings.set(u, w_s1)
         return w_s1
 
     def is_interned_str(self, s):
+        """Assumes an identifier (utf-8 encoded str)"""
         # interface for marshal_impl
         if not we_are_translated():
             assert type(s) is str
-        return self.interned_strings.get(s) is not None
+        u = s.decode('utf-8')
+        return self.interned_strings.get(u) is not None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -648,6 +648,9 @@
         assert s3 != s2
         s4 = s3.swapcase()
         assert intern(s4) is s2
+        s5 = "\ud800"
+        # previously failed
+        assert intern(s5) == s5
 
 
 class AppTestSysExcInfoDirect:

From noreply at buildbot.pypy.org  Mon Dec  8 09:38:05 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  8 Dec 2014 09:38:05 +0100 (CET)
Subject: [pypy-commit] pypy default: mention AF_XXX in project-ideas
Message-ID: <20141208083805.ABBD41C07E8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74858:1a94e37bcf6a
Date: 2014-12-08 10:37 +0200
http://bitbucket.org/pypy/pypy/changeset/1a94e37bcf6a/

Log:	mention AF_XXX in project-ideas

diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -35,6 +35,13 @@
 PyPy's bytearray type is very inefficient. It would be an interesting
 task to look into possible optimizations on this.
 
+Implement AF_XXX packet types for PyPy
+--------------------------------------
+
+PyPy is missing AF_XXX types of sockets. Implementing it is easy-to-medium
+task. `bug report`_
+
+.. _`bug report`: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets#more
 
 Implement copy-on-write list slicing
 ------------------------------------

From noreply at buildbot.pypy.org  Mon Dec  8 16:38:11 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  8 Dec 2014 16:38:11 +0100 (CET)
Subject: [pypy-commit] benchmarks default: (cfbolz, arigo)
Message-ID: <20141208153811.9D3711D35D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r282:d9d9946e1cda
Date: 2014-12-08 15:37 +0000
http://bitbucket.org/pypy/benchmarks/changeset/d9d9946e1cda/

Log:	(cfbolz, arigo)

	Add the Krakatau benchmark (Robert Grosse on pypy-dev). It's
	actually not all about the warm-up time; after warm-up it is still
	much slower on PyPy than on CPython.

diff too long, truncating to 2000 out of 14612 lines

diff --git a/own/bm_krakatau.py b/own/bm_krakatau.py
new file mode 100644
--- /dev/null
+++ b/own/bm_krakatau.py
@@ -0,0 +1,66 @@
+import sys, os
+import time
+import util, optparse
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'krakatau/Krakatau'))
+print sys.path
+
+import Krakatau.ssa
+from Krakatau.environment import Environment
+from Krakatau.java import javaclass
+from Krakatau.verifier.inference_verifier import verifyBytecode
+
+
+def makeGraph(m):
+    v = verifyBytecode(m.code)
+    s = Krakatau.ssa.ssaFromVerified(m.code, v)
+
+    # print _stats(s)
+    if s.procs:
+        # s.mergeSingleSuccessorBlocks()
+        # s.removeUnusedVariables()
+        s.inlineSubprocs()
+
+    s.condenseBlocks()
+    s.mergeSingleSuccessorBlocks()
+    # print _stats(s)
+    s.removeUnusedVariables()
+    s.constraintPropagation()
+    s.disconnectConstantVariables()
+    s.simplifyJumps()
+    s.mergeSingleSuccessorBlocks()
+    s.removeUnusedVariables()
+    # print _stats(s)
+    return s
+
+def decompileClass():
+    path = ['krakatau/rt.jar']
+    targets = ['javax/swing/plaf/nimbus/ToolBarSouthState']
+    e = Environment()
+    for part in path:
+        e.addToPath(part)
+
+    with e:
+        for i,target in enumerate(targets):
+            for _ in range(100):
+                c = e.getClass(target)
+                source = javaclass.generateAST(c, makeGraph).print_()
+
+
+def main(n):
+    l = []
+    for i in range(n):
+        t0 = time.time()
+        decompileClass()
+        time_elapsed = time.time() - t0
+        l.append(time_elapsed)
+    return l
+
+if __name__ == "__main__":
+    parser = optparse.OptionParser(
+        usage="%prog [options]",
+        description="Test the performance of the krakatau benchmark")
+    util.add_standard_options_to(parser)
+    options, args = parser.parse_args()
+
+    util.run_benchmark(options, options.num_runs, main)
diff --git a/own/cache.txt b/own/cache.txt
new file mode 100644
--- /dev/null
+++ b/own/cache.txt
@@ -0,0 +1,13 @@
+java/lang/Object;SUPER,PUBLIC
+java/lang/Object,javax/swing/plaf/nimbus/State;ABSTRACT,SUPER,PUBLIC
+java/lang/Object,javax/swing/plaf/nimbus/State,javax/swing/plaf/nimbus/ToolBarSouthState;SUPER
+java/lang/Object,java/lang/String;SUPER,FINAL,PUBLIC
+java/lang/Object,java/lang/Throwable;SUPER,PUBLIC
+java/lang/Object,java/lang/Throwable,java/lang/Exception;SUPER,PUBLIC
+java/lang/Object,java/lang/Throwable,java/lang/Exception,java/lang/RuntimeException;SUPER,PUBLIC
+java/lang/Object,java/lang/Throwable,java/lang/Exception,java/lang/RuntimeException,java/lang/IllegalMonitorStateException;SUPER,PUBLIC
+java/lang/Object,java/awt/Component;ABSTRACT,SUPER,PUBLIC
+java/lang/Object,java/awt/Component,java/awt/Container;SUPER,PUBLIC
+java/lang/Object,java/awt/Component,java/awt/Container,javax/swing/JComponent;ABSTRACT,SUPER,PUBLIC
+java/lang/Object,java/awt/Component,java/awt/Container,javax/swing/JComponent,javax/swing/JToolBar;SUPER,PUBLIC
+java/lang/Object,java/lang/Throwable,java/lang/Exception,java/lang/RuntimeException,java/lang/ClassCastException;SUPER,PUBLIC
diff --git a/own/krakatau/Krakatau/.gitattributes b/own/krakatau/Krakatau/.gitattributes
new file mode 100644
--- /dev/null
+++ b/own/krakatau/Krakatau/.gitattributes
@@ -0,0 +1,2 @@
+* text=auto
+*.test binary
\ No newline at end of file
diff --git a/own/krakatau/Krakatau/.gitignore b/own/krakatau/Krakatau/.gitignore
new file mode 100644
--- /dev/null
+++ b/own/krakatau/Krakatau/.gitignore
@@ -0,0 +1,6 @@
+*.pyc
+*.pyo
+test*.py
+Krakatau/plugins/*
+cache.txt
+tests/*.test
diff --git a/own/krakatau/Krakatau/Documentation/assembler.txt b/own/krakatau/Krakatau/Documentation/assembler.txt
new file mode 100644
--- /dev/null
+++ b/own/krakatau/Krakatau/Documentation/assembler.txt
@@ -0,0 +1,88 @@
+Krakatau Assembler Syntax
+
+This guide is intended to help write bytecode assembly files for use with the Krakatau assembler. It assumes that you are already familiar with the JVM classfile format and how to write bytecode. If not, you can find a simple tutorial to writing bytecode at https://greyhat.gatech.edu/wiki/index.php?title=Java_Bytecode_Tutorial. You can also find some examples of assembler files in the examples directory.
+
+Krakatau syntax is largely backwards compatible with the classic Jasmin assembler syntax. In a couple of places, backwards compatibility is broken either by the introduction of new keywords or to fix ambiguities in the Jasmin syntax. However, Krakatau is not necessarily compatible with the extensions introduced by JasminXT.
+
+The basic format for an assembler file consists of a list of classfile entries. Each entry will result in the generation of a seperate classfile, so a single assembly file can contain multiple classes where convienent. These entries are completely independent - mutiple classes never share constant pool entries, fields, methods, or directives, even the version directive. Each one has the format
+
+.bytecode major minor  (optional)
+class directives
+.class classref
+.super classref
+interface declarations
+class directives
+topitems
+.end class
+
+The .end class on the final entry may be ommitted. So the simplest possible assembler file to declare a class named Foo would be
+
+.class Foo
+.super java/lang/Object
+
+To declare three classes A, B, and C in the same file with B and C inheriting from A and different versions, you could do
+
+.class A
+.super java/lang/Object
+.end class
+.class B
+.super A
+.end class
+.class C
+.super A
+
+The classfile version is specified by the .bytecode directive. It is specified by major, minor, a pair of decimal integers. If ommitted, the default is version 49.0. So the following is equivalent to the earlier example
+
+.bytecode 49 0
+.class Foo
+.super java/lang/Object
+
+Other class directives include .runtimevisible, .runtimeinvisible, .signature, .attribute, .source, .inner, .innerlength, and .enclosing. These are used to control the attributes of the class and will be covered later.
+
+Topitems are the actual meat of the class. There are three types: fields, methods, and constant definitions. The last is unique to Krakatau and is closely related to the rest of the syntax. In Krakatau, there are multiple ways to specify a constant pool entry. The most common are via WORD tokens, symbolic references and numerical references. The later constist of square brackets with lowercase alphanumerics and underscores inside.
+
+When you specify .class Foo, the string Foo isn't directly included in the output. The classfile format says that the class field is actually a two byte index into the constant pool of the classfile. This points to a Class_info which points to a Utf8_info which holds the actual name of the class. Therefore, Krakatau implicitly creates constant pool entries and inserts the appropriate references. But this process can be controlled more directly.
+
+Instead of writing
+.class Foo
+.super java/lang/Object
+
+You could explicitly write out all the classfile references as follows
+
+.class [foocls]
+.super [objcls]
+
+.const [foocls] = Class [fooutf]
+.const [fooutf] = Utf8 Foo
+.const [objcls] = Class [objutf]
+.const [objutf] = Utf8 java/lang/Object
+
+There are two types of references. If the contents are a decimal int, then it is a direct numerical reference to a particular slot in the constant pool. You are responsible for making sure that everything is consistent and that the contents of that slot are valid. This option is most useful for specifiying the null entry [0]. For example, to express the Object class itself, one would do
+
+.class java/lang/Object
+.super [0]
+
+If the contents are any other nonempty lowercase alphanumeric + underscores string, it is interperted as a symbolic reference. This is a reference to some slot but you don't care which one. Krakatau will pick an available slot and fill it in automatically. Symbolic references may be ommitted from the generated classfile if unused or merged with identical entries, including automatically generated entries.
+
+With that out of the way, the basic form of a constant definition is
+
+.const ref = entrytype arguments
+
+You can also just define one reference in terms of another. You are responsible for making sure there are no circular references or duplicated definitions. If not, unpredictable results may occur.
+
+Examples include
+
+.const [1] = Class Foo
+.const [1] = Class [fooutf]
+.const [89] = Long 1234567L
+.const [mynat] = NameAndType main ([Ljava/lang/String;)V
+.const [myref] = [myotherref]
+.const [really42] = [42]
+
+
+
+
+
+
+
+
diff --git a/own/krakatau/Krakatau/Krakatau/__init__.py b/own/krakatau/Krakatau/Krakatau/__init__.py
new file mode 100644
--- /dev/null
+++ b/own/krakatau/Krakatau/Krakatau/__init__.py
@@ -0,0 +1,6 @@
+'''Monkeypatch a fix for bugs.python.org/issue9825 in case users are running an old version.'''
+import collections
+try:
+    del collections.OrderedDict.__del__
+except AttributeError:
+    pass
\ No newline at end of file
diff --git a/own/krakatau/Krakatau/Krakatau/assembler/__init__.py b/own/krakatau/Krakatau/Krakatau/assembler/__init__.py
new file mode 100644
diff --git a/own/krakatau/Krakatau/Krakatau/assembler/assembler.py b/own/krakatau/Krakatau/Krakatau/assembler/assembler.py
new file mode 100644
--- /dev/null
+++ b/own/krakatau/Krakatau/Krakatau/assembler/assembler.py
@@ -0,0 +1,580 @@
+import collections
+import struct, operator
+
+from . import instructions, codes
+from .. import constant_pool
+from ..classfile import ClassFile
+from ..method import Method
+from ..field import Field
+
+class AssemblerError(Exception):
+    def __init__(self, message, data=None):
+        super(AssemblerError, self).__init__(message)
+        self.data = data
+
+def error(msg):
+    raise AssemblerError(msg)
+
+class PoolRef(object):
+    def __init__(self, *args, **kwargs):
+        self.index = kwargs.get('index')
+        self.lbl = kwargs.get('lbl')
+        self.args = args
+
+    def toIndex(self, pool, forbidden=(), **kwargs):
+        if self.index is not None:
+            return self.index
+        if self.lbl:
+            self.index = pool.getLabel(self.lbl, forbidden, **kwargs)
+        else:
+            self.args = [(x.toIndex(pool) if isinstance(x, PoolRef) else x) for x in self.args]
+            self.index = pool.getItem(*self.args, **kwargs)
+        return self.index
+
+class PoolInfo(object):
+    def __init__(self):
+        self.pool = constant_pool.ConstPool()
+        self.lbls = {}
+        self.fixed = {} # constant pool entries in a specific slot
+        self.bootstrap = [] #entries for the BootstrapMethods attribute if any
+
+    def getLabel(self, lbl, forbidden=(), **kwargs):
+        if lbl in forbidden:
+            error('Circular constant pool reference: ' + ', '.join(forbidden))
+        forbidden = forbidden + (lbl,)
+        return self.lbls[lbl].toIndex(self, forbidden, **kwargs)
+
+    def getItem(self, type_, *args, **kwargs):
+        if type_ == 'InvokeDynamic':
+            self.bootstrap.append(args[:-1])
+            args = len(self.bootstrap)-1, args[-1]    
+        return self.pool.addItem((type_, tuple(args)), **kwargs)
+
+    def Utf8(self, s):
+        return self.getItem('Utf8', s)
+
+    def assignFixedSlots(self):
+        self.pool.reserved.update(self.fixed)
+        for i,v in self.fixed.items():
+            if v.args and v.args[0] in ('Double','Long'):
+                self.pool.reserved.add(i+1)
+                
+        #TODO - order these in terms of dependencies?
+        for index, value in self.fixed.items():
+            used = value.toIndex(self, index=index)
+            if used != index: #we need to copy an existing item
+                self.pool.copyItem(used, index)
+
+_format_ops = collections.defaultdict(tuple)
+_format_ops[''] = instructions.instrs_noarg
+_format_ops['>B'] = 'iload', 'lload', 'fload', 'dload', 'aload', 'istore', 'lstore', 'fstore', 'dstore', 'astore', 'ret'
+_format_ops['>h'] = 'ifeq', 'ifne', 'iflt', 'ifge', 'ifgt', 'ifle', 'if_icmpeq', 'if_icmpne', 'if_icmplt', 'if_icmpge', 'if_icmpgt', 'if_icmple', 'if_acmpeq', 'if_acmpne', 'goto', 'jsr', 'ifnull', 'ifnonnull'
+_format_ops['>H'] = 'ldc_w', 'ldc2_w', 'getstatic', 'putstatic', 'getfield', 'putfield', 'invokevirtual', 'invokespecial', 'invokestatic', 'new', 'anewarray', 'checkcast', 'instanceof'
+
+_format_ops['>b'] += 'bipush', 
+_format_ops['>Bb'] += 'iinc', 
+_format_ops['>h'] += 'sipush', 
+_format_ops['>HB'] += 'multianewarray',
+_format_ops['>HBB'] += 'invokeinterface',
+_format_ops['>HH'] += 'invokedynamic',
+_format_ops['>B'] += 'ldc', 'newarray'
+_format_ops['>i'] += 'goto_w', 'jsr_w'
+
+op_structs = {}
+for fmt, ops in _format_ops.items():
+    _s = struct.Struct(fmt)
+    for _op in ops:
+        op_structs[_op] = _s
+
+def getPadding(pos):
+    return (3-pos) % 4
+
+def getInstrLen(instr, pos):
+    op = instr[0]
+    if op in op_structs:
+        return 1 + op_structs[op].size
+    elif op == 'wide':
+        return 2 + 2 * len(instr[1][1])
+    else:
+        padding = getPadding(pos)
+        count = len(instr[1][1])
+        if op == 'tableswitch':
+            return 13 + padding + 4*count
+        else:
+            return 9 + padding + 8*count 
+
+def assembleInstruction(instr, labels, pos, pool):
+    def lbl2Off(lbl):
+        if lbl not in labels:
+            del labels[None]
+            error('Undefined label: {}\nDefined labels for current method are: {}'.format(lbl, ', '.join(sorted(labels))))
+        return labels[lbl] - pos
+
+    op = instr[0]
+    first = chr(instructions.allinstructions.index(op))
+
+    instr = [(x.toIndex(pool) if isinstance(x, PoolRef) else x) for x in instr[1:]]
+    if op in instructions.instrs_lbl:
+        instr[0] = lbl2Off(instr[0])
+    if op in op_structs:
+        rest = op_structs[op].pack(*instr)
+        return first+rest
+    elif op == 'wide':
+        subop, args = instr[0]
+        prefix = chr(instructions.allinstructions.index(subop))
+        fmt = '>Hh' if len(args) > 1 else '>H'
+        rest = struct.pack(fmt, *args)
+        return first + prefix + rest
+    else:
+        padding = getPadding(pos)
+        param, jumps, default = instr[0]
+        default = lbl2Off(default)
+
+        if op == 'tableswitch':
+            jumps = map(lbl2Off, jumps)
+            low, high = param, param + len(jumps)-1
+            temp = struct.Struct('>i')
+            part1 = first + '\0'*padding + struct.pack('>iii', default, low, high)
+            return part1 + ''.join(map(temp.pack, jumps))
+        elif op == 'lookupswitch':
+            jumps = {k:lbl2Off(lbl) for k,lbl in jumps}
+            jumps = sorted(jumps.items())
+            temp = struct.Struct('>ii')
+            part1 = first + '\0'*padding + struct.pack('>ii', default, len(jumps))
+            part2 = ''.join(map(temp.pack, *zip(*jumps))) if jumps else ''
+            return part1 + part2
+
+def groupList(pairs):
+    d = collections.defaultdict(list)
+    for k,v in pairs:
+        d[k].append(v)
+    return d
+
+def splitList(pairs):
+    d = groupList(pairs)
+    return d[False], d[True]
+       
+def assembleCodeAttr(statements, pool, version, addLineNumbers, jasmode):
+    directives, lines = splitList(statements)
+    dir_offsets = collections.defaultdict(list)
+
+    offsets = []
+    labels = {}
+    pos = 0
+    #first run through to calculate bytecode offsets
+    #this is greatly complicated due to the need to
+    #handle Jasmine line number directives
+    for t, statement in statements:
+        if t:
+            lbl, instr = statement
+            labels[lbl] = pos
+            if instr is not None:
+                offsets.append(pos)
+                pos += getInstrLen(instr, pos)
+        #some directives require us to keep track of the corresponding bytecode offset
+        elif statement[0] in ('.line','.stackmap'):
+            dir_offsets[statement[0]].append(pos)
+    code_len = pos
+
+    code_bytes = ''
+    for lbl, instr in lines:
+        if instr is not None:
+            code_bytes += assembleInstruction(instr, labels, len(code_bytes), pool)
+    assert(len(code_bytes) == code_len)
+
+    directive_dict = groupList(directives)
+    limits = groupList(directive_dict['.limit'])
+
+    stack = min(limits['stack'] + [65535]) 
+    locals_ = min(limits['locals'] + [65535]) 
+
+    excepts = []
+    for name, start, end, target in directive_dict['.catch']:
+        #Hack for compatibility with Jasmin
+        if jasmode and name.args and (name.args[1].args == ('Utf8','all')):
+            name.index = 0
+        vals = labels[start], labels[end], labels[target], name.toIndex(pool)
+        excepts.append(struct.pack('>HHHH',*vals))
+    
+    attributes = []
+
+    #StackMapTable
+    def pack_vt(vt):
+        s = chr(codes.vt_codes[vt[0]])
+        if vt[0] == 'Object':
+            s += struct.pack('>H', vt[1].toIndex(pool))        
+        elif vt[0] == 'Uninitialized':
+            s += struct.pack('>H', labels[vt[1]])
+        return s
+
+    if directive_dict['.stackmap']:
+        frames = []
+        last_pos = -1
+
+        for pos, info in zip(dir_offsets['.stackmap'], directive_dict['.stackmap']):
+            offset = pos - last_pos - 1
+            last_pos = pos
+            assert(offset >= 0)
+
+            tag = info[0]
+            if tag == 'same':
+                if offset >= 64:
+                    error('Max offset on a same frame is 63.')
+                frames.append(chr(offset))            
+            elif tag == 'same_locals_1_stack_item':
+                if offset >= 64:
+                    error('Max offset on a same_locals_1_stack_item frame is 63.')
+                frames.append(chr(64 + offset) + pack_vt(info[2][0]))            
+            elif tag == 'same_locals_1_stack_item_extended':
+                frames.append(struct.pack('>BH', 247, offset) + pack_vt(info[2][0]))            
+            elif tag == 'chop':
+                if not (1 <= info[1] <= 3):
+                    error('Chop frame can only remove 1-3 locals')
+                frames.append(struct.pack('>BH', 251-info[1], offset))
+            elif tag == 'same_extended':
+                frames.append(struct.pack('>BH', 251, offset))
+            elif tag == 'append':
+                local_vts = map(pack_vt, info[2])
+                if not (1 <= len(local_vts) <= 3):
+                    error('Append frame can only add 1-3 locals')
+                frames.append(struct.pack('>BH', 251+len(local_vts), offset) + ''.join(local_vts))
+            elif tag == 'full':
+                local_vts = map(pack_vt, info[2])
+                stack_vts = map(pack_vt, info[3])
+                frame = struct.pack('>BH', 255, offset)
+                frame += struct.pack('>H', len(local_vts)) + ''.join(local_vts)
+                frame += struct.pack('>H', len(stack_vts)) + ''.join(stack_vts)
+                frames.append(frame)
+
+        sm_body = ''.join(frames)
+        sm_attr = struct.pack('>HIH', pool.Utf8("StackMapTable"), len(sm_body)+2, len(frames)) + sm_body
+        attributes.append(sm_attr)
+
+    #line number attribute
+    if addLineNumbers and not directive_dict['line']:
+        dir_offsets['line'] = directive_dict['line'] = offsets
+    if directive_dict['line']:
+        lntable = [struct.pack('>HH',x,y) for x,y in zip(dir_offsets['line'], directive_dict['line'])]
+        ln_attr = struct.pack('>HIH', pool.Utf8("LineNumberTable"), 2+4*len(lntable), len(lntable)) + ''.join(lntable)        
+        attributes.append(ln_attr)
+
+    if directive_dict['.var']:
+        sfunc = struct.Struct('>HHHHH').pack
+        vartable = []
+        for index, name, desc, start, end in directive_dict['.var']:
+            start, end = labels[start], labels[end]
+            name, desc = name.toIndex(pool), desc.toIndex(pool)
+            vartable.append(sfunc(start, end-start, name, desc, index))
+        var_attr = struct.pack('>HIH', pool.Utf8("LocalVariableTable"), 2+10*len(vartable), len(vartable)) + ''.join(vartable)        
+        attributes.append(var_attr)
+
+    if not code_len:
+        return None
+
+    for attrname, data in directive_dict['.codeattribute']:
+        attr = struct.pack('>HI', attrname.toIndex(pool), len(data)) + data
+        attributes.append(attr)        
+
+
+    #Old versions use shorter fields for stack, locals, and code length
+    header_fmt = '>HHI' if version > (45,2) else '>BBH'
+
+    name_ind = pool.Utf8("Code")
+    attr_len = struct.calcsize(header_fmt) + 4 + len(code_bytes) + 8*len(excepts) + sum(map(len, attributes))
+    
+    assembled_bytes = struct.pack('>HI', name_ind, attr_len)
+    assembled_bytes += struct.pack(header_fmt, stack, locals_, len(code_bytes))
+    assembled_bytes += code_bytes
+    assembled_bytes += struct.pack('>H', len(excepts)) + ''.join(excepts)
+    assembled_bytes += struct.pack('>H', len(attributes)) + ''.join(attributes)
+    return assembled_bytes
+
+def _assembleEVorAnnotationSub(pool, init_args, isAnnot):
+    #call types
+    C_ANNOT, C_ANNOT2, C_EV = range(3)
+    init_callt = C_ANNOT if isAnnot else C_EV
+
+    stack = [(init_callt, init_args)]
+    parts = []
+    add = parts.append
+
+    while stack:
+        callt, args = stack.pop()
+
+        if callt == C_ANNOT:
+            typeref, keylines = args
+            add(struct.pack('>HH', typeref.toIndex(pool), len(keylines)))
+            for pair in reversed(keylines):
+                stack.append((C_ANNOT2, pair))
+
+        elif callt == C_ANNOT2:
+            name, val = args
+            add(struct.pack('>H', name.toIndex(pool)))
+            stack.append((C_EV, val))
+
+        elif callt == C_EV:
+            tag, data = args
+            assert(tag in codes.et_rtags)
+            add(tag)
+
+            if tag in 'BCDFIJSZsc':
+                add(struct.pack('>H', data[0].toIndex(pool)))
+            elif tag == 'e':
+                add(struct.pack('>HH', data[0].toIndex(pool), data[1].toIndex(pool)))
+            elif tag == '@':
+                stack.append((C_ANNOT, data[0]))
+            elif tag == '[':
+                add(struct.pack('>H', len(data[1])))
+                for arrval in reversed(data[1]):
+                    stack.append((C_EV, arrval))
+    return ''.join(parts)
+
+def assembleElementValue(val, pool):
+    return  _assembleEVorAnnotationSub(pool, val, False)
+
+def assembleAnnotation(annotation, pool):
+    return  _assembleEVorAnnotationSub(pool, annotation, True)
+
+def assembleMethod(header, statements, pool, version, addLineNumbers, jasmode):
+    mflags, (name, desc) = header
+    name = name.toIndex(pool)
+    desc = desc.toIndex(pool)
+
+    flagbits = map(Method.flagVals.get, mflags)
+    flagbits = reduce(operator.__or__, flagbits, 0)
+
+    meth_statements, code_statements = splitList(statements)
+
+    method_attributes = []
+    code_attr = assembleCodeAttr(code_statements, pool, version, addLineNumbers, jasmode)
+    if code_attr is not None:
+        method_attributes.append(code_attr)
+
+    directive_dict = groupList(meth_statements)
+    if directive_dict['.throws']:
+        t_inds = [struct.pack('>H', x.toIndex(pool)) for x in directive_dict['.throws']]
+        throw_attr = struct.pack('>HIH', pool.Utf8("Exceptions"), 2+2*len(t_inds), len(t_inds)) + ''.join(t_inds)        
+        method_attributes.append(throw_attr)
+
+    #Runtime annotations
+    for vis in ('Invisible','Visible'):
+        paramd = groupList(directive_dict['.runtime'+vis.lower()])
+
+        if None in paramd:
+            del paramd[None]
+
+        if paramd:
+            parts = []
+            for i in range(max(paramd)):
+                annotations = [assembleAnnotation(a, pool) for a in paramd[i]]
+                part = struct.pack('>H', len(annotations)) + ''.join(annotations)
+                parts.append(part)
+            attrlen = 1+sum(map(len, parts))
+            attr = struct.pack('>HIB', pool.Utf8("Runtime{}ParameterAnnotations".format(vis)), attrlen, len(parts)) + ''.join(parts)
+            method_attributes.append(attr)
+
+    if '.annotationdefault' in directive_dict:
+        val = directive_dict['.annotationdefault'][0]
+        data = assembleElementValue(val, pool)
+        attr = struct.pack('>HI', pool.Utf8("AnnotationDefault"), len(data)) + data        
+        method_attributes.append(attr)
+
+    assembleClassFieldMethodAttributes(method_attributes.append, directive_dict, pool)
+    return struct.pack('>HHHH', flagbits, name, desc, len(method_attributes)) + ''.join(method_attributes)
+
+def getLdcRefs(statements):
+    lines = [x[1][1] for x in statements if x[0] and x[1][0]]
+    instructions = [x[1] for x in lines if x[1] is not None]
+
+    for instr in instructions:
+        op = instr[0]
+        if op == 'ldc':
+            yield instr[1]
+ 
+def addLdcRefs(methods, pool):
+    def getRealRef(ref, forbidden=()):
+        '''Get the root PoolRef associated with a given PoolRef, following labels'''
+        if ref.index is None and ref.lbl:
+            if ref.lbl in forbidden:
+                error('Circular constant pool reference: ' + ', '.join(forbidden))
+            forbidden = forbidden + (ref.lbl,)
+            return getRealRef(pool.lbls[ref.lbl], forbidden) #recursive call
+        return ref
+
+    #We attempt to estimate how many slots are needed after merging identical entries
+    #So we can reserve the correct number of slots without leaving unused gaps
+    #However, in complex cases, such as string/class/mt referring to an explicit
+    #reference, we may overestimate
+    ldc_refs = collections.defaultdict(set)
+
+    for header, statements in methods:
+        for ref in getLdcRefs(statements):
+            ref = getRealRef(ref)
+            if ref.index is not None:
+                continue
+
+            type_ = ref.args[0]
+            if type_ in ('Int','Float'):
+                key = ref.args[1]
+            elif type_ in ('String','Class','MethodType'): 
+                uref = getRealRef(ref.args[1])
+                key = uref.index, uref.args[1:]
+            else: #for MethodHandles, don't even bother trying to estimate merging
+                key = ref.args[1:] 
+            ldc_refs[type_].add(key)    
+
+    #TODO - make this a little cleaner so we don't have to mess with the ConstantPool internals
+    num = sum(map(len, ldc_refs.values()))
+    slots = [pool.pool.getAvailableIndex() for _ in range(num)]
+    pool.pool.reserved.update(slots)
+
+    for type_ in ('Int','Float'):
+        for arg in ldc_refs[type_]:
+            pool.getItem(type_, arg, index=slots.pop())
+    for type_ in ('String','Class','MethodType'):
+        for ind,args in ldc_refs[type_]:
+            arg = ind if ind is not None else pool.Utf8(*args)
+            pool.getItem(type_, arg, index=slots.pop())
+    for type_ in ('MethodHandle',):
+        for code, ref in ldc_refs[type_]:
+            pool.getItem(type_, code, ref.toIndex(pool), index=slots.pop())
+    assert(not slots)
+    assert(not pool.pool.reserved)
+
+def assembleClassFieldMethodAttributes(addcb, directive_dict, pool):
+    for vis in ('Invisible','Visible'):
+        paramd = groupList(directive_dict['.runtime'+vis.lower()])
+        if None in paramd:
+            annotations = [assembleAnnotation(a, pool) for a in paramd[None]]
+            attrlen = 2+sum(map(len, annotations))
+            attr = struct.pack('>HIH', pool.Utf8("Runtime{}Annotations".format(vis)), attrlen, len(annotations)) + ''.join(annotations)
+            addcb(attr)
+
+    for name in directive_dict['.signature']:
+        attr = struct.pack('>HIH', pool.Utf8("Signature"), 2, name.toIndex(pool))
+        addcb(attr)
+
+    #.innerlength directive overrides the normal attribute length calculation
+    hasoverride = len(directive_dict['.innerlength']) > 0
+
+    for name, data in directive_dict['.attribute']:    
+        name_ind = name.toIndex(pool)
+
+        if hasoverride and pool.pool.getArgsCheck('Utf8', name_ind) == 'InnerClasses':
+            attrlen = directive_dict['.innerlength'][0]
+        else:
+            attrlen = len(data)
+
+        attr = struct.pack('>HI', name_ind, attrlen) + data
+        addcb(attr)
+
+def assembleClassAttributes(addcb, directive_dict, pool, addLineNumbers, jasmode, filename):
+
+    sourcefile = directive_dict.get('.source',[None])[0] #PoolRef or None
+    if jasmode and not sourcefile:
+        sourcefile = pool.Utf8(filename)
+    elif addLineNumbers and not sourcefile:
+        sourcefile = pool.Utf8("SourceFile")
+    if sourcefile:
+        attr = struct.pack('>HIH', pool.Utf8("SourceFile"), 2, sourcefile.toIndex(pool))
+        addcb(attr)
+
+    if '.inner' in directive_dict:
+        parts = []
+        for inner, outer, name, flags in directive_dict['.inner']:
+            flagbits = map(ClassFile.flagVals.get, flags)
+            flagbits = reduce(operator.__or__, flagbits, 0)
+            part = struct.pack('>HHHH', inner.toIndex(pool), outer.toIndex(pool), name.toIndex(pool), flagbits)
+            parts.append(part)
+
+        #.innerlength directive overrides the normal attribute length calculation
+        innerlen = 2+8*len(parts) if '.innerlength' not in directive_dict else directive_dict['.innerlength'][0]
+        attr = struct.pack('>HIH', pool.Utf8("InnerClasses"), innerlen, len(parts)) + ''.join(parts)
+        addcb(attr)
+
+    if '.enclosing' in directive_dict:
+        class_, nat = directive_dict['.enclosing'][0]
+        attr = struct.pack('>HIHH', pool.Utf8("EnclosingMethod"), 4, class_.toIndex(pool), nat.toIndex(pool))
+        addcb(attr)
+
+    assembleClassFieldMethodAttributes(addcb, directive_dict, pool)
+
+
+def assemble(tree, addLineNumbers, jasmode, filename):
+    pool = PoolInfo()
+    version, cattrs1, classdec, superdec, interface_decs, cattrs2, topitems = tree
+    if not version: #default to version 49.0 except in Jasmin compatibility mode
+        version = (45,3) if jasmode else (49,0)
+
+    #scan topitems, plus statements in each method to get cpool directives
+    interfaces = []
+    fields = []
+    methods = []
+    attributes = []
+
+    directive_dict = groupList(cattrs1 + cattrs2)
+    top_d = groupList(topitems)
+
+    for slot, value in top_d['const']:
+        if slot.index is not None:
+            pool.fixed[slot.index] = value
+        else:
+            pool.lbls[slot.lbl] = value
+    pool.assignFixedSlots()
+
+    #Now find all cp references used in an ldc instruction
+    #Since they must be <=255, we give them priority in assigning slots
+    #to maximize the chance of a successful assembly
+    addLdcRefs(top_d['method'], pool)
+
+    for flags, name, desc, const, field_directives in top_d['field']:
+        flagbits = map(Field.flagVals.get, flags)
+        flagbits = reduce(operator.__or__, flagbits, 0)
+        name = name.toIndex(pool)
+        desc = desc.toIndex(pool)
+
+        fattrs = []
+        if const is not None:
+            attr = struct.pack('>HIH', pool.Utf8("ConstantValue"), 2, const.toIndex(pool))
+            fattrs.append(attr)
+
+        assembleClassFieldMethodAttributes(fattrs.append, groupList(field_directives), pool)
+
+        field_code = struct.pack('>HHHH', flagbits, name, desc, len(fattrs)) + ''.join(fattrs)
+        fields.append(field_code)
+
+    for header, statements in top_d['method']:
+        methods.append(assembleMethod(header, statements, pool, version, addLineNumbers, jasmode))
+
+    if pool.bootstrap:
+        entries = [struct.pack('>H' + 'H'*len(bsargs), bsargs[0], len(bsargs)-1, *bsargs[1:]) for bsargs in pool.bootstrap]   
+        attrbody = ''.join(entries)
+        attrhead = struct.pack('>HIH', pool.Utf8("BootstrapMethods"), 2+len(attrbody), len(entries))
+        attributes.append(attrhead + attrbody)
+
+    #Explicit class attributes
+    assembleClassAttributes(attributes.append, directive_dict, pool, addLineNumbers, jasmode, filename)
+
+    interfaces = [struct.pack('>H', x.toIndex(pool)) for x in interface_decs]
+    intf, cflags, this = classdec
+    cflags = set(cflags)
+    if intf:
+        cflags.add('INTERFACE')
+    if jasmode:
+        cflags.add('SUPER')
+
+    flagbits = map(ClassFile.flagVals.get, cflags)
+    flagbits = reduce(operator.__or__, flagbits, 0)
+    this = this.toIndex(pool)
+    super_ = superdec.toIndex(pool)
+
+    major, minor = version
+    class_code = '\xCA\xFE\xBA\xBE' + struct.pack('>HH', minor, major)
+    class_code += pool.pool.bytes()
+    class_code += struct.pack('>HHH', flagbits, this, super_)
+    for stuff in (interfaces, fields, methods, attributes):
+        bytes_ = struct.pack('>H', len(stuff)) + ''.join(stuff)
+        class_code += bytes_
+
+    name = pool.pool.getArgs(this)[0]
+    return name, class_code
\ No newline at end of file
diff --git a/own/krakatau/Krakatau/Krakatau/assembler/codes.py b/own/krakatau/Krakatau/Krakatau/assembler/codes.py
new file mode 100644
--- /dev/null
+++ b/own/krakatau/Krakatau/Krakatau/assembler/codes.py
@@ -0,0 +1,11 @@
+_handle_types = 'getField getStatic putField putStatic invokeVirtual invokeStatic invokeSpecial newInvokeSpecial invokeInterface'.split()
+handle_codes = dict(zip(_handle_types, range(1,10)))
+
+newarr_codes = dict(zip('boolean char float double byte short int long'.split(), range(4,12)))
+
+vt_keywords = ['Top','Integer','Float','Double','Long','Null','UninitializedThis','Object','Uninitialized']
+vt_codes = {k:i for i,k in enumerate(vt_keywords)}
+
+
+et_rtags = dict(zip('BCDFIJSZsce@[', 'byte char double int float long short boolean string class enum annotation array'.split()))
+et_tags = {v:k for k,v in et_rtags.items()}
\ No newline at end of file
diff --git a/own/krakatau/Krakatau/Krakatau/assembler/disassembler.py b/own/krakatau/Krakatau/Krakatau/assembler/disassembler.py
new file mode 100644
--- /dev/null
+++ b/own/krakatau/Krakatau/Krakatau/assembler/disassembler.py
@@ -0,0 +1,539 @@
+import collections
+import re
+
+from . import instructions, tokenize, parse, assembler, codes
+from ..binUnpacker import binUnpacker
+from ..classfile import ClassFile
+
+MAX_INLINE_LENGTH = 50
+
+rhandle_codes = {v:k for k,v in codes.handle_codes.items()}
+rnewarr_codes = {v:k for k,v in codes.newarr_codes.items()}
+
+not_word_regex = '(?:{}|{}|{}|;)'.format(tokenize.int_base, tokenize.float_base, tokenize.t_CPINDEX)
+not_word_regex = re.compile(not_word_regex, re.VERBOSE)
+is_word_regex = re.compile(tokenize.t_WORD.__doc__+'$')
+assert(is_word_regex.match("''") is None)
+
+def isWord(s):
+    '''Determine if s can be used as an inline word'''
+    if s in parse.badwords or (not_word_regex.match(s) is not None):
+        return False
+    #eliminate unprintable characters below 32
+    #also, don't allow characters above 127 to keep things simpler
+    return (is_word_regex.match(s) is not None) and min(s) > ' ' and max(s) <= '\x7f'
+
+def rstring(s, allowWord=True):
+    '''Returns a representation of the string. If allowWord is true, it will be unquoted if possible'''
+    if allowWord and isWord(s):
+        return s
+    try:
+        if s.encode('ascii') == s:
+            return repr(str(s))
+    except (UnicodeEncodeError, UnicodeDecodeError):
+        pass
+    return repr(s)
+
+class PoolManager(object):
+    def __init__(self, pool):
+        self.const_pool = pool #keep this around for the float conversion function
+        self.pool = pool.pool
+        self.bootstrap_methods = [] #filled in externally
+        self.used = set() #which cp entries are used non inline and so must be printed
+
+        #For each type, store the function needed to generate the rhs of a constant pool specifier
+        temp1 = lambda ind: rstring(self.cparg1(ind))
+        temp2 = lambda ind: self.utfref(self.cparg1(ind))
+
+        self.cpref_table = {
+            "Utf8": temp1,
+
+            "Class": temp2,
+            "String": temp2,
+            "MethodType": temp2,
+
+            "NameAndType": self.multiref,
+            "Field": self.multiref,
+            "Method": self.multiref,
+            "InterfaceMethod": self.multiref,
+
+            "Int": self.ldc,
+            "Long": self.ldc,
+            "Float": self.ldc,
+            "Double": self.ldc,
+
+            "MethodHandle": self.methodhandle_notref,
+            "InvokeDynamic": self.invokedynamic_notref,
+            }
+
+    def cparg1(self, ind):
+        return self.pool[ind][1][0]
+
+    def inlineutf(self, ind, allowWord=True):
+        '''Returns the word if it's short enough to inline, else None'''
+        arg = self.cparg1(ind)
+        rstr = rstring(arg, allowWord=allowWord)
+        if len(rstr) <= MAX_INLINE_LENGTH:
+            return rstr
+        return None
+
+    def ref(self, ind):
+        self.used.add(ind)
+        return '[_{}]'.format(ind)
+
+    def utfref(self, ind):
+        if ind == 0:
+            return '[0]'
+        inline = self.inlineutf(ind)
+        return inline if inline is not None else self.ref(ind)
+
+    #Also works for Strings and MethodTypes
+    def classref(self, ind):
+        if ind == 0:
+            return '[0]'
+        inline = self.inlineutf(self.cparg1(ind))
+        return inline if inline is not None else self.ref(ind)
+
+    #For Field, Method, IMethod, and NameAndType. Effectively notref
+    def multiref(self, ind):
+        if ind == 0:
+            return '[0]'
+        typen, args = self.pool[ind]
+        if typen == "Utf8":
+            return self.utfref(ind)
+        elif typen == "Class":
+            return self.classref(ind)
+        return ' '.join(map(self.multiref, args))
+
+    #Special case for instruction fieldrefs as a workaround for Jasmin's awful syntax
+    def notjasref(self, ind):
+        typen, args = self.pool[ind]
+        cind = self.cparg1(ind)
+        inline = self.inlineutf(self.cparg1(cind))
+        if inline is None:
+            return self.ref(ind)
+        return inline + ' ' + self.multiref(args[1])
+
+    def ldc(self, ind):
+        typen, args = self.pool[ind]
+        arg = args[0]
+
+        if typen == 'String':
+            inline = self.inlineutf(arg, allowWord=False)
+            return inline if inline is not None else self.ref(ind)
+        elif typen in ('Int','Long','Float','Double'):
+            if typen == "Float" or typen == "Double":
+                arg = self.const_pool.getArgs(ind)[0]
+
+            rstr = repr(arg).rstrip("Ll")
+            if typen == "Float" or typen == "Long":
+                rstr += typen[0]
+            return rstr
+        else:
+            return self.ref(ind)
+
+    def methodhandle_notref(self, ind):
+        typen, args = self.pool[ind]
+        code = rhandle_codes[args[0]]
+        return code + ' ' + self.ref(args[1])
+
+    def invokedynamic_notref(self, ind):
+        typen, args = self.pool[ind]
+        bs_args = self.bootstrap_methods[args[0]]
+
+        parts = [self.methodhandle_notref(bs_args[0])]
+        parts += map(self.ref, bs_args[1:])
+        parts += [':', self.multiref(args[1])]
+        return ' '.join(parts)
+
+    def printConstDefs(self, add):
+        defs = {}
+
+        while self.used:
+            temp, self.used = self.used, set()
+            for ind in temp:
+                if ind in defs:
+                    continue
+                typen = self.pool[ind][0]
+                defs[ind] = self.cpref_table[typen](ind)
+
+        for ind in sorted(defs):
+            add('.const [_{}] = {} {}'.format(ind, self.pool[ind][0], defs[ind]))
+
+def getAttributeTriples(obj): #name_ind, name, data
+    return [(name_ind, name, data1) for (name_ind, data1), (name, data2) in zip(obj.attributes_raw, obj.attributes)]
+
+def getAttributesDict(obj):
+    d = collections.defaultdict(list)
+    for ind, name, attr in getAttributeTriples(obj):
+        d[name].append((ind, attr))
+    return d
+
+fmt_lookup = {k:v.format for k,v in assembler.op_structs.items()}
+def getInstruction(b, getlbl, poolm):
+    pos = b.off
+    op = b.get('B')
+
+    name = instructions.allinstructions[op]
+    if name == 'wide':
+        name2 = instructions.allinstructions[b.get('B')]
+        if name2 == 'iinc':
+            args = list(b.get('>Hh'))
+        else:
+            args = [b.get('>H')]
+
+        parts = [name, name2] + map(str, args)
+        return '\t' + ' '.join(parts)
+    elif name == 'tableswitch' or name == 'lookupswitch':
+        padding = assembler.getPadding(pos)
+        b.getRaw(padding)
+
+        default = getlbl(b.get('>i')+pos)
+
+        if name == 'lookupswitch':
+            num = b.get('>I')
+            entries = ['\t'+name]
+            entries += ['\t\t{} : {}'.format(b.get('>i'), getlbl(b.get('>i')+pos)) for _ in range(num)]
+        else:
+            low, high = b.get('>ii')
+            num = high-low+1
+            entries = ['\t{} {}'.format(name, low)]
+            entries += ['\t\t{}'.format(getlbl(b.get('>i')+pos)) for _ in range(num)]
+        entries += ['\t\tdefault : {}'.format(default)]
+        return '\n'.join(entries)
+    else:
+        args = list(b.get(fmt_lookup[name], forceTuple=True))
+        #remove extra padding 0
+        if name in ('invokeinterface','invokedynamic'):
+            args = args[:-1]
+
+        funcs = {
+                'OP_CLASS': poolm.classref,
+                'OP_CLASS_INT': poolm.classref,
+                'OP_FIELD': poolm.notjasref, #this is a special case due to the jasmin thing
+                'OP_METHOD': poolm.multiref,
+                'OP_METHOD_INT': poolm.multiref,
+                'OP_DYNAMIC': poolm.ref,
+                'OP_LDC1': poolm.ldc,
+                'OP_LDC2': poolm.ldc,
+                'OP_NEWARR': rnewarr_codes.get,
+            }
+
+        token_t = tokenize.wordget[name]
+        if token_t == 'OP_LBL':
+            assert(len(args) == 1)
+            args[0] = getlbl(args[0]+pos)
+        elif token_t in funcs:
+            args[0] = funcs[token_t](args[0])
+
+        parts = [name] + map(str, args)
+        return '\t' + ' '.join(parts)
+
+def disMethodCode(code, add, poolm):
+    if code is None:
+        return
+    add('\t; method code size: {} bytes'.format(code.codelen))
+    add('\t.limit stack {}'.format(code.stack))
+    add('\t.limit locals {}'.format(code.locals))
+
+    lbls = set()
+    def getlbl(x):
+        lbls.add(x)
+        return 'L'+str(x)
+
+    for e in code.except_raw:
+        parts = poolm.classref(e.type_ind), getlbl(e.start), getlbl(e.end), getlbl(e.handler)
+        add('\t.catch {} from {} to {} using {}'.format(*parts))
+
+    code_attributes = getAttributesDict(code)
+    frames = getStackMapTable(code_attributes, poolm, getlbl)
+
+    instrs = []
+    b = binUnpacker(code.bytecode_raw)
+    while b.size():
+        instrs.append((b.off, getInstruction(b, getlbl, poolm)))
+    instrs.append((b.off, None))
+
+    for off, instr in instrs:
+        if off in lbls:
+            add('L{}:'.format(off))
+        if off in frames:
+            add(frames[off])
+        if instr:
+            add(instr)
+
+    #Generic code attributes
+    for name in code_attributes:
+        #We can't disassemble these because Jasmin's format for these attributes
+        #is overly cumbersome and not easy to disassemble into, but we can't just
+        #leave them as binary blobs either as they are verified by the JVM and the
+        #later two contain constant pool references which won't be preserved even
+        #if the bytecode isn't changed. For now, we just ommit them entirely.
+        #TODO - find a better solution
+        if name in ("LineNumberTable","LocalVariableTable","LocalVariableTypeTable"):
+            continue
+
+        for name_ind, attr in code_attributes[name]:
+            add('.codeattribute {} {!r}'.format(poolm.utfref(name_ind), attr))
+
+def getVerificationType(bytes_, poolm, getLbl):
+    s = codes.vt_keywords[bytes_.get('>B')]
+    if s == 'Object':
+        s += ' ' + poolm.classref(bytes_.get('>H'))
+    elif s == 'Uninitialized':
+        s += ' ' + getLbl(bytes_.get('>H'))
+    return s
+
+def getStackMapTable(code_attributes, poolm, getLbl):
+    smt_attrs = code_attributes['StackMapTable']
+
+    frames = {}
+    offset = 0
+
+    if smt_attrs:
+        assert(len(smt_attrs) == 1)
+        bytes_ = binUnpacker(smt_attrs.pop()[1])
+        count = bytes_.get('>H')
+        getVT = lambda: getVerificationType(bytes_, poolm, getLbl)
+
+        for _ in range(count):
+            tag = bytes_.get('>B')
+            header, contents = None, []
+
+            if 0 <= tag <= 63:
+                offset += tag
+                header = 'same'
+            elif 64 <= tag <= 127:
+                offset += tag - 64
+                header = 'same_locals_1_stack_item'
+                contents.append('\tstack ' + getVT())
+            elif tag == 247:
+                offset += bytes_.get('>H')
+                header = 'same_locals_1_stack_item_extended'
+                contents.append('\tstack ' + getVT())
+            elif 248 <= tag <= 250:
+                offset += bytes_.get('>H')
+                header = 'chop ' + str(251-tag)
+            elif tag == 251:
+                offset += bytes_.get('>H')
+                header = 'same_extended'
+            elif 252 <= tag <= 254:
+                offset += bytes_.get('>H')
+                header = 'append'
+                contents.append('\tlocals ' + ' '.join(getVT() for _ in range(tag-251)))
+            elif tag == 255:
+                offset += bytes_.get('>H')
+                header = 'full'
+                local_count = bytes_.get('>H')
+                contents.append('\tlocals ' + ' '.join(getVT() for _ in range(local_count)))
+                stack_count = bytes_.get('>H')
+                contents.append('\tstack ' + ' '.join(getVT() for _ in range(stack_count)))
+
+            if contents:
+                contents.append('.end stack')
+            contents = ['.stack ' + header] + contents
+            frame = '\n'.join(contents)
+            frames[offset] = frame
+            offset += 1 #frames after the first have an offset one larger than the listed offset
+    return frames
+
+def disCFMAttribute(name_ind, name, bytes_, add, poolm):
+    for vis in ('Visible', 'Invisible'):
+        if name == 'Runtime{}Annotations'.format(vis):
+            count = bytes_.get('>H')
+            for _ in range(count):
+                disAnnotation(bytes_, '.runtime{} '.format(vis.lower()), add, poolm, '')
+            if count: #otherwise we'll create an empty generic attribute
+                return
+
+    if name == "Signature":
+        add('.signature {}'.format(poolm.utfref(bytes_.get('>H'))))
+        return
+    #Create generic attribute if it can't be represented by a standard directive
+    add('.attribute {} {!r}'.format(poolm.utfref(name_ind), bytes_.getRaw(bytes_.size())))
+
+def disMethodAttribute(name_ind, name, bytes_, add, poolm):
+    if name == 'Code':
+        return
+    elif name == 'AnnotationDefault':
+        disElementValue(bytes_, '.annotationdefault ', add, poolm, '')
+        return
+    elif name == 'Exceptions':
+        count = bytes_.get('>H')
+        for _ in range(count):
+            add('.throws ' + poolm.classref(bytes_.get('>H')))
+        if count: #otherwise we'll create an empty generic attribute
+            return
+
+    for vis in ('Visible', 'Invisible'):
+        if name == 'Runtime{}ParameterAnnotations'.format(vis):
+            for i in range(bytes_.get('>B')):
+                for _ in range(bytes_.get('>H')):
+                    disAnnotation(bytes_, '.runtime{} parameter {} '.format(vis.lower(), i), add, poolm, '')
+            return #generic fallback on empty list not yet supported
+
+    disCFMAttribute(name_ind, name, bytes_, add, poolm)
+
+def disMethod(method, add, poolm):
+    mflags = ' '.join(map(str.lower, method.flags))
+    add('.method {} {} : {}'.format(mflags, poolm.utfref(method.name_id), poolm.utfref(method.desc_id)))
+
+    for name_ind, name, attr in getAttributeTriples(method):
+        disMethodAttribute(name_ind, name, binUnpacker(attr), add, poolm)
+
+    disMethodCode(method.code, add, poolm)
+    add('.end method')
+
+def _disEVorAnnotationSub(bytes_, add, poolm, isAnnot, init_prefix, init_indent):
+    C_ANNOT, C_ANNOT2, C_ANNOT3, C_EV, C_EV2 = range(5)
+    init_callt = C_ANNOT if isAnnot else C_EV
+
+    stack = [(init_callt, init_prefix, init_indent)]
+    while stack:
+        callt, prefix, indent = stack.pop()
+
+        if callt == C_ANNOT:
+            add(indent + prefix + 'annotation ' + poolm.utfref(bytes_.get('>H')))
+            #ones we want to happen last should be first on the stack. Annot3 is the final call which ends the annotation
+            stack.append((C_ANNOT3, None, indent))
+            stack.extend([(C_ANNOT2, None, indent)] * bytes_.get('>H'))
+
+        elif callt == C_ANNOT2:
+            key = poolm.utfref(bytes_.get('>H'))
+            stack.append((C_EV, key + ' = ', indent+'\t'))
+
+        elif callt == C_ANNOT3:
+            add(indent + '.end annotation')
+
+        elif callt == C_EV:
+            tag = codes.et_rtags[bytes_.getRaw(1)]
+            if tag == 'annotation':
+                stack.append((C_ANNOT, prefix, indent + '\t'))
+            else:
+                if tag in ('byte','char','double','int','float','long','short','boolean','string'):
+                    val = poolm.ldc(bytes_.get('>H'))
+                elif tag == 'class':
+                    val = poolm.utfref(bytes_.get('>H'))
+                elif tag == 'enum':
+                    val = poolm.utfref(bytes_.get('>H')) + ' ' + poolm.utfref(bytes_.get('>H'))
+                elif tag == 'array':
+                    val = ''
+
+                add(indent + '{} {} {}'.format(prefix, tag, val))
+                if tag == 'array':
+                    for _ in range(bytes_.get('>H')):
+                        stack.append((C_EV, '', indent+'\t'))
+                    stack.append((C_EV2, None, indent))
+
+        elif callt == C_EV2:
+            add(indent + '.end array')
+
+def disElementValue(bytes_, prefix, add, poolm, indent):
+    _disEVorAnnotationSub(bytes_, add, poolm, False, prefix, indent)
+
+def disAnnotation(bytes_, prefix, add, poolm, indent):
+    _disEVorAnnotationSub(bytes_, add, poolm, True, prefix, indent)
+
+#Todo - make fields automatically unpack this themselves
+def getConstValue(field):
+    if not field.static:
+        return None
+    const_attrs = [attr for attr in field.attributes if attr[0] == 'ConstantValue']
+    if const_attrs:
+        assert(len(const_attrs) == 1)
+        bytes_ = binUnpacker(const_attrs[0][1])
+        return bytes_.get('>H')
+
+_classflags = [(v,k.lower()) for k,v in ClassFile.flagVals.items()]
+def disInnerClassesAttribute(name_ind, length, bytes_, add, poolm):
+    count = bytes_.get('>H')
+
+    if length != 2+8*count:
+        add('.innerlength {}'.format(length))
+
+    for _ in range(count):
+        inner, outer, innername, flagbits = bytes_.get('>HHHH')
+
+        flags = [v for k,v in _classflags if k&flagbits]
+        inner = poolm.classref(inner)
+        outer = poolm.classref(outer)
+        innername = poolm.utfref(innername)
+
+        add('.inner {} {} {} {}'.format(' '.join(flags), innername, inner, outer))
+
+    if not count:
+        add('.attribute InnerClasses "\\0\\0"')
+
+def disOtherClassAttribute(name_ind, name, bytes_, add, poolm):
+    assert(name != 'InnerClasses')
+    if name == 'EnclosingMethod':
+        cls, nat = bytes_.get('>HH')
+        add('.enclosing method {} {}'.format(poolm.classref(cls), poolm.multiref(nat)))
+        return
+    disCFMAttribute(name_ind, name, bytes_, add, poolm)
+
+def disassemble(cls):
+    lines = []
+    add = lines.append
+    poolm = PoolManager(cls.cpool)
+
+    # def add(s): print s
+    add('.version {0[0]} {0[1]}'.format(cls.version))
+
+    class_attributes = getAttributesDict(cls)
+    if 'SourceFile' in class_attributes:
+        bytes_ = binUnpacker(class_attributes['SourceFile'].pop()[1])
+        val_ind = bytes_.get('>H')
+        add('.source {}'.format(poolm.utfref(val_ind)))
+
+    if 'BootstrapMethods' in class_attributes:
+        bytes_ = binUnpacker(class_attributes['BootstrapMethods'].pop()[1])
+        count = bytes_.get('>H')
+        for _ in range(count):
+            arg1, argc = bytes_.get('>HH')
+            args = (arg1,) + bytes_.get('>'+'H'*argc, forceTuple=True)
+            poolm.bootstrap_methods.append(args)
+
+    cflags = ' '.join(map(str.lower, cls.flags))
+    add('.class {} {}'.format(cflags, poolm.classref(cls.this)))
+    add('.super {}'.format(poolm.classref(cls.super)))
+    for ii in cls.interfaces_raw:
+        add('.implements {}'.format(poolm.classref(ii)))
+
+    for name in class_attributes:
+        if name == "InnerClasses":
+            assert(len(class_attributes[name]) == 1)
+            for name_ind, (length, attr) in class_attributes[name]:
+                disInnerClassesAttribute(name_ind, length, binUnpacker(attr), add, poolm)
+        else:
+            for name_ind, attr in class_attributes[name]:
+                disOtherClassAttribute(name_ind, name, binUnpacker(attr), add, poolm)
+
+    add('')
+    for field in cls.fields:
+        fflags = ' '.join(map(str.lower, field.flags))
+        const = getConstValue(field)
+
+        if const is not None:
+            add('.field {} {} {} = {}'.format(fflags, poolm.utfref(field.name_id), poolm.utfref(field.desc_id), poolm.ldc(const)))
+        else:
+            add('.field {} {} {}'.format(fflags, poolm.utfref(field.name_id), poolm.utfref(field.desc_id)))
+
+        facount = 0
+        for name_ind, name, attr in getAttributeTriples(field):
+            if name == 'ConstantValue' and field.static:
+                continue
+            disMethodAttribute(name_ind, name, binUnpacker(attr), add, poolm)
+            facount += 1
+        if facount > 0:
+            add('.end field')
+            add('')
+
+    add('')
+
+    for method in cls.methods:
+        disMethod(method, add, poolm)
+        add('')
+
+    poolm.printConstDefs(add)
+    return '\n'.join(lines)
\ No newline at end of file
diff --git a/own/krakatau/Krakatau/Krakatau/assembler/instructions.py b/own/krakatau/Krakatau/Krakatau/assembler/instructions.py
new file mode 100644
--- /dev/null
+++ b/own/krakatau/Krakatau/Krakatau/assembler/instructions.py
@@ -0,0 +1,11 @@
+instrs_noarg = ('nop', 'aconst_null', 'iconst_m1', 'iconst_0', 'iconst_1', 'iconst_2', 'iconst_3', 'iconst_4', 'iconst_5', 'lconst_0', 'lconst_1', 'fconst_0', 'fconst_1', 'fconst_2', 'dconst_0', 'dconst_1', 'iload_0', 'iload_1', 'iload_2', 'iload_3', 'lload_0', 'lload_1', 'lload_2', 'lload_3', 'fload_0', 'fload_1', 'fload_2', 'fload_3', 'dload_0', 'dload_1', 'dload_2', 'dload_3', 'aload_0', 'aload_1', 'aload_2', 'aload_3', 'iaload', 'laload', 'faload', 'daload', 'aaload', 'baload', 'caload', 'saload', 'istore_0', 'istore_1', 'istore_2', 'istore_3', 'lstore_0', 'lstore_1', 'lstore_2', 'lstore_3', 'fstore_0', 'fstore_1', 'fstore_2', 'fstore_3', 'dstore_0', 'dstore_1', 'dstore_2', 'dstore_3', 'astore_0', 'astore_1', 'astore_2', 'astore_3', 'iastore', 'lastore', 'fastore', 'dastore', 'aastore', 'bastore', 'castore', 'sastore', 'pop', 'pop2', 'dup', 'dup_x1', 'dup_x2', 'dup2', 'dup2_x1', 'dup2_x2', 'swap', 'iadd', 'ladd', 'fadd', 'dadd', 'isub', 'lsub', 'fsub', 'dsub', 'imul', 'lmul', 'fmul', 'dmul', 'idiv', 'ldiv', 'fdiv', 'ddiv', 'irem', 'lrem', 'frem', 'drem', 'ineg', 'lneg', 'fneg', 'dneg', 'ishl', 'lshl', 'ishr', 'lshr', 'iushr', 'lushr', 'iand', 'land', 'ior', 'lor', 'ixor', 'lxor', 'i2l', 'i2f', 'i2d', 'l2i', 'l2f', 'l2d', 'f2i', 'f2l', 'f2d', 'd2i', 'd2l', 'd2f', 'i2b', 'i2c', 'i2s', 'lcmp', 'fcmpl', 'fcmpg', 'dcmpl', 'dcmpg', 'ireturn', 'lreturn', 'freturn', 'dreturn', 'areturn', 'return', 'arraylength', 'athrow', 'monitorenter', 'monitorexit')
+
+instrs_int = ('bipush', 'sipush', 'iload', 'lload', 'fload', 'dload', 'aload', 'istore', 'lstore', 'fstore', 'dstore', 'astore', 'ret')
+
+instrs_lbl = ('ifeq', 'ifne', 'iflt', 'ifge', 'ifgt', 'ifle', 'if_icmpeq', 'if_icmpne', 'if_icmplt', 'if_icmpge', 'if_icmpgt', 'if_icmple', 'if_acmpeq', 'if_acmpne', 'goto', 'jsr', 'ifnull', 'ifnonnull', 'goto_w', 'jsr_w')
+
+instrs_cp = ('ldc', 'ldc_w', 'ldc2_w', 'getstatic', 'putstatic', 'getfield', 'putfield', 'invokevirtual', 'invokespecial', 'invokestatic', 'invokedynamic', 'new', 'anewarray', 'checkcast', 'instanceof')
+
+instrs_other = ('iinc', 'tableswitch', 'lookupswitch', 'invokeinterface', 'newarray', 'wide', 'multianewarray')
+
+allinstructions = ('nop', 'aconst_null', 'iconst_m1', 'iconst_0', 'iconst_1', 'iconst_2', 'iconst_3', 'iconst_4', 'iconst_5', 'lconst_0', 'lconst_1', 'fconst_0', 'fconst_1', 'fconst_2', 'dconst_0', 'dconst_1', 'bipush', 'sipush', 'ldc', 'ldc_w', 'ldc2_w', 'iload', 'lload', 'fload', 'dload', 'aload', 'iload_0', 'iload_1', 'iload_2', 'iload_3', 'lload_0', 'lload_1', 'lload_2', 'lload_3', 'fload_0', 'fload_1', 'fload_2', 'fload_3', 'dload_0', 'dload_1', 'dload_2', 'dload_3', 'aload_0', 'aload_1', 'aload_2', 'aload_3', 'iaload', 'laload', 'faload', 'daload', 'aaload', 'baload', 'caload', 'saload', 'istore', 'lstore', 'fstore', 'dstore', 'astore', 'istore_0', 'istore_1', 'istore_2', 'istore_3', 'lstore_0', 'lstore_1', 'lstore_2', 'lstore_3', 'fstore_0', 'fstore_1', 'fstore_2', 'fstore_3', 'dstore_0', 'dstore_1', 'dstore_2', 'dstore_3', 'astore_0', 'astore_1', 'astore_2', 'astore_3', 'iastore', 'lastore', 'fastore', 'dastore', 'aastore', 'bastore', 'castore', 'sastore', 'pop', 'pop2', 'dup', 'dup_x1', 'dup_x2', 'dup2', 'dup2_x1', 'dup2_x2', 'swap', 'iadd', 'ladd', 'fadd', 'dadd', 'isub', 'lsub', 'fsub', 'dsub', 'imul', 'lmul', 'fmul', 'dmul', 'idiv', 'ldiv', 'fdiv', 'ddiv', 'irem', 'lrem', 'frem', 'drem', 'ineg', 'lneg', 'fneg', 'dneg', 'ishl', 'lshl', 'ishr', 'lshr', 'iushr', 'lushr', 'iand', 'land', 'ior', 'lor', 'ixor', 'lxor', 'iinc', 'i2l', 'i2f', 'i2d', 'l2i', 'l2f', 'l2d', 'f2i', 'f2l', 'f2d', 'd2i', 'd2l', 'd2f', 'i2b', 'i2c', 'i2s', 'lcmp', 'fcmpl', 'fcmpg', 'dcmpl', 'dcmpg', 'ifeq', 'ifne', 'iflt', 'ifge', 'ifgt', 'ifle', 'if_icmpeq', 'if_icmpne', 'if_icmplt', 'if_icmpge', 'if_icmpgt', 'if_icmple', 'if_acmpeq', 'if_acmpne', 'goto', 'jsr', 'ret', 'tableswitch', 'lookupswitch', 'ireturn', 'lreturn', 'freturn', 'dreturn', 'areturn', 'return', 'getstatic', 'putstatic', 'getfield', 'putfield', 'invokevirtual', 'invokespecial','invokestatic', 'invokeinterface', 'invokedynamic', 'new', 'newarray', 'anewarray', 'arraylength', 'athrow', 'checkcast', 'instanceof', 'monitorenter', 'monitorexit', 'wide', 'multianewarray', 'ifnull', 'ifnonnull', 'goto_w', 'jsr_w')
\ No newline at end of file
diff --git a/own/krakatau/Krakatau/Krakatau/assembler/parse.py b/own/krakatau/Krakatau/Krakatau/assembler/parse.py
new file mode 100644
--- /dev/null
+++ b/own/krakatau/Krakatau/Krakatau/assembler/parse.py
@@ -0,0 +1,554 @@
+import ast, struct
+import itertools
+
+from ..classfile import ClassFile
+from ..method import Method
+from ..field import Field
+
+#Important to import tokens here even though it appears unused, as ply uses it
+from .tokenize import tokens, wordget, flags
+from .assembler import PoolRef
+
+#Specify the starting symbol
+start = 'top'
+
+###############################################################################
+name_counter = itertools.count()
+def addRule(func, name, *rhs_rules):
+    def _inner(p):
+        func(p)
+    _inner.__doc__ = name + ' : ' + '\n| '.join(rhs_rules)
+    fname = 'p_{}'.format(next(name_counter))
+    globals()[fname] = _inner
+
+def list_sub(p):p[0] = p[1] + p[2:]
+def listRule(name): #returns a list
+    name2 = name + 's'
+    addRule(list_sub, name2, '{} {}'.format(name2, name), 'empty')    
+
+def nothing(p):pass
+def assign1(p):p[0] = p[1]
+def assign2(p):p[0] = p[2]
+def upper1(p): p[0] = p[1].upper()
+
+# Common Rules ################################################################
+addRule(nothing, 'sep', 'sep NEWLINE', 'NEWLINE')
+
+def p_empty(p):
+    'empty :'
+    p[0] = []
+
+def p_intl(p):
+    '''intl : INT_LITERAL'''
+    p[0] = ast.literal_eval(p[1])
+
+def p_longl(p):
+    '''longl : LONG_LITERAL'''
+    p[0] = ast.literal_eval(p[1][:-1])
+
+#Todo - find a better way of handling floats
+def parseFloat(s):
+    s = s[:-1]
+    if s.strip('-')[:2].lower() == '0x':
+        f = float.fromhex(s)
+    else:
+        f = float(s)
+    return struct.unpack('>i', struct.pack('>f', f))[0]
+
+def parseDouble(s):
+    if s.strip('-')[:2].lower() == '0x':
+        f = float.fromhex(s)
+    else:
+        f = float(s)
+    return struct.unpack('>q', struct.pack('>d', f))[0]
+
+def p_floatl(p):
+    '''floatl : FLOAT_LITERAL'''
+    p[0] = parseFloat(p[1])
+def p_doublel(p):
+    '''doublel : DOUBLE_LITERAL'''
+    p[0] = parseDouble(p[1])
+
+#We can allow keywords as inline classnames as long as they aren't flag names
+#which would be ambiguous. We don't allow directives to simplfy the grammar
+#rules, since they wouldn't be valid identifiers anyway.
+badwords = frozenset(map(str.lower, flags))
+badwords |= frozenset(k for k in wordget if '.' in k) 
+oktokens = frozenset(v for k,v in wordget.items() if k not in badwords)
+addRule(assign1, 'notflag', 'WORD', 'STRING_LITERAL', *oktokens)
+
+def p_ref(p):
+    '''ref : CPINDEX'''
+    s = p[1][1:-1]
+    try:
+        i = int(s)
+        if 0 <= i <= 0xFFFF:
+            p[0] = PoolRef(index=i)
+        else:
+            p[0] = PoolRef(lbl=s)    
+    except ValueError:
+        p[0] = PoolRef(lbl=s)
+
+def p_utf8_notref(p):
+    '''utf8_notref : notflag'''
+    p[0] = PoolRef('Utf8', p[1])
+
+def p_class_notref(p):
+    '''class_notref : utf8_notref'''
+    p[0] = PoolRef('Class', p[1])
+
+def p_string_notref(p):
+    '''string_notref : utf8_notref'''
+    p[0] = PoolRef('String', p[1])
+
+def p_nat_notref(p):
+    '''nameandtype_notref : utf8ref utf8ref'''
+    p[0] = PoolRef('NameAndType', p[1], p[2])
+
+def p_field_notref(p):
+    '''field_notref : classref nameandtyperef'''
+    p[0] = PoolRef('Field', p[1], p[2])
+
+def p_method_notref(p):
+    '''method_notref : classref nameandtyperef'''
+    p[0] = PoolRef('Method', p[1], p[2])
+
+def p_imethod_notref(p):
+    '''interfacemethod_notref : classref nameandtyperef'''
+    p[0] = PoolRef('InterfaceMethod', p[1], p[2])
+
+#constant pool types related to InvokeDynamic handled later
+
+for _name in ('utf8','class', 'nameandtype', 'method', 'interfacemethod', 'methodhandle'):
+    addRule(assign1, '{}ref'.format(_name), '{}_notref'.format(_name), 'ref')
+
+###############################################################################
+def p_classnoend(p):
+    '''classnoend : version_opt class_directive_lines classdec superdec interfacedecs class_directive_lines topitems'''
+    p[0] = tuple(p[1:])
+
+addRule(assign1, 'classwithend', 'classnoend D_END CLASS sep')
+listRule('classwithend')
+
+def p_top(p):
+    '''top : sep classwithends classnoend'''
+    p[0] = p[2] + [p[3]]
+#case where all classes have an end
+addRule(assign2, 'top', 'sep classwithends')
+
+def p_version(p):
+    '''version_opt : D_VERSION intl intl sep'''
+    p[0] = p[2], p[3]
+addRule(assign1, 'version_opt', 'empty')
+
+###############################################################################
+for c, type_ in zip('cmf', (ClassFile, Method, Field)):
+    _name = "{}flag".format(c)
+    addRule(upper1, _name, *list(type_.flagVals))
+    listRule(_name)
+
+def p_classdec(p):
+    '''classdec : D_CLASS cflags classref sep 
+                | D_INTERFACE cflags classref sep'''
+    #if interface, add interface to flags
+    p[0] = (p[1] == '.interface'), p[2], p[3]
+
+addRule(assign2, 'superdec', 'D_SUPER classref sep')
+addRule(assign2, 'interfacedec', 'D_IMPLEMENTS classref sep')
+listRule('interfacedec')
+
+addRule(assign1, 'class_directive', 'classattribute', 'innerlength_dir')
+addRule(assign1, 'class_directive_line', 'class_directive sep')
+listRule('class_directive_line')
+
+def p_topitem_c(p):
+    '''topitem : const_spec'''
+    p[0] = 'const', p[1]
+def p_topitem_f(p):
+    '''topitem : field_spec'''
+    p[0] = 'field', p[1]
+def p_topitem_m(p):
+    '''topitem : method_spec'''
+    p[0] = 'method', p[1]
+listRule('topitem')
+
+###############################################################################
+#invoke dynamic stuff
+from .codes import handle_codes
+_handle_token_types = set(wordget.get(x, 'WORD') for x in handle_codes)
+def p_handle(p):
+    p[0] = handle_codes[p[1]]
+p_handle.__doc__ = "handlecode : " + '\n| '.join(_handle_token_types)
+
+#The second argument's type depends on the code, so we require an explicit reference for simplicity
+def p_methodhandle_notref(p):
+    '''methodhandle_notref : handlecode ref'''
+    p[0] = PoolRef('MethodHandle', p[1], p[2])
+
+def p_methodtype_notref(p):
+    '''methodtype_notref : utf8_notref'''
+    p[0] = PoolRef('Methodtype', p[1])
+
+addRule(assign1, 'bootstrap_arg', 'ref') #TODO - allow inline constants and strings?
+listRule('bootstrap_arg')
+
+def p_invokedynamic_notref(p):
+    '''invokedynamic_notref : methodhandleref bootstrap_args COLON nameandtyperef'''
+    args = [p[1]] + p[2] + [p[4]]
+    p[0] = PoolRef('InvokeDynamic', *args)
+
+###############################################################################
+def p_const_spec(p):
+    '''const_spec : D_CONST ref EQUALS const_rhs sep'''
+    p[0] = p[2], p[4]
+
+def assignPoolSingle(typen):
+    def inner(p):
+        p[0] = PoolRef(typen, p[2])
+    return inner
+
+addRule(assign1, 'const_rhs', 'ref')
+for tt in ['UTF8', 'CLASS','STRING','NAMEANDTYPE','FIELD','METHOD','INTERFACEMETHOD',
+            'METHODHANDLE','METHODTYPE','INVOKEDYNAMIC']:
+    addRule(assign2, 'const_rhs', '{} {}_notref'.format(tt, tt.lower()))
+
+#these are special cases, since they take a single argument
+#and the notref version can't have a ref as its argument due to ambiguity
+for ptype in ('Class','String','MethodType'):
+    addRule(assignPoolSingle(ptype), 'const_rhs', ptype.upper() + ' ref')
+
+for ptype in ('Int','Float','Long','Double'):
+    addRule(assignPoolSingle(ptype), 'const_rhs', '{} {}l'.format(ptype.upper(), ptype.lower()))
+###############################################################################
+
+
+def p_field_spec(p):
+    '''field_spec : D_FIELD fflags utf8ref utf8ref field_constval fieldattribute_list'''
+    p[0] = p[2:7]
+
+addRule(nothing, 'field_constval', 'empty')
+addRule(assign2, 'field_constval', 'EQUALS ref', 
+                                    'EQUALS ldc1_notref', 
+                                    'EQUALS ldc2_notref')
+
+#Sadly, we must only allow .end field when at least one attribute is specified
+#in order to avoid grammatical ambiguity. JasminXT does not share this problem
+#because it lacks the .end class syntax which causes the conflict
+def p_field_attrlist1(p):
+    '''field_al_nonempty : fieldattribute sep field_al_nonempty'''
+    p[0] = [p[1]]+ p[3]
+def p_field_attrlist2(p):
+    '''field_al_nonempty : fieldattribute sep D_END FIELD sep'''
+    p[0] = [p[1]]
+
+addRule(assign2, 'fieldattribute_list', 'sep field_al_nonempty', 'sep empty')
+
+
+def p_method_spec(p):
+    '''method_spec : defmethod statements endmethod'''
+    p[0] = p[1],p[2]
+
+def p_defmethod_0(p):
+    '''defmethod : D_METHOD mflags jas_meth_namedesc sep'''
+    p[0] = p[2],p[3] 
+def p_defmethod_1(p):
+    '''defmethod : D_METHOD mflags utf8ref COLON utf8ref sep'''
+    p[0] = p[2],(p[3], p[5]) 
+
+def p_jas_meth_namedesc(p):
+    '''jas_meth_namedesc : WORD'''
+    name, paren, desc = p[1].rpartition('(')
+    name = PoolRef('Utf8', name)
+    desc = PoolRef('Utf8', paren+desc)
+    p[0] = name, desc
+addRule(nothing, 'endmethod', 'D_END METHOD sep')
+
+def p_statement_0(p):
+    '''statement : method_directive sep'''
+    p[0] = False, p[1]
+def p_statement_1(p):
+    '''statement : code_directive sep'''
+    p[0] = True, (False, p[1])
+def p_statement_2(p):
+    '''statement : empty instruction sep 
+                | lbldec instruction sep
+                | lbldec sep'''
+    p[0] = True, (True, ((p[1] or None), p[2]))
+listRule('statement')
+
+addRule(assign1, 'lbldec', 'lbl COLON')
+addRule(assign1, 'method_directive', 'methodattribute')
+addRule(assign1, 'code_directive', 'limit_dir', 'except_dir','localvar_dir','linenumber_dir','stack_dir', 'generic_codeattribute_dir')
+
+def p_limit_dir(p):
+    '''limit_dir : D_LIMIT LOCALS intl 
+                | D_LIMIT STACK intl'''
+    p[0] = p[1], (p[2], p[3])
+
+def p_except_dir(p):
+    '''except_dir : D_CATCH classref FROM lbl TO lbl USING lbl'''
+    p[0] = p[1], (p[2], p[4], p[6], p[8])
+
+def p_linenumber_dir(p):
+    '''linenumber_dir : D_LINE intl'''
+    p[0] = p[1], p[2]
+
+def p_localvar_dir(p):
+    '''localvar_dir : D_VAR intl IS utf8ref utf8ref FROM lbl TO lbl'''
+    p[0] = p[1], (p[2], p[4], p[5], p[7], p[9])
+
+def p_instruction(p):
+    '''instruction : OP_NONE
+                    | OP_INT intl
+                    | OP_INT_INT intl intl
+                    | OP_LBL lbl
+                    | OP_FIELD fieldref_or_jas
+                    | OP_METHOD methodref_or_jas
+                    | OP_METHOD_INT imethodref_or_jas intl
+                    | OP_DYNAMIC ref
+                    | OP_CLASS classref
+                    | OP_CLASS_INT classref intl
+                    | OP_LDC1 ldc1_ref
+                    | OP_LDC2 ldc2_ref
+                    | OP_NEWARR nacode
+                    | OP_LOOKUPSWITCH luswitch
+                    | OP_TABLESWITCH tblswitch
+                    | OP_WIDE wide_instr
+                    '''
+    if p[1] == 'invokenonvirtual':
+        p[1] = 'invokespecial'
+    p[0] = tuple(p[1:])
+    #these instructions have 0 padding at the end
+    #this is kind of an ungly hack, but the best way I could think of
+    if p[1] in ('invokeinterface','invokedynamic'):
+        p[0] += (0,)
+
+addRule(assign1, 'lbl', 'WORD')
+addRule(assign1, 'fieldref_or_jas', 'jas_fieldref', 'ref', 'inline_fieldref')
+def p_jas_fieldref(p):
+    '''jas_fieldref : WORD WORD'''
+    class_, sep, name = p[1].replace('.','/').rpartition('/')
+
+    desc = PoolRef('Utf8', p[2])
+    class_ = PoolRef('Class', PoolRef('Utf8', class_))
+    name = PoolRef('Utf8', name)
+    nt = PoolRef('NameAndType', name, desc)
+    p[0] = PoolRef('Field', class_, nt)
+
+#This is an ugly hack to work around the fact that Jasmin syntax would otherwise be impossible to 
+#handle with a LALR(1) parser
+def p_inline_fieldref_1(p):
+    '''inline_fieldref : WORD nameandtyperef
+                        | STRING_LITERAL nameandtyperef'''
+    class_ = PoolRef('Class', PoolRef('Utf8', p[1]))
+    p[0] = PoolRef('Field', class_, p[2])
+def p_inline_fieldref_2(p):
+    '''inline_fieldref : ref nameandtyperef'''
+    p[0] = PoolRef('Field', p[1], p[2])
+
+
+def p_jas_meth_classnamedesc(p):
+    '''jas_methodref : WORD'''
+    name, paren, desc = p[1].rpartition('(')
+    class_, sep, name = name.replace('.','/').rpartition('/')
+    desc = paren + desc
+
+    class_ = PoolRef('Class', PoolRef('Utf8', class_))
+    nt = PoolRef('NameAndType', PoolRef('Utf8', name), PoolRef('Utf8', desc))
+    p[0] = class_, nt
+
+addRule(assign1, 'methodref_or_jas', 'methodref')
+def p_methodref_or_jas(p):
+    '''methodref_or_jas : jas_methodref'''
+    p[0] = PoolRef('Method', *p[1])
+
+addRule(assign1, 'imethodref_or_jas', 'interfacemethodref')
+def p_imethodref_or_jas(p):
+    '''imethodref_or_jas : jas_methodref'''
+    p[0] = PoolRef('InterfaceMethod', *p[1])
+
+
+from .codes import newarr_codes
+_newarr_token_types = set(wordget.get(x, 'WORD') for x in newarr_codes)
+def p_nacode(p):
+    p[0] = newarr_codes[p[1]]
+p_nacode.__doc__ = "nacode : " + '\n| '.join(_newarr_token_types)
+
+addRule(assign1, 'ldc1_ref', 'ldc1_notref', 'ref')
+def p_ldc1_notref_string(p):
+    '''ldc1_notref : STRING_LITERAL'''
+    p[0] = PoolRef('String', PoolRef('Utf8', p[1]))
+def p_ldc1_notref_int(p):
+    '''ldc1_notref : intl'''
+    p[0] = PoolRef('Int', p[1])
+def p_ldc1_notref_float(p):
+    '''ldc1_notref : floatl'''
+    p[0] = PoolRef('Float', p[1])
+
+addRule(assign1, 'ldc2_ref', 'ldc2_notref', 'ref')
+def p_ldc2_notref_long(p):
+    '''ldc2_notref : longl'''
+    p[0] = PoolRef('Long', p[1])
+def p_ldc2_notref_double(p):
+    '''ldc2_notref : doublel'''
+    p[0] = PoolRef('Double', p[1])
+
+def p_defaultentry(p):
+    '''defaultentry : DEFAULT COLON lbl'''
+    p[0] = p[3]
+
+def p_luentry(p):
+    '''luentry : intl COLON lbl sep'''
+    p[0] = p[1], p[3]
+listRule('luentry')
+
+addRule(assign1, 'tblentry', 'lbl sep')
+listRule('tblentry')
+
+def p_lookupswitch(p):
+    '''luswitch : empty sep luentrys defaultentry'''
+    p[0] = p[1], p[3], p[4]
+
+def p_tableswitch(p):
+    '''tblswitch : intl sep tblentrys defaultentry'''
+    p[0] = p[1], p[3], p[4]
+
+def p_wide_instr(p):
+    '''wide_instr : OP_INT intl
+                | OP_INT_INT intl intl'''
+    p[0] = p[1], tuple(p[2:])
+
+#######################################################################
+# Explicit Attributes
+addRule(assign1, 'cfmattribute', 'annotation_dir', 'signature_dir', 'generic_attribute_dir')
+addRule(assign1, 'classattribute', 'cfmattribute', 'sourcefile_dir', 'inner_dir', 'enclosing_dir')
+addRule(assign1, 'fieldattribute', 'cfmattribute')
+addRule(assign1, 'methodattribute', 'cfmattribute', 'throws_dir', 'annotation_param_dir', 'annotation_def_dir')
+
+#Class, field, method
+def p_annotation_dir(p):
+    '''annotation_dir : D_RUNTIMEVISIBLE annotation
+                    | D_RUNTIMEINVISIBLE annotation'''
+    p[0] = p[1], (None, p[2])
+
+def p_signature_dir(p):
+    '''signature_dir : D_SIGNATURE utf8ref'''
+    p[0] = p[1], p[2]
+
+#Class only
+def p_sourcefile_dir(p):
+    '''sourcefile_dir : D_SOURCE utf8ref'''
+    p[0] = p[1], p[2]
+
+def p_inner_dir(p): 
+    '''inner_dir : D_INNER cflags utf8ref classref classref'''
+    p[0] = p[1], (p[4],p[5],p[3],p[2]) #use JasminXT's (flags, name, inner, outer) order but switch internally to correct order
+
+def p_enclosing_dir(p): 
+    '''enclosing_dir : D_ENCLOSING METHOD classref nameandtyperef'''
+    p[0] = p[1], (p[3], p[4])
+
+#This is included here even though strictly speaking, it's not an attribute. Rather it's a directive that affects the assembly
+#of the InnerClasses attribute
+def p_innerlength_dir(p): 
+    '''innerlength_dir : D_INNERLENGTH intl'''
+    p[0] = p[1], p[2]
+
+
+#Method only
+def p_throws_dir(p):
+    '''throws_dir : D_THROWS classref'''
+    p[0] = p[1], p[2]
+
+def p_annotation_param_dir(p):
+    '''annotation_param_dir : D_RUNTIMEVISIBLE PARAMETER intl annotation
+                           | D_RUNTIMEINVISIBLE PARAMETER intl annotation'''
+    p[0] = p[1], (p[3], p[4])
+def p_annotation_def_dir(p):
+    '''annotation_def_dir : D_ANNOTATIONDEFAULT element_value'''
+    p[0] = p[1], p[2]
+
+#Generic
+def p_generic_attribute_dir(p): 
+    '''generic_attribute_dir : D_ATTRIBUTE utf8ref STRING_LITERAL'''
+    p[0] = p[1], (p[2], p[3])
+
+def p_generic_codeattribute_dir(p): 
+    '''generic_codeattribute_dir : D_CODEATTRIBUTE utf8ref STRING_LITERAL'''
+    p[0] = p[1], (p[2], p[3])
+
+#######################################################################
+#Stack map stuff
+addRule(nothing, 'endstack', 'D_END STACK') #directives are not expected to end with a sep
+
+def assign1All(p):p[0] = tuple(p[1:])
+addRule(assign1All, 'verification_type', 'TOP', 'INTEGER', 'FLOAT', 'DOUBLE', 'LONG', 'NULL', 'UNINITIALIZEDTHIS',
+                                        'OBJECT classref', 'UNINITIALIZED lbl')
+listRule('verification_type')
+addRule(assign2, 'locals_vtlist', 'LOCALS verification_types sep')
+addRule(assign2, 'stack_vtlist', 'STACK verification_types sep')
+
+def p_stack_dir(p):
+    '''stack_dir_rest : SAME 
+                    | SAME_EXTENDED
+                    | CHOP intl 
+                    | SAME_LOCALS_1_STACK_ITEM sep stack_vtlist endstack
+                    | SAME_LOCALS_1_STACK_ITEM_EXTENDED sep stack_vtlist endstack
+                    | APPEND sep locals_vtlist endstack
+                    | FULL sep locals_vtlist stack_vtlist endstack'''
+    p[0] = '.stackmap', tuple(p[1:])
+addRule(assign2, 'stack_dir', 'D_STACK stack_dir_rest')
+#######################################################################
+#Annotation stuff
+from .codes import et_tags
+primtags = set(wordget.get(x, 'WORD') for x in 'byte char double int float long short boolean string'.split())
+addRule(assign1, 'primtag', *primtags)
+addRule(assign1, 'ldc_any', 'ldc1_notref', 'ldc2_notref', 'ref')
+
+def p_element_value_0(p):
+    '''element_value : primtag ldc_any
+                    | CLASS utf8ref
+                    | ENUM utf8ref utf8ref
+                    | ARRAY sep element_array'''
+    p[0] = et_tags[p[1]], tuple(p[2:])
+def p_element_value_1(p):
+    '''element_value : annotation'''
+    p[0] = '@', (p[1],)
+
+addRule(assign1, 'element_value_line', 'element_value sep')
+listRule('element_value_line')
+addRule(assign1, 'element_array', 'element_value_lines D_END ARRAY')
+
+def p_key_ev_line(p):
+    '''key_ev_line : utf8ref EQUALS element_value_line'''
+    p[0] = p[1], p[3]
+listRule('key_ev_line')
+
+def p_annotation(p):
+    '''annotation : ANNOTATION utf8ref sep key_ev_lines D_END ANNOTATION'''
+    p[0] = p[2], p[4]
+#######################################################################
+
+def p_error(p):
+    if p is None:
+        print "Syntax error: unexpected EOF"
+    else: #remember to subtract 1 from line number since we had a newline at the start of the file
+        print "Syntax error at line {}: unexpected token {!r}".format(p.lineno-1, p.value)
+    
+    #Ugly hack since Ply doesn't provide any useful error information
+    import inspect
+    frame = inspect.currentframe()
+    cvars = frame.f_back.f_locals
+    print 'Expected:', ', '.join(cvars['actions'][cvars['state']].keys())
+    print 'Found:', cvars['ltype']
+    print 'Current stack:', cvars['symstack']
+
+    #Discard the rest of the input so that Ply doesn't attempt error recovery
+    from ply import yacc
+    tok = yacc.token()
+    while tok is not None:
+        tok = yacc.token()
+
+def makeParser(**kwargs):
+    from ply import yacc
+    return yacc.yacc(**kwargs)
\ No newline at end of file
diff --git a/own/krakatau/Krakatau/Krakatau/assembler/tokenize.py b/own/krakatau/Krakatau/Krakatau/assembler/tokenize.py
new file mode 100644
--- /dev/null
+++ b/own/krakatau/Krakatau/Krakatau/assembler/tokenize.py
@@ -0,0 +1,140 @@
+import ast
+
+from ..classfile import ClassFile
+from ..method import Method
+from ..field import Field
+from .. import constant_pool
+from . import instructions as ins
+from . import codes
+
+directives = 'CLASS','INTERFACE','SUPER','IMPLEMENTS','CONST','FIELD','METHOD','END','LIMIT','CATCH','SOURCE','LINE','VAR','THROWS',
+directives += 'VERSION', 'STACK', 'RUNTIMEVISIBLE', 'RUNTIMEINVISIBLE', 'ANNOTATIONDEFAULT', 'INNER', 'ENCLOSING', 'SIGNATURE', 
+directives += 'ATTRIBUTE', 'CODEATTRIBUTE', 'INNERLENGTH'
+keywords = ['CLASS','METHOD','FIELD','LOCALS','STACK','FROM','TO','USING','DEFAULT','IS']
+keywords += ['SAME','SAME_LOCALS_1_STACK_ITEM','SAME_LOCALS_1_STACK_ITEM_EXTENDED','CHOP','SAME_EXTENDED','APPEND','FULL']
+keywords += ['ANNOTATION','ARRAY','PARAMETER']
+flags = ClassFile.flagVals.keys() + Method.flagVals.keys() + Field.flagVals.keys()
+
+lowwords = set().union(keywords, flags)
+casewords = set().union(codes.vt_keywords, constant_pool.name2Type.keys())
+
+wordget = {}
+wordget.update({w.lower():w.upper() for w in lowwords})
+wordget.update({w:w.upper() for w in casewords})
+wordget.update({'.'+w.lower():'D_'+w for w in directives})
+
+assert(set(wordget).isdisjoint(ins.allinstructions))
+for op in ins.instrs_noarg:
+    wordget[op] = 'OP_NONE'
+for op in ins.instrs_int:
+    wordget[op] = 'OP_INT'
+for op in ins.instrs_lbl:
+    wordget[op] = 'OP_LBL'
+for op in ('getstatic', 'putstatic', 'getfield', 'putfield'):
+    wordget[op] = 'OP_FIELD'
+#support invokenonvirtual for backwards compatibility with Jasmin
+for op in ('invokevirtual', 'invokespecial', 'invokestatic', 'invokenonvirtual'): 
+    wordget[op] = 'OP_METHOD'
+for op in ('new', 'anewarray', 'checkcast', 'instanceof'):
+    wordget[op] = 'OP_CLASS'
+for op in ('wide','lookupswitch','tableswitch'):
+    wordget[op] = 'OP_' + op.upper()
+
+wordget['ldc'] = 'OP_LDC1'
+wordget['ldc_w'] = 'OP_LDC1'
+wordget['ldc2_w'] = 'OP_LDC2'
+wordget['iinc'] = 'OP_INT_INT'
+wordget['newarray'] = 'OP_NEWARR'
+wordget['multianewarray'] = 'OP_CLASS_INT'
+wordget['invokeinterface'] = 'OP_METHOD_INT'
+wordget['invokedynamic'] = 'OP_DYNAMIC'
+
+for op in ins.allinstructions:
+    wordget.setdefault(op,op.upper())
+
+#special PLY value

From noreply at buildbot.pypy.org  Mon Dec  8 16:45:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  8 Dec 2014 16:45:01 +0100 (CET)
Subject: [pypy-commit] benchmarks default: (cfbolz, arigo)
Message-ID: <20141208154501.88B5B1D35D9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r283:56cceac3bdf5
Date: 2014-12-08 15:45 +0000
http://bitbucket.org/pypy/benchmarks/changeset/56cceac3bdf5/

Log:	(cfbolz, arigo)

	Add bm_krakatau.

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -82,7 +82,7 @@
              'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast',
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
              'json_bench', 'pidigits', 'hexiom2', 'eparse', 'deltablue',
-             'bm_dulwich_log']:
+             'bm_dulwich_log', 'bm_krakatau']:
     _register_new_bm(name, name, globals(), **opts.get(name, {}))
 
 for name in ['names', 'iteration', 'tcp', 'pb', ]:#'web']:#, 'accepts']:
diff --git a/own/bm_krakatau.py b/own/bm_krakatau.py
--- a/own/bm_krakatau.py
+++ b/own/bm_krakatau.py
@@ -1,9 +1,9 @@
-import sys, os
+import sys, os, cStringIO
 import time
 import util, optparse
 
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'krakatau/Krakatau'))
-print sys.path
+this_dir = os.path.dirname(__file__)
+sys.path.insert(0, os.path.join(this_dir, 'krakatau/Krakatau'))
 
 import Krakatau.ssa
 from Krakatau.environment import Environment
@@ -34,7 +34,7 @@
     return s
 
 def decompileClass():
-    path = ['krakatau/rt.jar']
+    path = [os.path.join(this_dir, 'krakatau/rt.jar')]
     targets = ['javax/swing/plaf/nimbus/ToolBarSouthState']
     e = Environment()
     for part in path:
@@ -49,11 +49,16 @@
 
 def main(n):
     l = []
-    for i in range(n):
-        t0 = time.time()
-        decompileClass()
-        time_elapsed = time.time() - t0
-        l.append(time_elapsed)
+    old_stdout = sys.stdout
+    sys.stdout = cStringIO.StringIO()
+    try:
+        for i in range(n):
+            t0 = time.time()
+            decompileClass()
+            time_elapsed = time.time() - t0
+            l.append(time_elapsed)
+    finally:
+        sys.stdout = old_stdout
     return l
 
 if __name__ == "__main__":

From noreply at buildbot.pypy.org  Mon Dec  8 17:15:26 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  8 Dec 2014 17:15:26 +0100 (CET)
Subject: [pypy-commit] benchmarks default: (cfbolz, arigo)
Message-ID: <20141208161526.8ADDB1D35D6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r284:ed338f71e5f7
Date: 2014-12-08 16:15 +0000
http://bitbucket.org/pypy/benchmarks/changeset/ed338f71e5f7/

Log:	(cfbolz, arigo)

	Add the MDP code (Robert Grosse on pypy-dev)

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -82,7 +82,7 @@
              'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast',
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
              'json_bench', 'pidigits', 'hexiom2', 'eparse', 'deltablue',
-             'bm_dulwich_log', 'bm_krakatau']:
+             'bm_dulwich_log', 'bm_krakatau', 'bm_mdp']:
     _register_new_bm(name, name, globals(), **opts.get(name, {}))
 
 for name in ['names', 'iteration', 'tcp', 'pb', ]:#'web']:#, 'accepts']:
diff --git a/own/bm_mdp.py b/own/bm_mdp.py
new file mode 100644
--- /dev/null
+++ b/own/bm_mdp.py
@@ -0,0 +1,299 @@
+import collections, time
+from collections import defaultdict as ddict
+from fractions import Fraction as frac
+
+def tarjanSCC(roots, getChildren):
+    """Return a list of strongly connected components in a graph. If getParents is passed instead of getChildren, the result will be topologically sorted.
+
+    roots - list of root nodes to search from
+    getChildren - function which returns children of a given node
+    """
+
+    sccs = []
+    indexCounter = itertools.count()
+    index = {}
+    lowlink = {}
+    removed = set()
+    subtree = []
+
+    #Use iterative version to avoid stack limits for large datasets
+    stack = [(node, 0) for node in roots]
+    while stack:
+        current, state = stack.pop()
+        if state == 0: #before recursing
+            if current not in index: #if it's in index, it was already visited (possibly earlier on the current search stack)
+                lowlink[current] = index[current] = next(indexCounter)
+                subtree.append(current)
+
+                stack.append((current, 1))
+                stack.extend((child, 0) for child in getChildren(current) if child not in removed)
+        else: #after recursing
+            children = [child for child in getChildren(current) if child not in removed]
+            for child in children:
+                if index[child] <= index[current]: #backedge (or selfedge)
+                    lowlink[current] = min(lowlink[current], index[child])
+                else:
+                    lowlink[current] = min(lowlink[current], lowlink[child])
+                assert(lowlink[current] <= index[current])
+
+            if index[current] == lowlink[current]:
+                scc = []
+                while not scc or scc[-1] != current:
+                    scc.append(subtree.pop())
+
+                sccs.append(tuple(scc))
+                removed.update(scc)
+    return sccs
+
+def topoSort(roots, getParents):
+    """Return a topological sorting of nodes in a graph.
+
+    roots - list of root nodes to search from
+    getParents - function which returns the parents of a given node
+    """
+
+    results = []
+    visited = set()
+
+    #Use iterative version to avoid stack limits for large datasets
+    stack = [(node,0) for node in roots]
+    while stack:
+        current, state = stack.pop()
+        if state == 0: #before recursing
+            if current not in visited:
+                visited.add(current)
+                stack.append((current,1))
+                stack.extend((parent,0) for parent in getParents(current))
+        else: #after recursing
+            assert(current in visited)
+            results.append(current)
+    return results
+
+def getDamages(L, A, D, B, stab, te):
+	x = (2*L)//5
+	x = ((x+2)*A*B)//(D*50) + 2
+	if stab:
+		x += x // 2
+	x = int(x * te)
+	return [(x*z)//255 for z in range(217,256)]
+
+def getCritDist(L, p, A1, A2, D1, D2, B, stab, te):
+	p = min(p, frac(1))
+	norm = getDamages(L, A1, D1, B, stab, te)
+	crit = getDamages(L*2, A2, D2, B, stab, te)
+
+	dist = ddict(frac)
+	for mult, vals in zip([1-p, p], [norm, crit]):
+		mult /= len(vals)
+		for x in vals:
+			dist[x] += mult
+	return dist
+
+def getAvg(dd):
+	return float(sum(k*v for k,v in dd))
+
+def plus12(x): return x + x//8
+def plus50(x): return x + x//2
+
+def replace(t, i, v):
+	if t[i] != v:
+		temp = t[:i] + (v,) + t[i+1:]
+		t = type(t)._make(temp)
+	return t
+
+
+poke_stats_t = collections.namedtuple('poke_stats_t', ['lvl', 'basespeed', 'hp', 'atk', 'df', 'speed', 'spec'])
+stats_t = collections.namedtuple('stats_t', ['atk', 'df', 'speed', 'spec'])
+NOMODS = stats_t(0,0,0,0)
+
+
+fixeddata_t = collections.namedtuple('fixeddata_t', ['maxhp', 'stats', 'lvl', 'badges','basespeed'])
+halfstate_t = collections.namedtuple('halfstate_t', ['fixed','hp','status','statmods','stats'])
+
+
+_statmod_table = [frac(i,2) for i in range(3,9)]
+_statmod_table = [1/x for x in reversed(_statmod_table)] + [frac(1)] + _statmod_table
+
+def applyHPChange(hstate, change):
+	hp = min(hstate.fixed.maxhp, max(0, hstate.hp + change))
+	return hstate._replace(hp=hp)
+
+def applyBadgeBoosts(badges, stats):
+	return stats_t(*[(plus12(x) if b else x) for x,b in zip(stats, badges)])
+
+attack_stats_t = collections.namedtuple('attack_stats_t', ['power', 'isspec', 'stab', 'te', 'crit'])
+attack_data = {
+	'Ember': attack_stats_t(40, True, True, 0.5, False),
+	'Dig': attack_stats_t(100, False, False, 1, False),
+	'Slash': attack_stats_t(70, False, False, 1, True),
+	'Water Gun': attack_stats_t(40, True, True, 2, False),
+	'Bubblebeam': attack_stats_t(65, True, True, 2, False),
+}
+
+def _applyActionSide1(state, act):
+	me, them, extra = state
+
+	if act == 'Super Potion':
+		me = applyHPChange(me, 50)
+		return {(me, them, extra): frac(1)}
+
+	mdata = attack_data[act]
+	aind = 3 if mdata.isspec else 0
+	dind = 3 if mdata.isspec else 1
+	pdiv = 64 if mdata.crit else 512
+	dmg_dist = getCritDist(me.fixed.lvl, frac(me.fixed.basespeed, pdiv),
+		me.stats[aind], me.fixed.stats[aind], them.stats[dind], them.fixed.stats[dind],
+		mdata.power, mdata.stab, mdata.te)
+
+	dist = ddict(frac)
+	for dmg, p in dmg_dist.items():
+		them2 = applyHPChange(them, -dmg)
+		dist[me, them2, extra] += p
+	return dist
+
+def _applyAction(state, side, act):
+	if side == 0:
+		return _applyActionSide1(state, act)
+	else:
+		me, them, extra = state
+		dist = _applyActionSide1((them, me, extra), act)
+		return {(k[1], k[0], k[2]): v for k,v in dist.items()}
+
+def printstate(state):
+	return 'char {} star {}'.format(state[0].hp, state[1].hp)
+def printsp(sp):
+	return '[{}, {}, {}]'.format(sp[0], printstate(sp[1]), ', '.join(map(str, sp[2:])))
+
+class Battle(object):
+	def __init__(self):
+		self.successors = {}
+		self.min = ddict(float)
+		self.max = ddict(lambda:1.0)
+		self.frozen = set()
+
+		self.win = 4, True
+		self.loss = 4, False
+		self.max[self.loss] = 0.0
+		self.min[self.win] = 1.0
+		self.frozen.update([self.win, self.loss])
+
+	def _getSuccessorsA(self, statep):
+		st, state = statep
+		for action in ['Dig', 'Super Potion']:
+			yield (1, state, action)
+
+	def _applyActionPair(self, state, side1, act1, side2, act2, dist, pmult):
+		for newstate, p in _applyAction(state, side1, act1).items():
+			if newstate[0].hp == 0:
+				newstatep = self.loss
+			elif newstate[1].hp == 0:
+				newstatep = self.win
+			else:
+				newstatep = 2, newstate, side2, act2
+			dist[newstatep] += p*pmult
+
+	def _getSuccessorsB(self, statep):
+		st, state, action = statep
+		dist = ddict(frac)
+		for eact, p in [('Water Gun', frac(64, 130)), ('Bubblebeam', frac(66, 130))]:
+			priority1 = state[0].stats.speed + 10000*(action == 'Super Potion')
+			priority2 = state[1].stats.speed + 10000*(action == 'X Defend')
+
+			if priority1 > priority2:
+				self._applyActionPair(state, 0, action, 1, eact, dist, p)
+			elif priority1 < priority2:
+				self._applyActionPair(state, 1, eact, 0, action, dist, p)
+			else:
+				self._applyActionPair(state, 0, action, 1, eact, dist, p/2)
+				self._applyActionPair(state, 1, eact, 0, action, dist, p/2)
+
+		return {k:float(p) for k,p in dist.items() if p>0}
+
+	def _getSuccessorsC(self, statep):
+		st, state, side, action = statep
+		dist = ddict(frac)
+		for newstate, p in _applyAction(state, side, action).items():
+			if newstate[0].hp == 0:
+				newstatep = self.loss
+			elif newstate[1].hp == 0:
+				newstatep = self.win
+			else:
+				newstatep = 0, newstate
+			dist[newstatep] += p
+		return {k:float(p) for k,p in dist.items() if p>0}
+
+	def getSuccessors(self, statep):
+		try:
+			return self.successors[statep]
+		except KeyError:
+			st = statep[0]
+		if st == 0:
+			result = list(self._getSuccessorsA(statep))
+		else:
+			if st == 1:
+				dist = self._getSuccessorsB(statep)
+			elif st == 2:
+				dist = self._getSuccessorsC(statep)
+			result = sorted(dist.items(), key=lambda t:(-t[1], t[0]))
+		self.successors[statep] = result
+		return result
+
+	def getSuccessorsList(self, statep):
+		if statep[0] == 4:
+			return []
+		temp = self.getSuccessors(statep)
+		if statep[0] != 0:
+			temp = zip(*temp)[0] if temp else []
+		return temp
+
+	def evaluate(self, tolerance=0.15):
+		badges = 1, 0, 0, 0
+
+		starfixed = fixeddata_t(59, stats_t(40, 44, 56, 50), 11, NOMODS, 115)
+		starhalf = halfstate_t(starfixed, 59, 0, NOMODS, stats_t(40, 44, 56, 50))
+		charfixed = fixeddata_t(63, stats_t(39, 34, 46, 38), 26, badges, 65)
+		charhalf = halfstate_t(charfixed, 63, 0, NOMODS, applyBadgeBoosts(badges, stats_t(39, 34, 46, 38)))
+		initial_state = charhalf, starhalf, 0
+		initial_statep = 0, initial_state
+
+		dmin, dmax, frozen = self.min, self.max, self.frozen
+		stateps = topoSort([initial_statep], self.getSuccessorsList)
+
+		itercount = 0
+		while dmax[initial_statep] - dmin[initial_statep] > tolerance:
+			itercount += 1
+			#print itercount, dmax[initial_statep] - dmin[initial_statep], len(frozen)
+
+			for sp in stateps:
+				if sp in frozen:
+					continue
+
+				if sp[0] == 0: #choice node
+					dmin[sp] = max(dmin[sp2] for sp2 in self.getSuccessors(sp))
+					dmax[sp] = max(dmax[sp2] for sp2 in self.getSuccessors(sp))
+				else:
+					dmin[sp] = sum(dmin[sp2]*p for sp2,p in self.getSuccessors(sp))
+					dmax[sp] = sum(dmax[sp2]*p for sp2,p in self.getSuccessors(sp))
+				if dmin[sp] >= dmax[sp]:
+					dmax[sp] = dmin[sp] = (dmin[sp] + dmax[sp])/2
+					frozen.add(sp)
+		return (dmax[initial_statep] + dmin[initial_statep])/2
+
+def main(n):
+    l = []
+    for i in range(n):
+        t0 = time.time()
+        Battle().evaluate(0.192)
+        time_elapsed = time.time() - t0
+        l.append(time_elapsed)
+    return l
+
+if __name__ == "__main__":
+    import util, optparse
+    parser = optparse.OptionParser(
+        usage="%prog [options]",
+        description="Test the performance of the MDP benchmark")
+    util.add_standard_options_to(parser)
+    options, args = parser.parse_args()
+
+    util.run_benchmark(options, options.num_runs, main)

From noreply at buildbot.pypy.org  Tue Dec  9 16:07:03 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Dec 2014 16:07:03 +0100 (CET)
Subject: [pypy-commit] pypy default: Kill old test.  Fix old test.
Message-ID: <20141209150703.21F7B1C09B2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74859:b23bf64e6925
Date: 2014-12-09 15:06 +0000
http://bitbucket.org/pypy/pypy/changeset/b23bf64e6925/

Log:	Kill old test. Fix old test.

diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py
--- a/rpython/jit/metainterp/test/test_list.py
+++ b/rpython/jit/metainterp/test/test_list.py
@@ -97,28 +97,6 @@
         assert res == f(10)
         self.check_resops(setarrayitem_gc=0, call=0, getarrayitem_gc=0)
 
-    def test_vlist_alloc_and_set(self):
-        # the check_loops fails, because [non-null] * n is only supported
-        # if n < 15 (otherwise it is implemented as a residual call)
-        jitdriver = JitDriver(greens = [], reds = ['n'])
-        def f(n):
-            l = [1] * 20
-            while n > 0:
-                jitdriver.can_enter_jit(n=n)
-                jitdriver.jit_merge_point(n=n)
-                l = [1] * 20
-                l[3] = 5
-                x = l[-17] + l[5] - 1
-                if n < 3:
-                    return x
-                n -= 1
-            return l[0]
-
-        res = self.meta_interp(f, [10], listops=True)
-        assert res == f(10)
-        py.test.skip("'[non-null] * n' for n >= 15 gives a residual call so far")
-        self.check_loops(setarrayitem_gc=0, getarrayitem_gc=0, call=0)
-
     def test_arraycopy_simpleoptimize(self):
         def f():
             l = [1, 2, 3, 4]
diff --git a/rpython/jit/metainterp/test/test_send.py b/rpython/jit/metainterp/test/test_send.py
--- a/rpython/jit/metainterp/test/test_send.py
+++ b/rpython/jit/metainterp/test/test_send.py
@@ -385,8 +385,7 @@
         self.check_target_token_count(4)
 
     def test_two_behaviors(self):
-        py.test.skip("XXX fix me!!!!!!! problem in optimize.py")
-        myjitdriver = JitDriver(greens = [], reds = ['x', 'y'])
+        myjitdriver = JitDriver(greens = [], reds = ['y', 'x'])
         class Int:
             def __init__(self, value):
                 self.value = value
@@ -402,11 +401,6 @@
             return x.value
         res = self.meta_interp(f, [len(cases)])
         assert res == 110
-        # The generated loops don't contain a new_with_vtable at all.  This
-        # is true if we replace "if cases[y]" above with "if not cases[y]"
-        # -- so there is no good reason that it fails.
-        self.check_loops(new_with_vtable=0)
-        self.check_trace_count(2)
 
     def test_behavior_change_after_a_while(self):
         myjitdriver = JitDriver(greens = [], reds = ['y', 'x'])

From noreply at buildbot.pypy.org  Tue Dec  9 16:32:04 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Dec 2014 16:32:04 +0100 (CET)
Subject: [pypy-commit] pypy default: Add "--verbose" as a cmdline option to
	translate.py
Message-ID: <20141209153204.2CD4C1D3650@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74860:4a5992cc2afd
Date: 2014-12-09 16:30 +0100
http://bitbucket.org/pypy/pypy/changeset/4a5992cc2afd/

Log:	Add "--verbose" as a cmdline option to translate.py

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -129,7 +129,8 @@
                default=False, cmdline=None),
 
     # misc
-    BoolOption("verbose", "Print extra information", default=False),
+    BoolOption("verbose", "Print extra information", default=False,
+               cmdline="--verbose"),
     StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"),
     StrOption("profopt", "Specify profile based optimization script",
               cmdline="--profopt"),

From noreply at buildbot.pypy.org  Tue Dec  9 16:32:05 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Dec 2014 16:32:05 +0100 (CET)
Subject: [pypy-commit] pypy default: (tanzim, arigo)
Message-ID: <20141209153205.835D21D3650@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74861:d49c95b3abf7
Date: 2014-12-09 16:31 +0100
http://bitbucket.org/pypy/pypy/changeset/d49c95b3abf7/

Log:	(tanzim, arigo)

	Ignore the jit_force_virtualizables that are common in or around
	pypy's interpreter main loop.

diff --git a/rpython/translator/backendopt/inline.py b/rpython/translator/backendopt/inline.py
--- a/rpython/translator/backendopt/inline.py
+++ b/rpython/translator/backendopt/inline.py
@@ -478,6 +478,7 @@
               'malloc': 2,
               'instrument_count': 0,
               'debug_assert': -1,
+              'jit_force_virtualizable': 0,
               }
 
 def block_weight(block, weights=OP_WEIGHTS):
diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py
--- a/rpython/translator/backendopt/storesink.py
+++ b/rpython/translator/backendopt/storesink.py
@@ -5,7 +5,7 @@
 from rpython.translator import simplify
 
 def has_side_effects(op):
-    if op.opname == 'debug_assert':
+    if op.opname == 'debug_assert' or op.opname == 'jit_force_virtualizable':
         return False
     try:
         return getattr(llop, op.opname).sideeffects

From noreply at buildbot.pypy.org  Tue Dec  9 16:32:07 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Dec 2014 16:32:07 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20141209153207.DA3A51D3650@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74862:b32441833d66
Date: 2014-12-09 16:31 +0100
http://bitbucket.org/pypy/pypy/changeset/b32441833d66/

Log:	merge heads

diff too long, truncating to 2000 out of 6151 lines

diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,6 +6,10 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
+**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
+on linux, linux64 and windows. We will make it the default on all platforms
+by the time of the next release.
+
 The first thing that you need is to compile PyPy yourself with the option
 ``--shared``. We plan to make ``--shared`` the default in the future. Consult
 the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -35,6 +35,13 @@
 PyPy's bytearray type is very inefficient. It would be an interesting
 task to look into possible optimizations on this.
 
+Implement AF_XXX packet types for PyPy
+--------------------------------------
+
+PyPy is missing AF_XXX types of sockets. Implementing it is easy-to-medium
+task. `bug report`_
+
+.. _`bug report`: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets#more
 
 Implement copy-on-write list slicing
 ------------------------------------
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -52,7 +52,8 @@
 
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
-constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
+if not OPENSSL_NO_SSL3:
+    constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 
@@ -656,7 +657,7 @@
     # set up context
     if protocol == PY_SSL_VERSION_TLS1:
         method = libssl_TLSv1_method()
-    elif protocol == PY_SSL_VERSION_SSL3:
+    elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
         method = libssl_SSLv3_method()
     elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
         method = libssl_SSLv2_method()
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -59,9 +59,9 @@
     _mixin_ = True
 
     def reduce(self, space):
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
+        _numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(_numpypy, MixedModule)
+        multiarray = _numpypy.get("multiarray")
         assert isinstance(multiarray, MixedModule)
         scalar = multiarray.get("scalar")
 
@@ -167,7 +167,7 @@
         if len(args_w) >= 1:
             for w_arg in args_w:
                 try:
-                    idx = support.index_w(space, w_arg)
+                    support.index_w(space, w_arg)
                 except OperationError:
                     raise oefmt(space.w_TypeError, "an integer is required")
             raise oefmt(space.w_ValueError, "axes don't match array")
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -34,8 +34,8 @@
 
 
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
-                        "unegative", "flat", "tostring","count_nonzero",
-                        "argsort"]
+                        "unegative", "flat", "tostring", "count_nonzero",
+                        "argsort", "cumsum", "logical_xor_reduce"]
 TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted']
 TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype']
 THREE_ARG_FUNCTIONS = ['where']
@@ -559,6 +559,11 @@
                 w_res = arr.descr_any(interp.space)
             elif self.name == "all":
                 w_res = arr.descr_all(interp.space)
+            elif self.name == "cumsum":
+                w_res = arr.descr_cumsum(interp.space)
+            elif self.name == "logical_xor_reduce":
+                logical_xor = ufuncs.get(interp.space).logical_xor
+                w_res = logical_xor.reduce(interp.space, arr, None)
             elif self.name == "unegative":
                 neg = ufuncs.get(interp.space).negative
                 w_res = neg.call(interp.space, [arr])
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -286,7 +286,6 @@
     def descr_hash(self, space):
         return space.wrap(self._compute_hash(space, 0x345678))
 
-
     def descr_str(self, space):
         if self.fields:
             return space.str(self.descr_get_descr(space))
diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py
--- a/pypy/module/micronumpy/flatiter.py
+++ b/pypy/module/micronumpy/flatiter.py
@@ -22,6 +22,9 @@
     def get_shape(self):
         return self.shape
 
+    def get_size(self):
+        return self.base().get_size()
+
     def create_iter(self, shape=None, backward_broadcast=False):
         assert isinstance(self.base(), W_NDimArray)
         return self.base().create_iter()
@@ -41,8 +44,8 @@
         return space.wrap(self.state.index)
 
     def descr_coords(self, space):
-        self.state = self.iter.update(self.state)
-        return space.newtuple([space.wrap(c) for c in self.state.indices])
+        coords = self.iter.indices(self.state)
+        return space.newtuple([space.wrap(c) for c in coords])
 
     def descr_iter(self):
         return self
@@ -54,7 +57,7 @@
         if self.iter.done(self.state):
             raise OperationError(space.w_StopIteration, space.w_None)
         w_res = self.iter.getitem(self.state)
-        self.state = self.iter.next(self.state)
+        self.iter.next(self.state, mutate=True)
         return w_res
 
     def descr_getitem(self, space, w_idx):
@@ -71,7 +74,7 @@
                                          base.get_order(), w_instance=base)
             return loop.flatiter_getitem(res, self.iter, state, step)
         finally:
-            self.state = self.iter.reset(self.state)
+            self.iter.reset(self.state, mutate=True)
 
     def descr_setitem(self, space, w_idx, w_value):
         if not (space.isinstance_w(w_idx, space.w_int) or
@@ -91,7 +94,7 @@
             arr = convert_to_array(space, w_value)
             loop.flatiter_setitem(space, dtype, arr, self.iter, state, step, length)
         finally:
-            self.state = self.iter.reset(self.state)
+            self.iter.reset(self.state, mutate=True)
 
 
 W_FlatIterator.typedef = TypeDef("numpy.flatiter",
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -41,16 +41,6 @@
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.flagsobj import _update_contiguous_flags
 
-class OpFlag(object):
-    def __init__(self):
-        self.rw = ''
-        self.broadcast = True
-        self.force_contig = False
-        self.force_align = False
-        self.native_byte_order = False
-        self.tmp_copy = ''
-        self.allocate = False
-
 
 class PureShapeIter(object):
     def __init__(self, shape, idx_w):
@@ -87,26 +77,24 @@
 
 
 class IterState(object):
-    _immutable_fields_ = ['iterator', 'index', 'indices', 'offset']
+    _immutable_fields_ = ['iterator', '_indices']
 
     def __init__(self, iterator, index, indices, offset):
         self.iterator = iterator
         self.index = index
-        self.indices = indices
+        self._indices = indices
         self.offset = offset
 
 
 class ArrayIter(object):
     _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
                           'strides[*]', 'backstrides[*]', 'factors[*]',
-                          'slice_shape', 'slice_stride', 'slice_backstride',
-                          'track_index', 'operand_type', 'slice_operand_type']
+                          'track_index']
 
     track_index = True
 
     @jit.unroll_safe
-    def __init__(self, array, size, shape, strides, backstrides, op_flags=OpFlag()):
-        from pypy.module.micronumpy import concrete
+    def __init__(self, array, size, shape, strides, backstrides):
         assert len(shape) == len(strides) == len(backstrides)
         _update_contiguous_flags(array)
         self.contiguous = (array.flags & NPY.ARRAY_C_CONTIGUOUS and
@@ -118,12 +106,6 @@
         self.shape_m1 = [s - 1 for s in shape]
         self.strides = strides
         self.backstrides = backstrides
-        self.slice_shape = 1
-        self.slice_stride = -1
-        if strides:
-            self.slice_stride = strides[-1]
-        self.slice_backstride = 1
-        self.slice_operand_type = concrete.SliceArray
 
         ndim = len(shape)
         factors = [0] * ndim
@@ -133,32 +115,35 @@
             else:
                 factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i]
         self.factors = factors
-        if op_flags.rw == 'r':
-            self.operand_type = concrete.ConcreteNonWritableArrayWithBase
-        else:
-            self.operand_type = concrete.ConcreteArrayWithBase
 
     @jit.unroll_safe
-    def reset(self, state=None):
+    def reset(self, state=None, mutate=False):
+        index = 0
         if state is None:
             indices = [0] * len(self.shape_m1)
         else:
             assert state.iterator is self
-            indices = state.indices
+            indices = state._indices
             for i in xrange(self.ndim_m1, -1, -1):
                 indices[i] = 0
-        return IterState(self, 0, indices, self.array.start)
+        offset = self.array.start
+        if not mutate:
+            return IterState(self, index, indices, offset)
+        state.index = index
+        state.offset = offset
 
     @jit.unroll_safe
-    def next(self, state):
+    def next(self, state, mutate=False):
         assert state.iterator is self
         index = state.index
         if self.track_index:
             index += 1
-        indices = state.indices
+        indices = state._indices
         offset = state.offset
         if self.contiguous:
             offset += self.array.dtype.elsize
+        elif self.ndim_m1 == 0:
+            offset += self.strides[0]
         else:
             for i in xrange(self.ndim_m1, -1, -1):
                 idx = indices[i]
@@ -169,13 +154,18 @@
                 else:
                     indices[i] = 0
                     offset -= self.backstrides[i]
-        return IterState(self, index, indices, offset)
+        if not mutate:
+            return IterState(self, index, indices, offset)
+        state.index = index
+        state.offset = offset
 
     @jit.unroll_safe
     def goto(self, index):
         offset = self.array.start
         if self.contiguous:
             offset += index * self.array.dtype.elsize
+        elif self.ndim_m1 == 0:
+            offset += index * self.strides[0]
         else:
             current = index
             for i in xrange(len(self.shape_m1)):
@@ -184,20 +174,20 @@
         return IterState(self, index, None, offset)
 
     @jit.unroll_safe
-    def update(self, state):
+    def indices(self, state):
         assert state.iterator is self
         assert self.track_index
-        if not self.contiguous:
-            return state
+        indices = state._indices
+        if not (self.contiguous or self.ndim_m1 == 0):
+            return indices
         current = state.index
-        indices = state.indices
         for i in xrange(len(self.shape_m1)):
             if self.factors[i] != 0:
                 indices[i] = current / self.factors[i]
                 current %= self.factors[i]
             else:
                 indices[i] = 0
-        return IterState(self, state.index, indices, state.offset)
+        return indices
 
     def done(self, state):
         assert state.iterator is self
@@ -216,12 +206,6 @@
         assert state.iterator is self
         self.array.setitem(state.offset, elem)
 
-    def getoperand(self, st, base):
-        impl = self.operand_type
-        res = impl([], self.array.dtype, self.array.order, [], [],
-                   self.array.storage, base)
-        res.start = st.offset
-        return res
 
 def AxisIter(array, shape, axis, cumulative):
     strides = array.get_strides()
@@ -245,42 +229,3 @@
         size /= shape[axis]
     shape[axis] = backstrides[axis] = 0
     return ArrayIter(array, size, shape, array.strides, backstrides)
-
-class SliceIter(ArrayIter):
-    '''
-    used with external loops, getitem and setitem return a SliceArray
-    view into the original array
-    '''
-    _immutable_fields_ = ['base', 'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]']
-
-    def __init__(self, array, size, shape, strides, backstrides, slice_shape,
-                 slice_stride, slice_backstride, op_flags, base):
-        from pypy.module.micronumpy import concrete
-        ArrayIter.__init__(self, array, size, shape, strides, backstrides, op_flags)
-        self.slice_shape = slice_shape
-        self.slice_stride = slice_stride
-        self.slice_backstride = slice_backstride
-        self.base = base
-        if op_flags.rw == 'r':
-            self.slice_operand_type = concrete.NonWritableSliceArray
-        else:
-            self.slice_operand_type = concrete.SliceArray
-
-    def getitem(self, state):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def getitem_bool(self, state):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def setitem(self, state, elem):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def getoperand(self, state, base):
-        assert state.iterator is self
-        impl = self.slice_operand_type
-        arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
-                   [self.slice_shape], self.array, self.base)
-        return arr
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -42,23 +42,38 @@
 
     # TODO handle __array_priorities__ and maybe flip the order
 
+    if w_lhs.get_size() == 1:
+        w_left = w_lhs.get_scalar_value().convert_to(space, calc_dtype)
+        left_iter = left_state = None
+    else:
+        w_left = None
+        left_iter, left_state = w_lhs.create_iter(shape)
+        left_iter.track_index = False
+
+    if w_rhs.get_size() == 1:
+        w_right = w_rhs.get_scalar_value().convert_to(space, calc_dtype)
+        right_iter = right_state = None
+    else:
+        w_right = None
+        right_iter, right_state = w_rhs.create_iter(shape)
+        right_iter.track_index = False
+
     if out is None:
         out = W_NDimArray.from_shape(space, shape, res_dtype,
                                      w_instance=lhs_for_subtype)
-    left_iter, left_state = w_lhs.create_iter(shape)
-    right_iter, right_state = w_rhs.create_iter(shape)
     out_iter, out_state = out.create_iter(shape)
-    left_iter.track_index = right_iter.track_index = False
     shapelen = len(shape)
     while not out_iter.done(out_state):
         call2_driver.jit_merge_point(shapelen=shapelen, func=func,
                                      calc_dtype=calc_dtype, res_dtype=res_dtype)
-        w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
-        w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
+        if left_iter:
+            w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
+            left_state = left_iter.next(left_state)
+        if right_iter:
+            w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
+            right_state = right_iter.next(right_state)
         out_iter.setitem(out_state, func(calc_dtype, w_left, w_right).convert_to(
             space, res_dtype))
-        left_state = left_iter.next(left_state)
-        right_state = right_iter.next(right_state)
         out_state = out_iter.next(out_state)
     return out
 
@@ -68,11 +83,12 @@
     reds='auto')
 
 def call1(space, shape, func, calc_dtype, res_dtype, w_obj, out):
+    obj_iter, obj_state = w_obj.create_iter(shape)
+    obj_iter.track_index = False
+
     if out is None:
         out = W_NDimArray.from_shape(space, shape, res_dtype, w_instance=w_obj)
-    obj_iter, obj_state = w_obj.create_iter(shape)
     out_iter, out_state = out.create_iter(shape)
-    obj_iter.track_index = False
     shapelen = len(shape)
     while not out_iter.done(out_state):
         call1_driver.jit_merge_point(shapelen=shapelen, func=func,
@@ -89,17 +105,14 @@
 
 def setslice(space, shape, target, source):
     if not shape:
-        # XXX - simplify
-        target_iter, target_state = target.create_iter(shape)
-        source_iter, source_state = source.create_iter(shape)
         dtype = target.dtype
-        val = source_iter.getitem(source_state)
+        val = source.getitem(source.start)
         if dtype.is_str_or_unicode():
             val = dtype.coerce(space, val)
         else:
             val = val.convert_to(space, dtype)
-        target_iter.setitem(target_state, val)
-        return target        
+        target.setitem(target.start, val)
+        return target
     return _setslice(space, shape, target, source)
 
 def _setslice(space, shape, target, source):
@@ -107,6 +120,7 @@
     # array implementations, not arrays
     target_iter, target_state = target.create_iter(shape)
     source_iter, source_state = source.create_iter(shape)
+    source_iter.track_index = False
     dtype = target.dtype
     shapelen = len(shape)
     while not target_iter.done(target_state):
@@ -152,6 +166,7 @@
 def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity):
     obj_iter, obj_state = obj.create_iter()
     out_iter, out_state = out.create_iter()
+    out_iter.track_index = False
     if identity is None:
         cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
         out_iter.setitem(out_state, cur_value)
@@ -225,10 +240,9 @@
             state = x_state
     return out
 
-axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce',
-                                    greens=['shapelen',
-                                            'func', 'dtype'],
-                                    reds='auto')
+axis_reduce_driver = jit.JitDriver(name='numpy_axis_reduce',
+                                   greens=['shapelen', 'func', 'dtype'],
+                                   reds='auto')
 
 def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity, cumulative,
                    temp):
@@ -241,21 +255,24 @@
         temp_iter = out_iter  # hack
         temp_state = out_state
     arr_iter, arr_state = arr.create_iter()
+    arr_iter.track_index = False
     if identity is not None:
         identity = identity.convert_to(space, dtype)
     shapelen = len(shape)
     while not out_iter.done(out_state):
-        axis_reduce__driver.jit_merge_point(shapelen=shapelen, func=func,
-                                            dtype=dtype)
-        assert not arr_iter.done(arr_state)
+        axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func,
+                                           dtype=dtype)
         w_val = arr_iter.getitem(arr_state).convert_to(space, dtype)
-        out_state = out_iter.update(out_state)
-        if out_state.indices[axis] == 0:
+        arr_state = arr_iter.next(arr_state)
+
+        out_indices = out_iter.indices(out_state)
+        if out_indices[axis] == 0:
             if identity is not None:
                 w_val = func(dtype, identity, w_val)
         else:
             cur = temp_iter.getitem(temp_state)
             w_val = func(dtype, cur, w_val)
+
         out_iter.setitem(out_state, w_val)
         out_state = out_iter.next(out_state)
         if cumulative:
@@ -263,7 +280,6 @@
             temp_state = temp_iter.next(temp_state)
         else:
             temp_state = out_state
-        arr_state = arr_iter.next(arr_state)
     return out
 
 
@@ -382,9 +398,9 @@
     while not arr_iter.done(arr_state):
         nonzero_driver.jit_merge_point(shapelen=shapelen, dims=dims, dtype=dtype)
         if arr_iter.getitem_bool(arr_state):
-            arr_state = arr_iter.update(arr_state)
+            arr_indices = arr_iter.indices(arr_state)
             for d in dims:
-                res_iter.setitem(res_state, box(arr_state.indices[d]))
+                res_iter.setitem(res_state, box(arr_indices[d]))
                 res_state = res_iter.next(res_state)
         arr_state = arr_iter.next(arr_state)
     return res
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -1139,9 +1139,9 @@
         from pypy.interpreter.mixedmodule import MixedModule
         from pypy.module.micronumpy.concrete import SliceArray
 
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
+        _numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(_numpypy, MixedModule)
+        multiarray = _numpypy.get("multiarray")
         assert isinstance(multiarray, MixedModule)
         reconstruct = multiarray.get("_reconstruct")
         parameters = space.newtuple([self.getclass(space), space.newtuple(
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -6,7 +6,7 @@
 from pypy.module.micronumpy import ufuncs, support, concrete
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.descriptor import decode_w_dtype
-from pypy.module.micronumpy.iterators import ArrayIter, SliceIter, OpFlag
+from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
                                             shape_agreement, shape_agreement_multiple)
 
@@ -36,6 +36,16 @@
     return ret
 
 
+class OpFlag(object):
+    def __init__(self):
+        self.rw = ''
+        self.broadcast = True
+        self.force_contig = False
+        self.force_align = False
+        self.native_byte_order = False
+        self.tmp_copy = ''
+        self.allocate = False
+
 def parse_op_flag(space, lst):
     op_flag = OpFlag()
     for w_item in lst:
@@ -142,11 +152,73 @@
         raise NotImplementedError('not implemented yet')
 
 
-def get_iter(space, order, arr, shape, dtype, op_flags):
+class OperandIter(ArrayIter):
+    _immutable_fields_ = ['slice_shape', 'slice_stride', 'slice_backstride',
+                          'operand_type', 'base']
+
+    def getitem(self, state):
+        # cannot be called - must return a boxed value
+        assert False
+
+    def getitem_bool(self, state):
+        # cannot be called - must return a boxed value
+        assert False
+
+    def setitem(self, state, elem):
+        # cannot be called - must return a boxed value
+        assert False
+
+
+class ConcreteIter(OperandIter):
+    def __init__(self, array, size, shape, strides, backstrides,
+                 op_flags, base):
+        OperandIter.__init__(self, array, size, shape, strides, backstrides)
+        self.slice_shape = 1
+        self.slice_stride = -1
+        if strides:
+            self.slice_stride = strides[-1]
+        self.slice_backstride = 1
+        if op_flags.rw == 'r':
+            self.operand_type = concrete.ConcreteNonWritableArrayWithBase
+        else:
+            self.operand_type = concrete.ConcreteArrayWithBase
+        self.base = base
+
+    def getoperand(self, state):
+        assert state.iterator is self
+        impl = self.operand_type
+        res = impl([], self.array.dtype, self.array.order, [], [],
+                   self.array.storage, self.base)
+        res.start = state.offset
+        return res
+
+
+class SliceIter(OperandIter):
+    def __init__(self, array, size, shape, strides, backstrides, slice_shape,
+                 slice_stride, slice_backstride, op_flags, base):
+        OperandIter.__init__(self, array, size, shape, strides, backstrides)
+        self.slice_shape = slice_shape
+        self.slice_stride = slice_stride
+        self.slice_backstride = slice_backstride
+        if op_flags.rw == 'r':
+            self.operand_type = concrete.NonWritableSliceArray
+        else:
+            self.operand_type = concrete.SliceArray
+        self.base = base
+
+    def getoperand(self, state):
+        assert state.iterator is self
+        impl = self.operand_type
+        arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
+                   [self.slice_shape], self.array, self.base)
+        return arr
+
+
+def get_iter(space, order, arr, shape, dtype, op_flags, base):
     imp = arr.implementation
     backward = is_backward(imp, order)
     if arr.is_scalar():
-        return ArrayIter(imp, 1, [], [], [], op_flags=op_flags)
+        return ConcreteIter(imp, 1, [], [], [], op_flags, base)
     if (imp.strides[0] < imp.strides[-1] and not backward) or \
        (imp.strides[0] > imp.strides[-1] and backward):
         # flip the strides. Is this always true for multidimension?
@@ -161,7 +233,7 @@
         backstrides = imp.backstrides
     r = calculate_broadcast_strides(strides, backstrides, imp.shape,
                                     shape, backward)
-    return ArrayIter(imp, imp.get_size(), shape, r[0], r[1], op_flags=op_flags)
+    return ConcreteIter(imp, imp.get_size(), shape, r[0], r[1], op_flags, base)
 
 def calculate_ndim(op_in, oa_ndim):
     if oa_ndim >=0:
@@ -398,7 +470,7 @@
         self.iters = []
         for i in range(len(self.seq)):
             it = get_iter(space, self.order, self.seq[i], self.shape,
-                          self.dtypes[i], self.op_flags[i])
+                          self.dtypes[i], self.op_flags[i], self)
             it.contiguous = False
             self.iters.append((it, it.reset()))
 
@@ -437,7 +509,7 @@
         return space.wrap(self)
 
     def getitem(self, it, st):
-        res = it.getoperand(st, self)
+        res = it.getoperand(st)
         return W_NDimArray(res)
 
     def descr_getitem(self, space, w_idx):
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -3,13 +3,13 @@
 
 class AppTestNumSupport(BaseNumpyAppTest):
     def test_zeros(self):
-        from numpypy import zeros
+        from numpy import zeros
         a = zeros(3)
         assert len(a) == 3
         assert a[0] == a[1] == a[2] == 0
 
     def test_empty(self):
-        from numpypy import empty
+        from numpy import empty
         import gc
         for i in range(1000):
             a = empty(3)
@@ -26,26 +26,26 @@
                 "empty() returned a zeroed out array every time")
 
     def test_where(self):
-        from numpypy import where, ones, zeros, array
+        from numpy import where, ones, zeros, array
         a = [1, 2, 3, 0, -3]
         a = where(array(a) > 0, ones(5), zeros(5))
         assert (a == [1, 1, 1, 0, 0]).all()
 
     def test_where_differing_dtypes(self):
-        from numpypy import array, ones, zeros, where
+        from numpy import array, ones, zeros, where
         a = [1, 2, 3, 0, -3]
         a = where(array(a) > 0, ones(5, dtype=int), zeros(5, dtype=float))
         assert (a == [1, 1, 1, 0, 0]).all()
 
     def test_where_broadcast(self):
-        from numpypy import array, where
+        from numpy import array, where
         a = where(array([[1, 2, 3], [4, 5, 6]]) > 3, [1, 1, 1], 2)
         assert (a == [[2, 2, 2], [1, 1, 1]]).all()
         a = where(True, [1, 1, 1], 2)
         assert (a == [1, 1, 1]).all()
 
     def test_where_errors(self):
-        from numpypy import where, array
+        from numpy import where, array
         raises(ValueError, "where([1, 2, 3], [3, 4, 5])")
         raises(ValueError, "where([1, 2, 3], [3, 4, 5], [6, 7])")
         assert where(True, 1, 2) == array(1)
@@ -58,14 +58,14 @@
     #    xxx
 
     def test_where_invalidates(self):
-        from numpypy import where, ones, zeros, array
+        from numpy import where, ones, zeros, array
         a = array([1, 2, 3, 0, -3])
         b = where(a > 0, ones(5), zeros(5))
         a[0] = 0
         assert (b == [1, 1, 1, 0, 0]).all()
 
     def test_dot_basic(self):
-        from numpypy import array, dot, arange
+        from numpy import array, dot, arange
         a = array(range(5))
         assert dot(a, a) == 30.0
 
@@ -100,7 +100,7 @@
         assert (dot([[1,2],[3,4]],[5,6]) == [17, 39]).all()
 
     def test_dot_constant(self):
-        from numpypy import array, dot
+        from numpy import array, dot
         a = array(range(5))
         b = a.dot(2.5)
         for i in xrange(5):
@@ -111,7 +111,7 @@
         assert c == 12.0
 
     def test_dot_out(self):
-        from numpypy import arange, dot
+        from numpy import arange, dot
         a = arange(12).reshape(3, 4)
         b = arange(12).reshape(4, 3)
         out = arange(9).reshape(3, 3)
@@ -124,19 +124,19 @@
                                 'right type, nr dimensions, and be a C-Array)')
 
     def test_choose_basic(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), array([4, 5, 6]), array([7, 8, 9])
         r = array([2, 1, 0]).choose([a, b, c])
         assert (r == [7, 5, 3]).all()
 
     def test_choose_broadcast(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         r = array([2, 1, 0]).choose([a, b, c])
         assert (r == [13, 5, 3]).all()
 
     def test_choose_out(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         r = array([2, 1, 0]).choose([a, b, c], out=None)
         assert (r == [13, 5, 3]).all()
@@ -146,7 +146,7 @@
         assert (a == [13, 5, 3]).all()
 
     def test_choose_modes(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         raises(ValueError, "array([3, 1, 0]).choose([a, b, c])")
         raises(ValueError, "array([3, 1, 0]).choose([a, b, c], mode='raises')")
@@ -158,20 +158,20 @@
         assert (r == [4, 5, 3]).all()
 
     def test_choose_dtype(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1.2, 2, 3]), [4, 5, 6], 13
         r = array([2, 1, 0]).choose([a, b, c])
         assert r.dtype == float
 
     def test_choose_dtype_out(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         x = array([0, 0, 0], dtype='i2')
         r = array([2, 1, 0]).choose([a, b, c], out=x)
         assert r.dtype == 'i2'
 
     def test_put_basic(self):
-        from numpypy import arange, array
+        from numpy import arange, array
         a = arange(5)
         a.put([0, 2], [-44, -55])
         assert (a == array([-44, 1, -55, 3, 4])).all()
@@ -183,7 +183,7 @@
         assert (a == array([0, 7, 2, 3, 4])).all()
 
     def test_put_modes(self):
-        from numpypy import array, arange
+        from numpy import array, arange
         a = arange(5)
         a.put(22, -5, mode='clip')
         assert (a == array([0, 1, 2, 3, -5])).all()
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -14,15 +14,12 @@
             else:
                 from . import dummy_module as numpy
                 sys.modules['numpy'] = numpy
-            sys.modules['numpypy'] = numpy
         else:
             import os
             path = os.path.dirname(__file__) + '/dummy_module.py'
             cls.space.appexec([cls.space.wrap(path)], """(path):
             import imp
-            numpy = imp.load_source('numpy', path)
-            import sys
-            sys.modules['numpypy'] = numpy
+            imp.load_source('numpy', path)
             """)
         cls.w_non_native_prefix = cls.space.wrap(NPY.OPPBYTE)
         cls.w_native_prefix = cls.space.wrap(NPY.NATBYTE)
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -132,13 +132,13 @@
             cls.w_c_pow = cls.space.wrap(interp2app(cls_c_pow))
 
     def test_fabs(self):
-        from numpypy import fabs, dtype
+        from numpy import fabs, dtype
 
         a = dtype('complex128').type(complex(-5., 5.))
         raises(TypeError, fabs, a)
 
     def test_fmax(self):
-        from numpypy import fmax, array
+        from numpy import fmax, array
         nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
         a = array((complex(ninf, 10), complex(10, ninf),
                    complex( inf, 10), complex(10,  inf),
@@ -165,7 +165,7 @@
         assert (fmax(a, b) == res).all()
 
     def test_fmin(self):
-        from numpypy import fmin, array
+        from numpy import fmin, array
         nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
         a = array((complex(ninf, 10), complex(10, ninf),
                    complex( inf, 10), complex(10,  inf),
@@ -192,11 +192,11 @@
         assert (fmin(a, b) == res).all()
 
     def test_signbit(self):
-        from numpypy import signbit
+        from numpy import signbit
         raises(TypeError, signbit, complex(1,1))
 
     def test_reciprocal(self):
-        from numpypy import array, reciprocal
+        from numpy import array, reciprocal
         inf = float('inf')
         nan = float('nan')
         #complex
@@ -218,21 +218,21 @@
                 assert (a[0].imag - e.imag) < rel_err
 
     def test_floorceiltrunc(self):
-        from numpypy import array, floor, ceil, trunc
+        from numpy import array, floor, ceil, trunc
         a = array([ complex(-1.4, -1.4), complex(-1.5, -1.5)])
         raises(TypeError, floor, a)
         raises(TypeError, ceil, a)
         raises(TypeError, trunc, a)
 
     def test_copysign(self):
-        from numpypy import copysign, dtype
+        from numpy import copysign, dtype
         complex128 = dtype('complex128').type
         a = complex128(complex(-5., 5.))
         b = complex128(complex(0., 0.))
         raises(TypeError, copysign, a, b)
 
     def test_exp2(self):
-        from numpypy import array, exp2
+        from numpy import array, exp2
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
@@ -268,7 +268,7 @@
 
     def test_expm1(self):
         import math, cmath
-        from numpypy import array, expm1
+        from numpy import array, expm1
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
@@ -307,7 +307,7 @@
                 self.rAlmostEqual(t1, t2, rel_err=rel_err, msg=msg)
 
     def test_not_complex(self):
-        from numpypy import (radians, deg2rad, degrees, rad2deg,
+        from numpy import (radians, deg2rad, degrees, rad2deg,
                   logaddexp, logaddexp2, fmod,
                   arctan2)
         raises(TypeError, radians, complex(90,90))
@@ -320,7 +320,7 @@
         raises (TypeError, fmod, complex(90,90), 3)
 
     def test_isnan_isinf(self):
-        from numpypy import isnan, isinf, array
+        from numpy import isnan, isinf, array
         assert (isnan(array([0.2+2j, complex(float('inf'),0),
                 complex(0,float('inf')), complex(0,float('nan')),
                 complex(float('nan'), 0)], dtype=complex)) == \
@@ -333,7 +333,7 @@
 
 
     def test_square(self):
-        from numpypy import square
+        from numpy import square
         assert square(complex(3, 4)) == complex(3,4) * complex(3, 4)
 
     def test_power_simple(self):
@@ -364,8 +364,8 @@
             self.rAlmostEqual(float(n_r_a[i].imag), float(p_r[i].imag), msg=msg)
 
     def test_conjugate(self):
-        from numpypy import conj, conjugate, dtype
-        import numpypy as np
+        from numpy import conj, conjugate, dtype
+        import numpy as np
 
         c0 = dtype('complex128').type(complex(2.5, 0))
         c1 = dtype('complex64').type(complex(1, 2))
@@ -390,7 +390,7 @@
     def test_logn(self):
         import math, cmath
         # log and log10 are tested in math (1:1 from rcomplex)
-        from numpypy import log2, array, log1p
+        from numpy import log2, array, log1p
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
@@ -447,7 +447,7 @@
                 self.rAlmostEqual(t1, t2, rel_err=rel_err, msg=msg)
 
     def test_logical_ops(self):
-        from numpypy import logical_and, logical_or, logical_xor, logical_not
+        from numpy import logical_and, logical_or, logical_xor, logical_not
 
         c1 = complex(1, 1)
         c3 = complex(3, 0)
@@ -461,7 +461,7 @@
         assert (logical_not([c1, c0]) == [False, True]).all()
 
     def test_minimum(self):
-        from numpypy import array, minimum
+        from numpy import array, minimum
 
         a = array([-5.0+5j, -5.0-5j, -0.0-10j, 1.0+10j])
         b = array([ 3.0+10.0j, 3.0, -2.0+2.0j, -3.0+4.0j])
@@ -470,7 +470,7 @@
             assert c[i] == min(a[i], b[i])
 
     def test_maximum(self):
-        from numpypy import array, maximum
+        from numpy import array, maximum
 
         a = array([-5.0+5j, -5.0-5j, -0.0-10j, 1.0+10j])
         b = array([ 3.0+10.0j, 3.0, -2.0+2.0j, -3.0+4.0j])
@@ -480,10 +480,10 @@
 
     def test_basic(self):
         import sys
-        from numpypy import (dtype, add, array, dtype,
+        from numpy import (dtype, add, array, dtype,
             subtract as sub, multiply, divide, negative, absolute as abs,
             floor_divide, real, imag, sign)
-        from numpypy import (equal, not_equal, greater, greater_equal, less,
+        from numpy import (equal, not_equal, greater, greater_equal, less,
                 less_equal, isnan)
         assert real(4.0) == 4.0
         assert imag(0.0) == 0.0
@@ -579,7 +579,7 @@
             assert repr(abs(inf_c)) == 'inf'
             assert repr(abs(complex(float('nan'), float('nan')))) == 'nan'
             # numpy actually raises an AttributeError,
-            # but numpypy raises a TypeError
+            # but numpy.raises a TypeError
             if '__pypy__' in sys.builtin_module_names:
                 exct, excm = TypeError, 'readonly attribute'
             else:
@@ -592,7 +592,7 @@
             assert(imag(c2) == 4.0)
 
     def test_conj(self):
-        from numpypy import array
+        from numpy import array
 
         a = array([1 + 2j, 1 - 2j])
         assert (a.conj() == [1 - 2j, 1 + 2j]).all()
@@ -603,7 +603,7 @@
         if self.isWindows:
             skip('windows does not support c99 complex')
         import sys
-        import numpypy as np
+        import numpy as np
         rAlmostEqual = self.rAlmostEqual
 
         for t, testcases in (
@@ -658,6 +658,6 @@
             sys.stderr.write('\n')
 
     def test_complexbox_to_pycomplex(self):
-        from numpypy import dtype
+        from numpy import dtype
         x = dtype('complex128').type(3.4j)
         assert complex(x) == 3.4j
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -23,7 +23,7 @@
             from numpy.core.multiarray import typeinfo
         except ImportError:
             # running on dummy module
-            from numpypy import typeinfo
+            from numpy import typeinfo
         assert typeinfo['Number'] == np.number
         assert typeinfo['LONGLONG'] == ('q', 9, 64, 8, 9223372036854775807L,
                                         -9223372036854775808L, np.longlong)
@@ -39,7 +39,7 @@
                                     np.dtype('int').type)
 
     def test_dtype_basic(self):
-        from numpypy import dtype
+        from numpy import dtype
         import sys
 
         d = dtype('?')
@@ -104,7 +104,7 @@
         assert d.shape == ()
 
     def test_dtype_eq(self):
-        from numpypy import dtype
+        from numpy import dtype
 
         assert dtype("int8") == "int8"
         assert "int8" == dtype("int8")
@@ -112,7 +112,7 @@
         assert dtype(bool) == bool
 
     def test_dtype_aliases(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype('bool8') is dtype('bool')
         assert dtype('byte') is dtype('int8')
         assert dtype('ubyte') is dtype('uint8')
@@ -132,7 +132,7 @@
         assert dtype('clongdouble').num in (15, 16)
 
     def test_dtype_with_types(self):
-        from numpypy import dtype
+        from numpy import dtype
 
         assert dtype(bool).num == 0
         if self.ptr_size == 4:
@@ -156,7 +156,7 @@
         assert dtype('complex').num == 15
 
     def test_array_dtype_attr(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
 
         a = array(range(5), long)
         assert a.dtype is dtype(long)
@@ -182,7 +182,7 @@
         assert np.dtype('S5').isbuiltin == 0
 
     def test_repr_str(self):
-        from numpypy import dtype
+        from numpy import dtype
         b = dtype(int).newbyteorder().newbyteorder().byteorder
         assert '.dtype' in repr(dtype)
         d = dtype('?')
@@ -208,7 +208,7 @@
         assert str(d) == "|V16"
 
     def test_bool_array(self):
-        from numpypy import array, False_, True_
+        from numpy import array, False_, True_
 
         a = array([0, 1, 2, 2.5], dtype='?')
         assert a[0] is False_
@@ -216,7 +216,7 @@
             assert a[i] is True_
 
     def test_copy_array_with_dtype(self):
-        from numpypy import array, longlong, False_
+        from numpy import array, longlong, False_
 
         a = array([0, 1, 2, 3], dtype=long)
         # int on 64-bit, long in 32-bit
@@ -230,35 +230,35 @@
         assert b[0] is False_
 
     def test_zeros_bool(self):
-        from numpypy import zeros, False_
+        from numpy import zeros, False_
 
         a = zeros(10, dtype=bool)
         for i in range(10):
             assert a[i] is False_
 
     def test_ones_bool(self):
-        from numpypy import ones, True_
+        from numpy import ones, True_
 
         a = ones(10, dtype=bool)
         for i in range(10):
             assert a[i] is True_
 
     def test_zeros_long(self):
-        from numpypy import zeros, longlong
+        from numpy import zeros, longlong
         a = zeros(10, dtype=long)
         for i in range(10):
             assert isinstance(a[i], longlong)
             assert a[1] == 0
 
     def test_ones_long(self):
-        from numpypy import ones, longlong
+        from numpy import ones, longlong
         a = ones(10, dtype=long)
         for i in range(10):
             assert isinstance(a[i], longlong)
             assert a[1] == 1
 
     def test_overflow(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         assert array([128], 'b')[0] == -128
         assert array([256], 'B')[0] == 0
         assert array([32768], 'h')[0] == -32768
@@ -273,7 +273,7 @@
         raises(OverflowError, "array([2**64], 'Q')")
 
     def test_bool_binop_types(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         types = [
             '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd',
             'e'
@@ -285,7 +285,7 @@
             assert (a + array([0], t)).dtype is dtype(t)
 
     def test_binop_types(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
                  ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
                  ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
@@ -309,7 +309,7 @@
             assert (d1, d2) == (d1, d2) and d3 is dtype(dout)
 
     def test_add(self):
-        import numpypy as np
+        import numpy as np
         for dtype in ["int8", "int16", "I"]:
             a = np.array(range(5), dtype=dtype)
             b = a + a
@@ -325,22 +325,22 @@
         assert len(d) == 2
 
     def test_shape(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype(long).shape == ()
 
     def test_cant_subclass(self):
-        from numpypy import dtype
+        from numpy import dtype
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
     def test_can_subclass(self):
-        import numpypy
-        class xyz(numpypy.void):
+        import numpy
+        class xyz(numpy.void):
             pass
         assert True
 
     def test_index(self):
-        import numpypy as np
+        import numpy as np
         for dtype in [np.int8, np.int16, np.int32, np.int64]:
             a = np.array(range(10), dtype=dtype)
             b = np.array([0] * 10, dtype=dtype)
@@ -348,7 +348,7 @@
                 a[idx] += 1
 
     def test_hash(self):
-        import numpypy as numpy
+        import numpy
         for tp, value in [
             (numpy.int8, 4),
             (numpy.int16, 5),
@@ -392,9 +392,10 @@
         t5 = dtype([('x', '<f4'), ('y', '<i4')])
         t6 = dtype([('y', '<i4'), ('x', '<f4')])
         assert hash(t5) != hash(t6)
+
     def test_pickle(self):
         import numpy as np
-        from numpypy import array, dtype
+        from numpy import array, dtype
         from cPickle import loads, dumps
         a = array([1,2,3])
         if self.ptr_size == 8:
@@ -407,7 +408,7 @@
         assert np.dtype(('<f8', 2)).__reduce__() == (dtype, ('V16', 0, 1), (3, '|', (dtype('float64'), (2,)), None, None, 16, 1, 0))
 
     def test_newbyteorder(self):
-        import numpypy as np
+        import numpy as np
         import sys
         sys_is_le = sys.byteorder == 'little'
         native_code = sys_is_le and '<' or '>'
@@ -479,7 +480,7 @@
 
 class AppTestTypes(BaseAppTestDtypes):
     def test_abstract_types(self):
-        import numpypy as numpy
+        import numpy
 
         raises(TypeError, numpy.generic, 0)
         raises(TypeError, numpy.number, 0)
@@ -525,16 +526,16 @@
         #assert a.dtype is numpy.dtype('|V4')
 
     def test_new(self):
-        import numpypy as np
+        import numpy as np
         assert np.int_(4) == 4
         assert np.float_(3.4) == 3.4
 
     def test_pow(self):
-        from numpypy import int_
+        from numpy import int_
         assert int_(4) ** 2 == 16
 
     def test_bool(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.bool_.mro() == [numpy.bool_, numpy.generic, object]
         assert numpy.bool_(3) is numpy.True_
@@ -549,7 +550,7 @@
         assert numpy.bool_("False") is numpy.True_
 
     def test_int8(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int8.mro() == [numpy.int8, numpy.signedinteger,
                                     numpy.integer, numpy.number,
@@ -573,7 +574,7 @@
         assert numpy.int8('128') == -128
 
     def test_uint8(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.uint8.mro() == [numpy.uint8, numpy.unsignedinteger,
                                      numpy.integer, numpy.number,
@@ -598,7 +599,7 @@
         assert numpy.uint8('256') == 0
 
     def test_int16(self):
-        import numpypy as numpy
+        import numpy
 
         x = numpy.int16(3)
         assert x == 3
@@ -640,7 +641,7 @@
             assert numpy.uint32('4294967296') == 0
 
     def test_int_(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int_ is numpy.dtype(int).type
         assert numpy.int_.mro() == [numpy.int_, numpy.signedinteger,
@@ -751,7 +752,7 @@
 
     def test_complex_format(self):
         import sys
-        import numpypy as numpy
+        import numpy
 
         for complex_ in (numpy.complex128, numpy.complex64,):
             for real, imag, should in [
@@ -791,7 +792,7 @@
             assert "{:g}".format(numpy.complex_(0.5+1.5j)) == '{:g}'.format(0.5+1.5j)
 
     def test_complex(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.complex_ is numpy.complex128
         assert numpy.csingle is numpy.complex64
@@ -811,7 +812,7 @@
         assert d.char == 'F'
 
     def test_subclass_type(self):
-        import numpypy as numpy
+        import numpy
 
         class X(numpy.float64):
             def m(self):
@@ -825,12 +826,12 @@
         assert b.dtype.type is numpy.float64
 
     def test_long_as_index(self):
-        from numpypy import int_, float64
+        from numpy import int_, float64
         assert (1, 2, 3)[int_(1)] == 2
         raises(TypeError, lambda: (1, 2, 3)[float64(1)])
 
     def test_int(self):
-        from numpypy import int32, int64, int_
+        from numpy import int32, int64, int_
         import sys
         assert issubclass(int_, int)
         if sys.maxint == (1<<31) - 1:
@@ -841,7 +842,7 @@
             assert int_ is int64
 
     def test_various_types(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int16 is numpy.short
         assert numpy.int8 is numpy.byte
@@ -863,7 +864,7 @@
         assert not issubclass(numpy.longfloat, numpy.float64)
 
     def test_mro(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int16.__mro__ == (numpy.int16, numpy.signedinteger,
                                        numpy.integer, numpy.number,
@@ -872,7 +873,7 @@
 
     def test_operators(self):
         from operator import truediv
-        from numpypy import float64, int_, True_, False_
+        from numpy import float64, int_, True_, False_
         assert 5 / int_(2) == int_(2)
         assert truediv(int_(3), int_(2)) == float64(1.5)
         assert truediv(3, int_(2)) == float64(1.5)
@@ -898,7 +899,7 @@
 
     def test_alternate_constructs(self):
         import numpy as np
-        from numpypy import dtype
+        from numpy import dtype
         nnp = self.non_native_prefix
         byteorder = self.native_prefix
         assert dtype('i8') == dtype(byteorder + 'i8') == dtype('=i8') == dtype(long)
@@ -921,7 +922,7 @@
         assert d.str == '|S0'
 
     def test_dtype_str(self):
-        from numpypy import dtype
+        from numpy import dtype
         byteorder = self.native_prefix
         assert dtype('i8').str == byteorder + 'i8'
         assert dtype('<i8').str == '<i8'
@@ -948,7 +949,7 @@
         assert dtype(('f8', 2)).str == "|V16"
 
     def test_intp(self):
-        from numpypy import dtype
+        from numpy import dtype
         for s in ['p', 'int']:
             assert dtype(s) is dtype('intp')
         for s in ['P', 'uint']:
@@ -967,16 +968,16 @@
             assert dtype('P').name == 'uint64'
 
     def test_alignment(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype('i4').alignment == 4
 
     def test_isnative(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype('i4').isnative == True
         assert dtype('>i8').isnative == False
 
     def test_any_all_nonzero(self):
-        import numpypy as numpy
+        import numpy
         x = numpy.bool_(True)
         assert x.any() is numpy.True_
         assert x.all() is numpy.True_
@@ -999,7 +1000,7 @@
         assert x.__nonzero__() is True
 
     def test_ravel(self):
-        from numpypy import float64, int8, array
+        from numpy import float64, int8, array
         x = float64(42.5).ravel()
         assert x.dtype == float64
         assert (x == array([42.5])).all()
@@ -1022,7 +1023,7 @@
 
 class AppTestStrUnicodeDtypes(BaseNumpyAppTest):
     def test_mro(self):
-        from numpypy import str_, unicode_, character, flexible, generic
+        from numpy import str_, unicode_, character, flexible, generic
         import sys
         if '__pypy__' in sys.builtin_module_names:
             assert str_.mro() == [str_, character, flexible, generic,
@@ -1036,7 +1037,7 @@
                                       flexible, generic, object]
 
     def test_str_dtype(self):
-        from numpypy import dtype, str_
+        from numpy import dtype, str_
 
         raises(TypeError, "dtype('Sx')")
         for t in ['S8', '|S8', '=S8']:
@@ -1057,7 +1058,7 @@
             assert d.str == '|S1'
 
     def test_unicode_dtype(self):
-        from numpypy import dtype, unicode_
+        from numpy import dtype, unicode_
 
         raises(TypeError, "dtype('Ux')")
         d = dtype('U8')
@@ -1069,11 +1070,11 @@
         assert d.num == 19
 
     def test_string_boxes(self):
-        from numpypy import str_
+        from numpy import str_
         assert isinstance(str_(3), str_)
 
     def test_unicode_boxes(self):
-        from numpypy import unicode_
+        from numpy import unicode_
         import sys
         if '__pypy__' in sys.builtin_module_names:
             exc = raises(NotImplementedError, unicode_, 3)
@@ -1084,7 +1085,7 @@
 
     def test_character_dtype(self):
         import numpy as np
-        from numpypy import array, character
+        from numpy import array, character
         x = array([["A", "B"], ["C", "D"]], character)
         assert (x == [["A", "B"], ["C", "D"]]).all()
         d = np.dtype('c')
@@ -1097,7 +1098,7 @@
     spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
 
     def test_create(self):
-        from numpypy import dtype, void
+        from numpy import dtype, void
 
         raises(ValueError, "dtype([('x', int), ('x', float)])")
         d = dtype([("x", "<i4"), ("y", "<f4"), ("z", "<u2"), ("v", "<f8")])
@@ -1156,7 +1157,7 @@
             raises(NotImplementedError, np.dtype, d)
 
     def test_create_subarrays(self):
-        from numpypy import dtype
+        from numpy import dtype
         d = dtype([("x", "float", (2,)), ("y", "int64", (2,))])
         assert d.itemsize == 32
         assert d.name == "void256"
@@ -1277,7 +1278,7 @@
         assert repr(d) == "dtype([('f0', '<f8'), ('f1', '<f8')])"
 
     def test_pickle_record(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         from cPickle import loads, dumps
 
         d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)])
@@ -1288,7 +1289,7 @@
         assert new_d.__reduce__() == d.__reduce__()
 
     def test_pickle_record_subarrays(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         from cPickle import loads, dumps
 
         d = dtype([("x", "int32", (3,)), ("y", "int32", (2,)), ("z", "int32", (4,)), ("value", float, (5,))])
@@ -1320,7 +1321,7 @@
             cls.w_check_non_native = cls.space.wrap(interp2app(check_non_native))
 
     def test_non_native(self):
-        from numpypy import array
+        from numpy import array
         a = array([1, 2, 3], dtype=self.non_native_prefix + 'i2')
         assert a[0] == 1
         assert (a + a)[1] == 4
@@ -1343,7 +1344,7 @@
 
 class AppTestObjectDtypes(BaseNumpyAppTest):
     def test_scalar_from_object(self):
-        from numpypy import array
+        from numpy import array
         import sys
         class Polynomial(object):
             pass
diff --git a/pypy/module/micronumpy/test/test_iterators.py b/pypy/module/micronumpy/test/test_iterators.py
--- a/pypy/module/micronumpy/test/test_iterators.py
+++ b/pypy/module/micronumpy/test/test_iterators.py
@@ -31,16 +31,14 @@
         s = i.next(s)
         assert s.offset == 3
         assert not i.done(s)
-        assert s.indices == [0,0]
-        s = i.update(s)
-        assert s.indices == [0,3]
+        assert s._indices == [0,0]
+        assert i.indices(s) == [0,3]
         #cause a dimension overflow
         s = i.next(s)
         s = i.next(s)
         assert s.offset == 5
-        assert s.indices == [0,3]
-        s = i.update(s)
-        assert s.indices == [1,0]
+        assert s._indices == [0,3]
+        assert i.indices(s) == [1,0]
 
         #Now what happens if the array is transposed? strides[-1] != 1
         # therefore layout is non-contiguous
@@ -56,12 +54,12 @@
         s = i.next(s)
         assert s.offset == 9
         assert not i.done(s)
-        assert s.indices == [0,3]
+        assert s._indices == [0,3]
         #cause a dimension overflow
         s = i.next(s)
         s = i.next(s)
         assert s.offset == 1
-        assert s.indices == [1,0]
+        assert s._indices == [1,0]
 
     def test_iterator_goto(self):
         shape = [3, 5]
@@ -74,9 +72,9 @@
         assert not i.contiguous
         s = i.reset()
         assert s.index == 0
-        assert s.indices == [0, 0]
+        assert s._indices == [0, 0]
         assert s.offset == a.start
         s = i.goto(11)
         assert s.index == 11
-        assert s.indices is None
+        assert s._indices is None
         assert s.offset == a.start + 5
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -269,13 +269,13 @@
                 assert not a.flags['F']
 
     def test_ndmin(self):
-        from numpypy import array
+        from numpy import array
 
         arr = array([[[1]]], ndmin=1)
         assert arr.shape == (1, 1, 1)
 
     def test_noop_ndmin(self):
-        from numpypy import array
+        from numpy import array
         arr = array([1], ndmin=3)
         assert arr.shape == (1, 1, 1)
 
@@ -289,7 +289,7 @@
         assert a.dtype == np.intp
 
     def test_array_copy(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(12)).reshape(3,4)
         b = array(a, ndmin=4)
         assert b.shape == (1, 1, 3, 4)
@@ -353,12 +353,12 @@
         assert str(exc.value) == 'buffer is read-only'
 
     def test_type(self):
-        from numpypy import array
+        from numpy import array
         ar = array(range(5))
         assert type(ar) is type(ar + ar)
 
     def test_ndim(self):
-        from numpypy import array
+        from numpy import array
         x = array(0.2)
         assert x.ndim == 0
         x = array([1, 2])
@@ -367,12 +367,12 @@
         assert x.ndim == 2
         x = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
         assert x.ndim == 3
-        # numpy actually raises an AttributeError, but numpypy raises an
+        # numpy actually raises an AttributeError, but numpy.raises an
         # TypeError
         raises((TypeError, AttributeError), 'x.ndim = 3')
 
     def test_zeros(self):
-        from numpypy import zeros
+        from numpy import zeros
         a = zeros(15)
         # Check that storage was actually zero'd.
         assert a[10] == 0.0
@@ -431,7 +431,7 @@
         assert type(b) is np.ndarray
 
     def test_size(self):
-        from numpypy import array,arange,cos
+        from numpy import array,arange,cos
         assert array(3).size == 1
         a = array([1, 2, 3])
         assert a.size == 3
@@ -440,13 +440,13 @@
         assert ten == 10
 
     def test_empty(self):
-        from numpypy import empty
+        from numpy import empty
         a = empty(2)
         a[1] = 1.0
         assert a[1] == 1.0
 
     def test_ones(self):
-        from numpypy import ones, dtype
+        from numpy import ones, dtype
         a = ones(3)
         assert len(a) == 3
         assert a[0] == 1
@@ -458,7 +458,7 @@
         assert b.dtype is dtype(complex)
 
     def test_arange(self):
-        from numpypy import arange, dtype
+        from numpy import arange, dtype
         a = arange(3)
         assert (a == [0, 1, 2]).all()
         assert a.dtype is dtype(int)
@@ -480,7 +480,7 @@
         assert arange(False, True, True).dtype is dtype(int)
 
     def test_copy(self):
-        from numpypy import arange, array
+        from numpy import arange, array
         a = arange(5)
         b = a.copy()
         for i in xrange(5):
@@ -522,7 +522,7 @@
             raises(NotImplementedError, a.copy, order=True)
 
     def test_iterator_init(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert a[3] == 3
 
@@ -546,7 +546,7 @@
         assert (a == [[1], [2]]).all()
 
     def test_getitem(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         raises(IndexError, "a[5]")
         a = a + a
@@ -555,14 +555,14 @@
         raises(IndexError, "a[-6]")
 
     def test_getitem_float(self):
-        from numpypy import array
+        from numpy import array
         a = array([1, 2, 3, 4])
         assert a[1.2] == 2
         assert a[1.6] == 2
         assert a[-1.2] == 4
 
     def test_getitem_tuple(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         raises(IndexError, "a[(1,2)]")
         for i in xrange(5):
@@ -572,28 +572,28 @@
             assert a[i] == b[i]
 
     def test_getitem_nd(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(15).reshape(3, 5)
         assert a[1, 3] == 8
         assert a.T[1, 2] == 11
 
     def test_getitem_obj_index(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         assert a[self.CustomIndexObject(1)] == 1
 
     def test_getitem_obj_prefer_index_to_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         assert a[self.CustomIndexIntObject(0, 1)] == 0
 
     def test_getitem_obj_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         assert a[self.CustomIntObject(1)] == 1
 
     def test_setitem(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         a[-1] = 5.0
         assert a[4] == 5.0
@@ -612,7 +612,7 @@
         assert a[2] == -0.005
 
     def test_setitem_tuple(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         raises(IndexError, "a[(1,2)] = [0,1]")
         for i in xrange(5):
@@ -630,25 +630,25 @@
         assert a[2] == 100
 
     def test_setitem_obj_index(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         a[self.CustomIndexObject(1)] = 100
         assert a[1] == 100
 
     def test_setitem_obj_prefer_index_to_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         a[self.CustomIndexIntObject(0, 1)] = 100
         assert a[0] == 100
 
     def test_setitem_obj_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         a[self.CustomIntObject(1)] = 100
         assert a[1] == 100
 
     def test_delitem(self):
-        import numpypy as np
+        import numpy as np
         a = np.arange(10)
         exc = raises(ValueError, 'del a[2]')
         assert exc.value.message == 'cannot delete array elements'
@@ -665,7 +665,7 @@
         # numpy will swallow errors in __int__ and __index__ and
         # just raise IndexError.
 
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         exc = raises(IndexError, "a[ErrorIndex()] == 0")
         assert exc.value.message == 'cannot convert index to integer'
@@ -673,7 +673,7 @@
         assert exc.value.message == 'cannot convert index to integer'
 
     def test_setslice_array(self):
-        from numpypy import array
+        from numpy import array
         a = array(5)
         exc = raises(ValueError, "a[:] = 4")
         assert exc.value[0] == "cannot slice a 0-d array"
@@ -687,7 +687,7 @@
         assert b[1] == 0.
 
     def test_setslice_of_slice_array(self):
-        from numpypy import array, zeros
+        from numpy import array, zeros
         a = zeros(5)
         a[::2] = array([9., 10., 11.])
         assert a[0] == 9.
@@ -706,7 +706,7 @@
         assert a[0] == 3.
 
     def test_setslice_list(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5), float)
         b = [0., 1.]
         a[1:4:2] = b
@@ -714,7 +714,7 @@
         assert a[3] == 1.
 
     def test_setslice_constant(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5), float)
         a[1:4:2] = 0.
         assert a[1] == 0.
@@ -722,7 +722,7 @@
 
     def test_newaxis(self):
         import math
-        from numpypy import array, cos, zeros, newaxis
+        from numpy import array, cos, zeros, newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
@@ -743,7 +743,7 @@
         assert o == 3
 
     def test_newaxis_slice(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
 
         a = array(range(5))
         b = array(range(1,5))
@@ -755,14 +755,14 @@
         assert (a[newaxis,1:] == c).all()
 
     def test_newaxis_assign(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
 
         a = array(range(5))
         a[newaxis,1] = [2]
         assert a[1] == 2
 
     def test_newaxis_virtual(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
 
         a = array(range(5))
         b = (a + a)[newaxis]
@@ -770,20 +770,20 @@
         assert (b == c).all()
 
     def test_newaxis_then_slice(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
         a = array(range(5))
         b = a[newaxis]
         assert b.shape == (1, 5)
         assert (b[0,1:] == a[1:]).all()
 
     def test_slice_then_newaxis(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
         a = array(range(5))
         b = a[2:]
         assert (b[newaxis] == [[2, 3, 4]]).all()
 
     def test_scalar(self):
-        from numpypy import array, dtype, int_
+        from numpy import array, dtype, int_
         a = array(3)
         exc = raises(IndexError, "a[0]")
         assert exc.value[0] == "0-d arrays can't be indexed"
@@ -816,13 +816,13 @@
         assert a == 2.5
 
     def test_len(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert len(a) == 5
         assert len(a + a) == 5
 
     def test_shape(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert a.shape == (5,)
         b = a + a
@@ -832,7 +832,7 @@
         assert array([]).shape == (0,)
 
     def test_set_shape(self):
-        from numpypy import array, zeros
+        from numpy import array, zeros
         a = array([])
         raises(ValueError, "a.shape = []")
         a = array(range(12))
@@ -855,7 +855,7 @@
         raises(AttributeError, 'a.shape = 6')
 
     def test_reshape(self):
-        from numpypy import array, zeros
+        from numpy import array, zeros
         for a in [array(1), array([1])]:
             for s in [(), (1,)]:
                 b = a.reshape(s)
@@ -902,7 +902,7 @@
             raises(NotImplementedError, a.reshape, (0,), order='F')
 
     def test_slice_reshape(self):
-        from numpypy import zeros, arange
+        from numpy import zeros, arange
         a = zeros((4, 2, 3))
         b = a[::2, :, :]
         b.shape = (2, 6)
@@ -938,32 +938,32 @@
         raises(ValueError, arange(10).reshape, (5, -1, -1))
 
     def test_reshape_varargs(self):
-        from numpypy import arange
+        from numpy import arange
         z = arange(96).reshape(12, -1)
         y = z.reshape(4, 3, 8)
         assert y.shape == (4, 3, 8)
 
     def test_scalar_reshape(self):
-        from numpypy import array
+        from numpy import array
         a = array(3)
         assert a.reshape([1, 1]).shape == (1, 1)
         assert a.reshape([1]).shape == (1,)
         raises(ValueError, "a.reshape(3)")
 
     def test_strides(self):
-        from numpypy import array
+        from numpy import array
         a = array([[1.0, 2.0],
                    [3.0, 4.0]])
         assert a.strides == (16, 8)
         assert a[1:].strides == (16, 8)
 
     def test_strides_scalar(self):
-        from numpypy import array
+        from numpy import array
         a = array(42)
         assert a.strides == ()

From noreply at buildbot.pypy.org  Tue Dec  9 17:29:45 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Tue,  9 Dec 2014 17:29:45 +0100 (CET)
Subject: [pypy-commit] pypy recursion_and_inlining: Use framestack instead
 of portal_trace_positions.
Message-ID: <20141209162945.923751C09B2@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: recursion_and_inlining
Changeset: r74863:fe3efdc5abfa
Date: 2014-12-08 15:22 +0000
http://bitbucket.org/pypy/pypy/changeset/fe3efdc5abfa/

Log:	Use framestack instead of portal_trace_positions.

	The latter does not, despite first appearences, model the frame
	stack: it models all call positions the portal has gone through in
	its history. If I'd looked more carefully, I might have noticed that
	the portal has a semi-hidden framestack attribute, which has a semi-
	hidden greenkey attribute. This records exactly what we want, and
	also solves the problem that we're no longer tied to being the main
	jitcode.

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -958,27 +958,29 @@
                 # loop.
                 portal_code = targetjitdriver_sd.mainjitcode
                 inline = True
-                if self.metainterp.is_main_jitcode(portal_code):
-                    count = 0
-                    for gk, _ in self.metainterp.portal_trace_positions:
-                        if gk is None:
-                            continue
-                        assert len(gk) == len(greenboxes)
-                        i = 0
-                        for i in range(len(gk)):
-                            if not gk[i].same_constant(greenboxes[i]):
-                                break
-                        else:
-                            count += 1
-                    memmgr = self.metainterp.staticdata.warmrunnerdesc.memory_manager
-                    if count >= memmgr.max_unroll_recursion:
-                        # This function is recursive and has exceeded the
-                        # maximum number of unrollings we allow. We want to stop
-                        # inlining it further and to make sure that, if it
-                        # hasn't happened already, the function is traced
-                        # separately as soon as possible.
-                        warmrunnerstate.dont_trace_here(greenboxes)
-                        inline = False
+                count = 0
+                for f in self.metainterp.framestack:
+                    if f.jitcode is not portal_code:
+                        continue
+                    gk = f.greenkey
+                    if gk is None:
+                        continue
+                    assert len(gk) == len(greenboxes)
+                    i = 0
+                    for i in range(len(gk)):
+                        if not gk[i].same_constant(greenboxes[i]):
+                            break
+                    else:
+                        count += 1
+                memmgr = self.metainterp.staticdata.warmrunnerdesc.memory_manager
+                if count >= memmgr.max_unroll_recursion:
+                    # This function is recursive and has exceeded the
+                    # maximum number of unrollings we allow. We want to stop
+                    # inlining it further and to make sure that, if it
+                    # hasn't happened already, the function is traced
+                    # separately as soon as possible.
+                    warmrunnerstate.dont_trace_here(greenboxes)
+                    inline = False
                 if inline:
                     return self.metainterp.perform_call(portal_code, allboxes,
                                 greenkey=greenboxes)

From noreply at buildbot.pypy.org  Tue Dec  9 17:29:46 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Tue,  9 Dec 2014 17:29:46 +0100 (CET)
Subject: [pypy-commit] pypy recursion_and_inlining: We no longer need a
 separate inline variable.
Message-ID: <20141209162946.C30F61C09B2@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: recursion_and_inlining
Changeset: r74864:b02aa3253678
Date: 2014-12-08 15:39 +0000
http://bitbucket.org/pypy/pypy/changeset/b02aa3253678/

Log:	We no longer need a separate inline variable.

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -957,7 +957,6 @@
                 # recursion, which would be equivalent to unrolling a while
                 # loop.
                 portal_code = targetjitdriver_sd.mainjitcode
-                inline = True
                 count = 0
                 for f in self.metainterp.framestack:
                     if f.jitcode is not portal_code:
@@ -980,8 +979,7 @@
                     # hasn't happened already, the function is traced
                     # separately as soon as possible.
                     warmrunnerstate.dont_trace_here(greenboxes)
-                    inline = False
-                if inline:
+                else:
                     return self.metainterp.perform_call(portal_code, allboxes,
                                 greenkey=greenboxes)
             assembler_call = True

From noreply at buildbot.pypy.org  Tue Dec  9 17:29:47 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Tue,  9 Dec 2014 17:29:47 +0100 (CET)
Subject: [pypy-commit] pypy recursion_and_inlining: Test that recursing past
 a specified threshold turns off recursion inlining.
Message-ID: <20141209162947.F23691C09B2@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: recursion_and_inlining
Changeset: r74865:64b3af5150aa
Date: 2014-12-09 15:44 +0000
http://bitbucket.org/pypy/pypy/changeset/64b3af5150aa/

Log:	Test that recursing past a specified threshold turns off recursion
	inlining.

diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -1112,6 +1112,37 @@
         assert res == 2095
         self.check_resops(call_assembler=12)
 
+    def test_inline_recursion_limit(self):
+        driver = JitDriver(greens = ["threshold", "loop"], reds=["i"])
+        @dont_look_inside
+        def f():
+            set_param(driver, "max_unroll_recursion", 10)
+        def portal(threshold, loop, i):
+            f()
+            if i > threshold:
+                return i
+            while True:
+                driver.jit_merge_point(threshold=threshold, loop=loop, i=i)
+                if loop:
+                    portal(threshold, False, 0)
+                else:
+                    portal(threshold, False, i + 1)
+                    return i
+                if i > 10:
+                    return 1
+                i += 1
+                driver.can_enter_jit(threshold=threshold, loop=loop, i=i)
+
+        res1 = portal(10, True, 0)
+        res2 = self.meta_interp(portal, [10, True, 0], inline=True)
+        assert res1 == res2
+        self.check_resops(call_assembler=2)
+
+        res1 = portal(9, True, 0)
+        res2 = self.meta_interp(portal, [9, True, 0], inline=True)
+        assert res1 == res2
+        self.check_resops(call_assembler=0)
+
     def test_handle_jitexception_in_portal(self):
         # a test for _handle_jitexception_in_portal in blackhole.py
         driver = JitDriver(greens = ['codeno'], reds = ['i', 'str'],

From noreply at buildbot.pypy.org  Tue Dec  9 17:29:49 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Tue,  9 Dec 2014 17:29:49 +0100 (CET)
Subject: [pypy-commit] pypy recursion_and_inlining: Add output to JIT
 logging informing people of when recursive functions have stopped being
 inlined.
Message-ID: <20141209162949.3311E1C09B2@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: recursion_and_inlining
Changeset: r74866:8b9a6b6f9c9d
Date: 2014-12-09 15:44 +0000
http://bitbucket.org/pypy/pypy/changeset/8b9a6b6f9c9d/

Log:	Add output to JIT logging informing people of when recursive
	functions have stopped being inlined.

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -978,6 +978,9 @@
                     # inlining it further and to make sure that, if it
                     # hasn't happened already, the function is traced
                     # separately as soon as possible.
+                    if have_debug_prints():
+                        loc = targetjitdriver_sd.warmstate.get_location_str(greenboxes)
+                        debug_print("recursive function (not inlined):", loc)
                     warmrunnerstate.dont_trace_here(greenboxes)
                 else:
                     return self.metainterp.perform_call(portal_code, allboxes,

From noreply at buildbot.pypy.org  Tue Dec  9 17:29:55 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Tue,  9 Dec 2014 17:29:55 +0100 (CET)
Subject: [pypy-commit] pypy recursion_and_inlining: Merge default.
Message-ID: <20141209162955.7959C1C09B2@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: recursion_and_inlining
Changeset: r74867:fd5a11ac1b71
Date: 2014-12-09 15:48 +0000
http://bitbucket.org/pypy/pypy/changeset/fd5a11ac1b71/

Log:	Merge default.

diff too long, truncating to 2000 out of 18633 lines

diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -59,7 +59,7 @@
     def __init__(self, basename, core=False, compiler=None, usemodules='',
                  skip=None):
         self.basename = basename
-        self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket']
+        self._usemodules = usemodules.split() + ['signal', 'time', 'itertools', '_socket']
         self._compiler = compiler
         self.core = core
         self.skip = skip
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -29,7 +29,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -40,7 +40,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
     "cStringIO", "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -64,19 +64,15 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
diff --git a/pypy/doc/config/objspace.usemodules.rctime.txt b/pypy/doc/config/objspace.usemodules.rctime.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules.rctime.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Use the 'rctime' module. 
-
-'rctime' is our `rffi`_ based implementation of the builtin 'time' module.
-It supersedes the less complete :config:`objspace.usemodules.time`,
-at least for C-like targets (the C and LLVM backends).
-
-.. _`rffi`: ../rffi.html
diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt
--- a/pypy/doc/config/objspace.usemodules.time.txt
+++ b/pypy/doc/config/objspace.usemodules.time.txt
@@ -1,5 +1,1 @@
 Use the 'time' module. 
-
-Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version
-of the application-level 'time' module, at least for C-like targets (the C
-and LLVM backends).
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -205,23 +205,28 @@
 The above is true both in CPython and in PyPy.  Differences
 can occur about whether a built-in function or method will
 call an overridden method of *another* object than ``self``.
-In PyPy, they are generally always called, whereas not in
-CPython.  For example, in PyPy, ``dict1.update(dict2)``
-considers that ``dict2`` is just a general mapping object, and
-will thus call overridden ``keys()``  and ``__getitem__()``
-methods on it.  So the following code prints ``42`` on PyPy
-but ``foo`` on CPython::
+In PyPy, they are often called in cases where CPython would not.
+Two examples::
 
-    >>>> class D(dict):
-    ....     def __getitem__(self, key):
-    ....         return 42
-    ....
-    >>>>
-    >>>> d1 = {}
-    >>>> d2 = D(a='foo')
-    >>>> d1.update(d2)
-    >>>> print d1['a']
-    42
+    class D(dict):
+        def __getitem__(self, key):
+            return "%r from D" % (key,)
+
+    class A(object):
+        pass
+
+    a = A()
+    a.__dict__ = D()
+    a.foo = "a's own foo"
+    print a.foo
+    # CPython => a's own foo
+    # PyPy => 'foo' from D
+
+    glob = D(foo="base item")
+    loc = {}
+    exec "print foo" in glob, loc
+    # CPython => base item
+    # PyPy => 'foo' from D
 
 
 Mutating classes of objects which are already used as dictionary keys
@@ -292,6 +297,9 @@
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
+Notably missing from the list above are ``str`` and ``unicode``.  If your
+code relies on comparing strings with ``is``, then it might break in PyPy.
+
 
 Miscellaneous
 -------------
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,6 +6,10 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
+**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
+on linux, linux64 and windows. We will make it the default on all platforms
+by the time of the next release.
+
 The first thing that you need is to compile PyPy yourself with the option
 ``--shared``. We plan to make ``--shared`` the default in the future. Consult
 the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -35,6 +35,13 @@
 PyPy's bytearray type is very inefficient. It would be an interesting
 task to look into possible optimizations on this.
 
+Implement AF_XXX packet types for PyPy
+--------------------------------------
+
+PyPy is missing AF_XXX types of sockets. Implementing it is easy-to-medium
+task. `bug report`_
+
+.. _`bug report`: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets#more
 
 Implement copy-on-write list slicing
 ------------------------------------
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -43,3 +43,11 @@
 .. branch nditer-external_loop
 
 Implement `external_loop` arguement to numpy's nditer
+
+.. branch kill-rctime
+
+Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
+
+.. branch: ssa-flow
+
+Use SSA form for flow graphs inside build_flow() and part of simplify_graph()
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,15 +1,13 @@
-"""
-Python control flow graph generation and bytecode assembly.
-"""
+"""Python control flow graph generation and bytecode assembly."""
 
-from pypy.interpreter.astcompiler import ast, symtable
-from pypy.interpreter import pycode
+from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import we_are_translated
+
+from pypy.interpreter.astcompiler import ast, misc, symtable
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pycode import PyCode
 from pypy.tool import stdlib_opcode as ops
 
-from pypy.interpreter.error import OperationError
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import rfloat
-
 
 class Instruction(object):
     """Represents a single opcode."""
@@ -21,14 +19,12 @@
         self.has_jump = False
 
     def size(self):
-        """Return the size of bytes of this instruction when it is encoded."""
+        """Return the size of bytes of this instruction when it is
+        encoded.
+        """
         if self.opcode >= ops.HAVE_ARGUMENT:
-            if self.arg > 0xFFFF:
-                return 6
-            else:
-                return 3
-        else:
-            return 1
+            return (6 if self.arg > 0xFFFF else 3)
+        return 1
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -54,9 +50,9 @@
 class Block(object):
     """A basic control flow block.
 
-    It has one entry point and several possible exit points.  Its instructions
-    may be jumps to other blocks, or if control flow reaches the end of the
-    block, it continues to next_block.
+    It has one entry point and several possible exit points.  Its
+    instructions may be jumps to other blocks, or if control flow
+    reaches the end of the block, it continues to next_block.
     """
 
     def __init__(self):
@@ -71,10 +67,10 @@
             stack.append(nextblock)
 
     def post_order(self):
-        """Return this block and its children in post order.
-        This means that the graph of blocks is first cleaned up to
-        ignore back-edges, thus turning it into a DAG.  Then the DAG
-        is linearized.  For example:
+        """Return this block and its children in post order.  This means
+        that the graph of blocks is first cleaned up to ignore
+        back-edges, thus turning it into a DAG.  Then the DAG is
+        linearized.  For example:
 
                    A --> B -\           =>     [A, D, B, C]
                      \-> D ---> C
@@ -105,7 +101,9 @@
         return resultblocks
 
     def code_size(self):
-        """Return the encoded size of all the instructions in this block."""
+        """Return the encoded size of all the instructions in this
+        block.
+        """
         i = 0
         for instr in self.instructions:
             i += instr.size()
@@ -141,6 +139,7 @@
             i += 1
     return result
 
+
 def _list_to_dict(l, offset=0):
     result = {}
     index = offset
@@ -300,11 +299,11 @@
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
         last_extended_arg_count = 0
-        # The reason for this loop is extended jumps.  EXTENDED_ARG extends the
-        # bytecode size, so it might invalidate the offsets we've already given.
-        # Thus we have to loop until the number of extended args is stable.  Any
-        # extended jump at all is extremely rare, so performance is not too
-        # concerning.
+        # The reason for this loop is extended jumps.  EXTENDED_ARG
+        # extends the bytecode size, so it might invalidate the offsets
+        # we've already given.  Thus we have to loop until the number of
+        # extended args is stable.  Any extended jump at all is
+        # extremely rare, so performance is not too concerning.
         while True:
             extended_arg_count = 0
             offset = 0
@@ -330,7 +329,8 @@
                                     instr.opcode = ops.JUMP_ABSOLUTE
                                     absolute = True
                                 elif target_op == ops.RETURN_VALUE:
-                                    # Replace JUMP_* to a RETURN into just a RETURN
+                                    # Replace JUMP_* to a RETURN into
+                                    # just a RETURN
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
@@ -345,7 +345,8 @@
                         instr.arg = jump_arg
                         if jump_arg > 0xFFFF:
                             extended_arg_count += 1
-            if extended_arg_count == last_extended_arg_count and not force_redo:
+            if (extended_arg_count == last_extended_arg_count and
+                not force_redo):
                 break
             else:
                 last_extended_arg_count = extended_arg_count
@@ -360,12 +361,14 @@
         while True:
             try:
                 w_key = space.next(w_iter)
-            except OperationError, e:
+            except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise
                 break
             w_index = space.getitem(w_consts, w_key)
-            consts_w[space.int_w(w_index)] = space.getitem(w_key, first)
+            w_constant = space.getitem(w_key, first)
+            w_constant = misc.intern_if_common_string(space, w_constant)
+            consts_w[space.int_w(w_index)] = w_constant
         return consts_w
 
     def _get_code_flags(self):
@@ -433,15 +436,16 @@
                         continue
                     addr = offset - current_off
                     # Python assumes that lineno always increases with
-                    # increasing bytecode address (lnotab is unsigned char).
-                    # Depending on when SET_LINENO instructions are emitted this
-                    # is not always true.  Consider the code:
+                    # increasing bytecode address (lnotab is unsigned
+                    # char).  Depending on when SET_LINENO instructions
+                    # are emitted this is not always true.  Consider the
+                    # code:
                     #     a = (1,
                     #          b)
-                    # In the bytecode stream, the assignment to "a" occurs after
-                    # the loading of "b".  This works with the C Python compiler
-                    # because it only generates a SET_LINENO instruction for the
-                    # assignment.
+                    # In the bytecode stream, the assignment to "a"
+                    # occurs after the loading of "b".  This works with
+                    # the C Python compiler because it only generates a
+                    # SET_LINENO instruction for the assignment.
                     if line or addr:
                         while addr > 255:
                             push(chr(255))
@@ -484,22 +488,22 @@
         free_names = _list_from_dict(self.free_vars, len(cell_names))
         flags = self._get_code_flags() | self.compile_info.flags
         bytecode = ''.join([block.get_code() for block in blocks])
-        return pycode.PyCode(self.space,
-                             self.argcount,
-                             len(self.var_names),
-                             stack_depth,
-                             flags,
-                             bytecode,
-                             list(consts_w),
-                             names,
-                             var_names,
-                             self.compile_info.filename,
-                             self.name,
-                             self.first_lineno,
-                             lnotab,
-                             free_names,
-                             cell_names,
-                             self.compile_info.hidden_applevel)
+        return PyCode(self.space,
+                      self.argcount,
+                      len(self.var_names),
+                      stack_depth,
+                      flags,
+                      bytecode,
+                      list(consts_w),
+                      names,
+                      var_names,
+                      self.compile_info.filename,
+                      self.name,
+                      self.first_lineno,
+                      lnotab,
+                      free_names,
+                      cell_names,
+                      self.compile_info.hidden_applevel)
 
 
 def _list_from_dict(d, offset=0):
@@ -510,134 +514,134 @@
 
 
 _static_opcode_stack_effects = {
-    ops.NOP : 0,
-    ops.STOP_CODE : 0,
+    ops.NOP: 0,
+    ops.STOP_CODE: 0,
 
-    ops.POP_TOP : -1,
-    ops.ROT_TWO : 0,
-    ops.ROT_THREE : 0,
-    ops.ROT_FOUR : 0,
-    ops.DUP_TOP : 1,
+    ops.POP_TOP: -1,
+    ops.ROT_TWO: 0,
+    ops.ROT_THREE: 0,
+    ops.ROT_FOUR: 0,
+    ops.DUP_TOP: 1,
 
-    ops.UNARY_POSITIVE : 0,
-    ops.UNARY_NEGATIVE : 0,
-    ops.UNARY_NOT : 0,
-    ops.UNARY_CONVERT : 0,
-    ops.UNARY_INVERT : 0,
+    ops.UNARY_POSITIVE: 0,
+    ops.UNARY_NEGATIVE: 0,
+    ops.UNARY_NOT: 0,
+    ops.UNARY_CONVERT: 0,
+    ops.UNARY_INVERT: 0,
 
-    ops.LIST_APPEND : -1,
-    ops.SET_ADD : -1,
-    ops.MAP_ADD : -2,
-    ops.STORE_MAP : -2,
+    ops.LIST_APPEND: -1,
+    ops.SET_ADD: -1,
+    ops.MAP_ADD: -2,
+    ops.STORE_MAP: -2,
 
-    ops.BINARY_POWER : -1,
-    ops.BINARY_MULTIPLY : -1,
-    ops.BINARY_DIVIDE : -1,
-    ops.BINARY_MODULO : -1,
-    ops.BINARY_ADD : -1,
-    ops.BINARY_SUBTRACT : -1,
-    ops.BINARY_SUBSCR : -1,
-    ops.BINARY_FLOOR_DIVIDE : -1,
-    ops.BINARY_TRUE_DIVIDE : -1,
-    ops.BINARY_LSHIFT : -1,
-    ops.BINARY_RSHIFT : -1,
-    ops.BINARY_AND : -1,
-    ops.BINARY_OR : -1,
-    ops.BINARY_XOR : -1,
+    ops.BINARY_POWER: -1,
+    ops.BINARY_MULTIPLY: -1,
+    ops.BINARY_DIVIDE: -1,
+    ops.BINARY_MODULO: -1,
+    ops.BINARY_ADD: -1,
+    ops.BINARY_SUBTRACT: -1,
+    ops.BINARY_SUBSCR: -1,
+    ops.BINARY_FLOOR_DIVIDE: -1,
+    ops.BINARY_TRUE_DIVIDE: -1,
+    ops.BINARY_LSHIFT: -1,
+    ops.BINARY_RSHIFT: -1,
+    ops.BINARY_AND: -1,
+    ops.BINARY_OR: -1,
+    ops.BINARY_XOR: -1,
 
-    ops.INPLACE_FLOOR_DIVIDE : -1,
-    ops.INPLACE_TRUE_DIVIDE : -1,
-    ops.INPLACE_ADD : -1,
-    ops.INPLACE_SUBTRACT : -1,
-    ops.INPLACE_MULTIPLY : -1,
-    ops.INPLACE_DIVIDE : -1,
-    ops.INPLACE_MODULO : -1,
-    ops.INPLACE_POWER : -1,
-    ops.INPLACE_LSHIFT : -1,
-    ops.INPLACE_RSHIFT : -1,
-    ops.INPLACE_AND : -1,
-    ops.INPLACE_OR : -1,
-    ops.INPLACE_XOR : -1,
+    ops.INPLACE_FLOOR_DIVIDE: -1,
+    ops.INPLACE_TRUE_DIVIDE: -1,
+    ops.INPLACE_ADD: -1,
+    ops.INPLACE_SUBTRACT: -1,
+    ops.INPLACE_MULTIPLY: -1,
+    ops.INPLACE_DIVIDE: -1,
+    ops.INPLACE_MODULO: -1,
+    ops.INPLACE_POWER: -1,
+    ops.INPLACE_LSHIFT: -1,
+    ops.INPLACE_RSHIFT: -1,
+    ops.INPLACE_AND: -1,
+    ops.INPLACE_OR: -1,
+    ops.INPLACE_XOR: -1,
 
-    ops.SLICE+0 : 1,
-    ops.SLICE+1 : 0,
-    ops.SLICE+2 : 0,
-    ops.SLICE+3 : -1,
-    ops.STORE_SLICE+0 : -2,
-    ops.STORE_SLICE+1 : -3,
-    ops.STORE_SLICE+2 : -3,
-    ops.STORE_SLICE+3 : -4,
-    ops.DELETE_SLICE+0 : -1,
-    ops.DELETE_SLICE+1 : -2,
-    ops.DELETE_SLICE+2 : -2,
-    ops.DELETE_SLICE+3 : -3,
+    ops.SLICE+0: 1,
+    ops.SLICE+1: 0,
+    ops.SLICE+2: 0,
+    ops.SLICE+3: -1,
+    ops.STORE_SLICE+0: -2,
+    ops.STORE_SLICE+1: -3,
+    ops.STORE_SLICE+2: -3,
+    ops.STORE_SLICE+3: -4,
+    ops.DELETE_SLICE+0: -1,
+    ops.DELETE_SLICE+1: -2,
+    ops.DELETE_SLICE+2: -2,
+    ops.DELETE_SLICE+3: -3,
 
-    ops.STORE_SUBSCR : -2,
-    ops.DELETE_SUBSCR : -2,
+    ops.STORE_SUBSCR: -2,
+    ops.DELETE_SUBSCR: -2,
 
-    ops.GET_ITER : 0,
-    ops.FOR_ITER : 1,
-    ops.BREAK_LOOP : 0,
-    ops.CONTINUE_LOOP : 0,
-    ops.SETUP_LOOP : 0,
+    ops.GET_ITER: 0,
+    ops.FOR_ITER: 1,
+    ops.BREAK_LOOP: 0,
+    ops.CONTINUE_LOOP: 0,
+    ops.SETUP_LOOP: 0,
 
-    ops.PRINT_EXPR : -1,
-    ops.PRINT_ITEM : -1,
-    ops.PRINT_NEWLINE : 0,
-    ops.PRINT_ITEM_TO : -2,
-    ops.PRINT_NEWLINE_TO : -1,
+    ops.PRINT_EXPR: -1,
+    ops.PRINT_ITEM: -1,
+    ops.PRINT_NEWLINE: 0,
+    ops.PRINT_ITEM_TO: -2,
+    ops.PRINT_NEWLINE_TO: -1,
 
-    ops.WITH_CLEANUP : -1,
-    ops.POP_BLOCK : 0,
-    ops.END_FINALLY : -1,
-    ops.SETUP_WITH : 1,
-    ops.SETUP_FINALLY : 0,
-    ops.SETUP_EXCEPT : 0,
+    ops.WITH_CLEANUP: -1,
+    ops.POP_BLOCK: 0,
+    ops.END_FINALLY: -1,
+    ops.SETUP_WITH: 1,
+    ops.SETUP_FINALLY: 0,
+    ops.SETUP_EXCEPT: 0,
 
-    ops.LOAD_LOCALS : 1,
-    ops.RETURN_VALUE : -1,
-    ops.EXEC_STMT : -3,
-    ops.YIELD_VALUE : 0,
-    ops.BUILD_CLASS : -2,
-    ops.BUILD_MAP : 1,
-    ops.BUILD_SET : 1,
-    ops.COMPARE_OP : -1,
+    ops.LOAD_LOCALS: 1,
+    ops.RETURN_VALUE: -1,
+    ops.EXEC_STMT: -3,
+    ops.YIELD_VALUE: 0,
+    ops.BUILD_CLASS: -2,
+    ops.BUILD_MAP: 1,
+    ops.BUILD_SET: 1,
+    ops.COMPARE_OP: -1,
 
-    ops.LOOKUP_METHOD : 1,
+    ops.LOOKUP_METHOD: 1,
 
-    ops.LOAD_NAME : 1,
-    ops.STORE_NAME : -1,
-    ops.DELETE_NAME : 0,
+    ops.LOAD_NAME: 1,
+    ops.STORE_NAME: -1,
+    ops.DELETE_NAME: 0,
 
-    ops.LOAD_FAST : 1,
-    ops.STORE_FAST : -1,
-    ops.DELETE_FAST : 0,
+    ops.LOAD_FAST: 1,
+    ops.STORE_FAST: -1,
+    ops.DELETE_FAST: 0,
 
-    ops.LOAD_ATTR : 0,
-    ops.STORE_ATTR : -2,
-    ops.DELETE_ATTR : -1,
+    ops.LOAD_ATTR: 0,
+    ops.STORE_ATTR: -2,
+    ops.DELETE_ATTR: -1,
 
-    ops.LOAD_GLOBAL : 1,
-    ops.STORE_GLOBAL : -1,
-    ops.DELETE_GLOBAL : 0,
+    ops.LOAD_GLOBAL: 1,
+    ops.STORE_GLOBAL: -1,
+    ops.DELETE_GLOBAL: 0,
 
-    ops.LOAD_CLOSURE : 1,
-    ops.LOAD_DEREF : 1,
-    ops.STORE_DEREF : -1,
+    ops.LOAD_CLOSURE: 1,
+    ops.LOAD_DEREF: 1,
+    ops.STORE_DEREF: -1,
 
-    ops.LOAD_CONST : 1,
+    ops.LOAD_CONST: 1,
 
-    ops.IMPORT_STAR : -1,
-    ops.IMPORT_NAME : -1,
-    ops.IMPORT_FROM : 1,
+    ops.IMPORT_STAR: -1,
+    ops.IMPORT_NAME: -1,
+    ops.IMPORT_FROM: 1,
 
-    ops.JUMP_FORWARD : 0,
-    ops.JUMP_ABSOLUTE : 0,
-    ops.JUMP_IF_TRUE_OR_POP : 0,
-    ops.JUMP_IF_FALSE_OR_POP : 0,
-    ops.POP_JUMP_IF_TRUE : -1,
-    ops.POP_JUMP_IF_FALSE : -1,
-    ops.JUMP_IF_NOT_DEBUG : 0,
+    ops.JUMP_FORWARD: 0,
+    ops.JUMP_ABSOLUTE: 0,
+    ops.JUMP_IF_TRUE_OR_POP: 0,
+    ops.JUMP_IF_FALSE_OR_POP: 0,
+    ops.POP_JUMP_IF_TRUE: -1,
+    ops.POP_JUMP_IF_FALSE: -1,
+    ops.JUMP_IF_NOT_DEBUG: 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -106,3 +106,13 @@
     except IndexError:
         return name
     return "_%s%s" % (klass[i:], name)
+
+
+def intern_if_common_string(space, w_const):
+    # only intern identifier-like strings
+    if not space.is_w(space.type(w_const), space.w_str):
+        return w_const
+    for c in space.str_w(w_const):
+        if not (c.isalnum() or c == '_'):
+            return w_const
+    return space.new_interned_w_str(w_const)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -272,6 +272,11 @@
                 if w_const is None:
                     return tup
                 consts_w[i] = w_const
+            # intern the string constants packed into the tuple here,
+            # because assemble.py will see the result as just a tuple constant
+            for i in range(len(consts_w)):
+                consts_w[i] = misc.intern_if_common_string(
+                    self.space, consts_w[i])
         else:
             consts_w = []
         w_consts = self.space.newtuple(consts_w)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -14,7 +14,7 @@
     UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
-from pypy.interpreter.miscutils import ThreadLocals
+from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
 
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
@@ -384,7 +384,7 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        self.interned_strings = {}
+        self.interned_strings = make_weak_value_dictionary(self, str, W_Root)
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
@@ -522,11 +522,6 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: rctime take precedence over time
-        # XXX this could probably be done as a "requires" in the config
-        if 'rctime' in modules and 'time' in modules:
-            modules.remove('time')
-
         self._builtinmodule_list = modules
         return self._builtinmodule_list
 
@@ -782,25 +777,30 @@
             return self.w_False
 
     def new_interned_w_str(self, w_s):
+        assert isinstance(w_s, W_Root)   # and is not None
         s = self.str_w(w_s)
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        self.interned_strings[s] = w_s
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = w_s
+            self.interned_strings.set(s, w_s1)
+        return w_s1
 
     def new_interned_str(self, s):
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        w_s = self.interned_strings[s] = self.wrap(s)
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = self.wrap(s)
+            self.interned_strings.set(s, w_s1)
+        return w_s1
+
+    def is_interned_str(self, s):
+        # interface for marshal_impl
+        if not we_are_translated():
+            assert type(s) is str
+        return self.interned_strings.get(s) is not None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -31,3 +31,19 @@
 
     def getallvalues(self):
         return {0: self._value}
+
+
+def make_weak_value_dictionary(space, keytype, valuetype):
+    "NOT_RPYTHON"
+    if space.config.translation.rweakref:
+        from rpython.rlib.rweakref import RWeakValueDictionary
+        return RWeakValueDictionary(keytype, valuetype)
+    else:
+        class FakeWeakValueDict(object):
+            def __init__(self):
+                self._dict = {}
+            def get(self, key):
+                return self._dict.get(key, None)
+            def set(self, key, value):
+                self._dict[key] = value
+        return FakeWeakValueDict()
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -131,7 +131,6 @@
         # class bodies only have CO_NEWLOCALS.
         # CO_NEWLOCALS: make a locals dict unless optimized is also set
         # CO_OPTIMIZED: no locals dict needed at all
-        # NB: this method is overridden in nestedscope.py
         flags = code.co_flags
         if not (flags & pycode.CO_OPTIMIZED):
             if flags & pycode.CO_NEWLOCALS:
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -970,7 +970,12 @@
             sys.stdout = out
         output = s.getvalue()
         assert "CALL_METHOD" in output
-            
+
+    def test_interned_strings(self):
+        source = """x = ('foo_bar42', 5); y = 'foo_bar42'; z = x[0]"""
+        exec source
+        assert y is z
+
 
 class AppTestExceptions:
     def test_indentation_error(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -378,3 +378,41 @@
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+
+    def test_interned_strings_are_weak(self):
+        import weakref, gc, random
+        space = self.space
+        assert space.config.translation.rweakref
+        w1 = space.new_interned_str("abcdef")
+        w2 = space.new_interned_str("abcdef")
+        assert w2 is w1
+        #
+        # check that 'w1' goes away if we don't hold a reference to it
+        rw1 = weakref.ref(w1)
+        del w1, w2
+        i = 10
+        while rw1() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
+        #
+        s = "foobar%r" % random.random()
+        w0 = space.wrap(s)
+        w1 = space.new_interned_w_str(w0)
+        assert w1 is w0
+        w2 = space.new_interned_w_str(w0)
+        assert w2 is w0
+        w3 = space.wrap(s)
+        assert w3 is not w0
+        w4 = space.new_interned_w_str(w3)
+        assert w4 is w0
+        #
+        # check that 'w0' goes away if we don't hold a reference to it
+        # (even if we hold a reference to 'w3')
+        rw0 = weakref.ref(w0)
+        del w0, w1, w2, w4
+        i = 10
+        while rw0() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -158,21 +158,14 @@
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
-    _attrs_            = ['value_fits_long', 'vmin', 'vrangemax']
-    _immutable_fields_ = ['value_fits_long', 'vmin', 'vrangemax']
+    _attrs_            = ['value_fits_long', 'value_smaller_than_long']
+    _immutable_fields_ = ['value_fits_long', 'value_smaller_than_long']
     is_primitive_integer = True
 
     def __init__(self, *args):
         W_CTypePrimitive.__init__(self, *args)
         self.value_fits_long = self.size <= rffi.sizeof(lltype.Signed)
-        if self.size < rffi.sizeof(lltype.Signed):
-            assert self.value_fits_long
-            sh = self.size * 8
-            self.vmin = r_uint(-1) << (sh - 1)
-            self.vrangemax = (r_uint(1) << sh) - 1
-        else:
-            self.vmin = r_uint(0)
-            self.vrangemax = r_uint(-1)
+        self.value_smaller_than_long = self.size < rffi.sizeof(lltype.Signed)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -192,8 +185,17 @@
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
             value = misc.as_long(self.space, w_ob)
-            if self.size < rffi.sizeof(lltype.Signed):
-                if r_uint(value) - self.vmin > self.vrangemax:
+            if self.value_smaller_than_long:
+                size = self.size
+                if size == 1:
+                    signextended = misc.signext(value, 1)
+                elif size == 2:
+                    signextended = misc.signext(value, 2)
+                elif size == 4:
+                    signextended = misc.signext(value, 4)
+                else:
+                    raise AssertionError("unsupported size")
+                if value != signextended:
                     self._overflow(w_ob)
             misc.write_raw_signed_data(cdata, value, self.size)
         else:
@@ -221,7 +223,7 @@
             length = w_cdata.get_array_length()
             populate_list_from_raw_array(res, buf, length)
             return res
-        elif self.value_fits_long:
+        elif self.value_smaller_than_long:
             res = [0] * w_cdata.get_array_length()
             misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
             return res
@@ -235,8 +237,8 @@
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
-                overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                overflowed = misc.pack_list_to_raw_array_bounds_signed(
+                    int_list, cdata, self.size)
                 if overflowed != 0:
                     self._overflow(self.space.wrap(overflowed))
             return True
@@ -314,8 +316,8 @@
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
-            overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
+                int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.wrap(overflowed))
             return True
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -216,6 +216,19 @@
 neg_msg = "can't convert negative number to unsigned"
 ovf_msg = "long too big to convert"
 
+ at specialize.arg(1)
+def signext(value, size):
+    # 'value' is sign-extended from 'size' bytes to a full integer.
+    # 'size' should be a constant smaller than a full integer size.
+    if size == rffi.sizeof(rffi.SIGNEDCHAR):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SIGNEDCHAR, value))
+    elif size == rffi.sizeof(rffi.SHORT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SHORT, value))
+    elif size == rffi.sizeof(rffi.INT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+    else:
+        raise AssertionError("unsupported size")
+
 # ____________________________________________________________
 
 class _NotStandardObject(Exception):
@@ -334,13 +347,26 @@
 
 # ____________________________________________________________
 
-def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+def pack_list_to_raw_array_bounds_signed(int_list, target, size):
     for TP, TPP in _prim_signed_types:
         if size == rffi.sizeof(TP):
             ptr = rffi.cast(TPP, target)
             for i in range(len(int_list)):
                 x = int_list[i]
-                if r_uint(x) - vmin > vrangemax:
+                y = rffi.cast(TP, x)
+                if x != rffi.cast(lltype.Signed, y):
+                    return x      # overflow
+                ptr[i] = y
+            return 0
+    raise NotImplementedError("bad integer size")
+
+def pack_list_to_raw_array_bounds_unsigned(int_list, target, size, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) > vrangemax:
                     return x      # overflow
                 ptr[i] = rffi.cast(TP, x)
             return 0
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -304,7 +304,7 @@
             py.test.skip("works with internals of _file impl on py.py")
         state = [0]
         def read(fd, n=None):
-            if fd != 42:
+            if fd != 424242:
                 return cls.old_read(fd, n)
             if state[0] == 0:
                 state[0] += 1
@@ -315,7 +315,7 @@
             return ''
         os.read = read
         stdin = W_File(cls.space)
-        stdin.file_fdopen(42, 'rb', 1)
+        stdin.file_fdopen(424242, 'rb', 1)
         stdin.name = '<stdin>'
         cls.w_stream = stdin
 
diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py
--- a/pypy/module/_file/test/test_file_extra.py
+++ b/pypy/module/_file/test/test_file_extra.py
@@ -221,7 +221,7 @@
     expected_filename = str(udir.join('sample'))
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -281,7 +281,7 @@
     expected_filename = '<fdopen>'
     expected_mode = 'rb'
     extra_args = ()
-    spaceconfig = {'usemodules': ['binascii', 'rctime', 'struct']}
+    spaceconfig = {'usemodules': ['binascii', 'time', 'struct']}
 
     def setup_method(self, method):
         space = self.space
@@ -359,7 +359,7 @@
 #  A few extra tests
 
 class AppTestAFewExtra:
-    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'rctime',
+    spaceconfig = {'usemodules': ['_socket', 'array', 'binascii', 'time',
                                   'struct']}
 
     def setup_method(self, method):
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -1,6 +1,6 @@
 class AppTestCProfile(object):
     spaceconfig = {
-        "usemodules": ['_lsprof', 'rctime'],
+        "usemodules": ['_lsprof', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -5,7 +5,7 @@
 
 class AppTestMD5(object):
     spaceconfig = {
-        'usemodules': ['_md5', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_md5', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -254,7 +254,7 @@
         start = _GetTickCount()
 
         while True:
-            from pypy.module.rctime.interp_time import State
+            from pypy.module.time.interp_time import State
             interrupt_event = space.fromcache(State).get_interrupt_event()
             handles = [self.handle, interrupt_event]
 
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -1,6 +1,6 @@
 class AppTestRandom:
     spaceconfig = {
-        "usemodules": ['_random', 'rctime'],
+        "usemodules": ['_random', 'time'],
     }
 
     def test_dict(self):
diff --git a/pypy/module/_sha/test/test_sha.py b/pypy/module/_sha/test/test_sha.py
--- a/pypy/module/_sha/test/test_sha.py
+++ b/pypy/module/_sha/test/test_sha.py
@@ -5,7 +5,7 @@
 
 class AppTestSHA(object):
     spaceconfig = {
-        'usemodules': ['_sha', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_sha', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -52,7 +52,8 @@
 
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
-constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
+if not OPENSSL_NO_SSL3:
+    constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 
@@ -656,7 +657,7 @@
     # set up context
     if protocol == PY_SSL_VERSION_TLS1:
         method = libssl_TLSv1_method()
-    elif protocol == PY_SSL_VERSION_SSL3:
+    elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
         method = libssl_SSLv3_method()
     elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
         method = libssl_SSLv2_method()
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -24,12 +24,19 @@
 
 separate_module_source = """
 #include <openssl/crypto.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 
 static unsigned int _ssl_locks_count = 0;
 static struct RPyOpaque_ThreadLock *_ssl_locks;
 
 static unsigned long _ssl_thread_id_function(void) {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (unsigned long)GetCurrentThreadId();
+#else
+    return (unsigned long)pthread_self();
+#endif
 }
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -53,7 +53,7 @@
 
 class AppTestBZ2File(CheckAllocation):
     spaceconfig = {
-        'usemodules': ['bz2', 'binascii', 'rctime', 'struct']
+        'usemodules': ['bz2', 'binascii', 'time', 'struct']
     }
 
     def setup_class(cls):
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -2,6 +2,7 @@
 #define CPPYY_CAPI
 
 #include <stddef.h>
+#include "src/precommondefs.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -15,102 +16,167 @@
     typedef void* (*cppyy_methptrgetter_t)(cppyy_object_t);
 
     /* name to opaque C++ scope representation -------------------------------- */
+    RPY_EXTERN
     int cppyy_num_scopes(cppyy_scope_t parent);
+    RPY_EXTERN
     char* cppyy_scope_name(cppyy_scope_t parent, int iscope);
 
+    RPY_EXTERN
     char* cppyy_resolve_name(const char* cppitem_name);
+    RPY_EXTERN
     cppyy_scope_t cppyy_get_scope(const char* scope_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_get_template(const char* template_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj);
 
     /* memory management ------------------------------------------------------ */
+    RPY_EXTERN
     cppyy_object_t cppyy_allocate(cppyy_type_t type);
+    RPY_EXTERN
     void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self);
+    RPY_EXTERN
     void cppyy_destruct(cppyy_type_t type, cppyy_object_t self);
 
     /* method/function dispatching -------------------------------------------- */
+    RPY_EXTERN
     void   cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char   cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     short  cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     int    cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long   cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     float  cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args);
+    RPY_EXTERN
     cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type);
 
+    RPY_EXTERN
     cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx);
 
     /* handling of function argument buffer ----------------------------------- */
+    RPY_EXTERN
     void*  cppyy_allocate_function_args(int nargs);
+    RPY_EXTERN
     void   cppyy_deallocate_function_args(void* args);
+    RPY_EXTERN
     size_t cppyy_function_arg_sizeof();
+    RPY_EXTERN
     size_t cppyy_function_arg_typeoffset();
 
     /* scope reflection information ------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_namespace(cppyy_scope_t scope);
+    RPY_EXTERN
     int cppyy_is_enum(const char* type_name);
 
     /* class reflection information ------------------------------------------- */
+    RPY_EXTERN
     char* cppyy_final_name(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_scoped_final_name(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_has_complex_hierarchy(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_num_bases(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_base_name(cppyy_type_t type, int base_index);
+    RPY_EXTERN
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
 
     /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */
+    RPY_EXTERN
     ptrdiff_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
 
     /* method/function reflection information --------------------------------- */
+    RPY_EXTERN
     int cppyy_num_methods(cppyy_scope_t scope);
+    RPY_EXTERN
     cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth);
+    RPY_EXTERN
     cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name);
 
+    RPY_EXTERN
     char* cppyy_method_name(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_result_type(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_req_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_arg_type(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_arg_default(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx);
 
+    RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_template_arg_name(cppyy_scope_t scope, cppyy_index_t idx, cppyy_index_t iarg);
 
+    RPY_EXTERN
     cppyy_method_t cppyy_get_method(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     cppyy_index_t cppyy_get_global_operator(
         cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op);
 
     /* method properties ------------------------------------------------------ */
+    RPY_EXTERN
     int cppyy_is_constructor(cppyy_type_t type, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_is_staticmethod(cppyy_type_t type, cppyy_index_t idx);
 
     /* data member reflection information ------------------------------------- */
+    RPY_EXTERN
     int cppyy_num_datamembers(cppyy_scope_t scope);
+    RPY_EXTERN
     char* cppyy_datamember_name(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
 
+    RPY_EXTERN
     int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 
     /* data member properties ------------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_publicdata(cppyy_type_t type, int datamember_index);
+    RPY_EXTERN
     int cppyy_is_staticdata(cppyy_type_t type, int datamember_index);
 
     /* misc helpers ----------------------------------------------------------- */
+    RPY_EXTERN
     long long cppyy_strtoll(const char* str);
+    RPY_EXTERN
     unsigned long long cppyy_strtoull(const char* str);
+    RPY_EXTERN
     void cppyy_free(void* ptr);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_charp2stdstring(const char* str);
+    RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
 #ifdef __cplusplus
diff --git a/pypy/module/cppyy/src/dummy_backend.cxx b/pypy/module/cppyy/src/dummy_backend.cxx
--- a/pypy/module/cppyy/src/dummy_backend.cxx
+++ b/pypy/module/cppyy/src/dummy_backend.cxx
@@ -1,4 +1,3 @@
-#include "src/precommondefs.h"
 #include "cppyy.h"
 #include "capi.h"
 
@@ -349,29 +348,24 @@
 
 
 /* name to opaque C++ scope representation -------------------------------- */
-RPY_EXTERN
 int cppyy_num_scopes(cppyy_scope_t handle) {
     return 0;
 }
 
-RPY_EXTERN
 char* cppyy_resolve_name(const char* cppitem_name) {
     return cppstring_to_cstring(cppitem_name);
 }
 
-RPY_EXTERN
 cppyy_scope_t cppyy_get_scope(const char* scope_name) {
     return s_handles[scope_name];  // lookup failure will return 0 (== error)
 }
 
-RPY_EXTERN
 cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t /* obj */) {
     return klass;
 }
 
 
 /* memory management ------------------------------------------------------ */
-RPY_EXTERN
 void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
     if (handle == s_handles["example01"])
        delete (dummy::example01*)self;
@@ -379,7 +373,6 @@
 
 
 /* method/function dispatching -------------------------------------------- */
-RPY_EXTERN
 void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long idx = (long)method;
     if (idx == s_methods["static_example01::staticSetPayload_payload*_double"]) {
@@ -469,7 +462,6 @@
     }
 }
 
-RPY_EXTERN
 unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     unsigned char result = 0;
     const long idx = (long)method;
@@ -482,7 +474,6 @@
     return result;
 }
 
-RPY_EXTERN
 char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char result = 0;
     const long idx = (long)method;
@@ -498,7 +489,6 @@
     return result;
 }
 
-RPY_EXTERN
 short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     short result = 0;
     const long idx = (long)method; 
@@ -514,7 +504,6 @@
     return result;
 }
 
-RPY_EXTERN
 int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     int result = 0;
     const long idx = (long)method;
@@ -547,7 +536,6 @@
     return result;
 }
 
-RPY_EXTERN
 long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long result = 0;
     const long idx = (long)method;
@@ -689,7 +677,6 @@
     return result;
 }
 
-RPY_EXTERN
 long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long long result = 0;
     const long idx = (long)method;
@@ -705,7 +692,6 @@
     return result;
 }   
 
-RPY_EXTERN
 float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     float result = 0;
     const long idx = (long)method;
@@ -718,7 +704,6 @@
     return result;
 }   
 
-RPY_EXTERN
 double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     double result = 0.;
     const long idx = (long)method;
@@ -740,7 +725,6 @@
     return result;
 }
 
-RPY_EXTERN
 char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char* result = 0;
     const long idx = (long)method;
@@ -753,7 +737,6 @@
     return result;
 }
 
-RPY_EXTERN
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
     void* result = 0;
     const long idx = (long)method;
@@ -776,14 +759,12 @@
     return (cppyy_object_t)result;
 }
 
-RPY_EXTERN
 cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) {
     return (cppyy_methptrgetter_t)0;
 }
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXTERN
 void* cppyy_allocate_function_args(int nargs) {
     CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
     for (int i = 0; i < nargs; ++i)
@@ -793,36 +774,30 @@
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXTERN
 void cppyy_deallocate_function_args(void* args) {
     free(args);
 }
 
-RPY_EXTERN
 size_t cppyy_function_arg_sizeof() {
     return sizeof(CPPYY_G__value);
 }
 
-RPY_EXTERN
 size_t cppyy_function_arg_typeoffset() {
     return offsetof(CPPYY_G__value, type);
 }
 
 
 /* scope reflection information ------------------------------------------- */
-RPY_EXTERN
 int cppyy_is_namespace(cppyy_scope_t /* handle */) {
     return 0;
 }   
 
-RPY_EXTERN
 int cppyy_is_enum(const char* /* type_name */) {
     return 0;
 }
     
     
 /* class reflection information ------------------------------------------- */
-RPY_EXTERN
 char* cppyy_final_name(cppyy_type_t handle) {
     for (Handles_t::iterator isp = s_handles.begin(); isp != s_handles.end(); ++isp) {
         if (isp->second == handle)
@@ -831,75 +806,61 @@
     return cppstring_to_cstring("<unknown>");
 }
 
-RPY_EXTERN
 char* cppyy_scoped_final_name(cppyy_type_t handle) {
     return cppyy_final_name(handle);
 }   
 
-RPY_EXTERN
 int cppyy_has_complex_hierarchy(cppyy_type_t /* handle */) {
     return 0;
 }
 
-RPY_EXTERN
 int cppyy_num_bases(cppyy_type_t /*handle*/) {
    return 0;
 }
 
 
 /* method/function reflection information --------------------------------- */
-RPY_EXTERN
 int cppyy_num_methods(cppyy_scope_t handle) {
     return s_scopes[handle].m_methods.size();
 }
 
-RPY_EXTERN
 cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) {
     return (cppyy_index_t)imeth;
 }
 
-RPY_EXTERN
 char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name);
 }
 
-RPY_EXTERN
 char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_returntype);
 }
     
-RPY_EXTERN
 int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return s_scopes[handle].m_methods[method_index].m_argtypes.size();
 }
 
-RPY_EXTERN
 int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppyy_method_num_args(handle, method_index);
 }
 
-RPY_EXTERN
 char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_argtypes[arg_index]);
 }
 
-RPY_EXTERN
 char* cppyy_method_arg_default(
         cppyy_scope_t /* handle */, cppyy_index_t /* method_index */, int /* arg_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXTERN
 char* cppyy_method_signature(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXTERN
 int cppyy_method_is_template(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return 0;
 }
     
-RPY_EXTERN
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end()) {
         long id = s_scopes[handle].m_method_offset + (long)method_index;
@@ -911,7 +872,6 @@
 
 
 /* method properties -----------------------------------------------------  */
-RPY_EXTERN
 int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kConstructor;
@@ -919,7 +879,6 @@
     return 0;
 }
 
-RPY_EXTERN
 int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kStatic;
@@ -929,34 +888,28 @@
 
 
 /* data member reflection information ------------------------------------- */
-RPY_EXTERN
 int cppyy_num_datamembers(cppyy_scope_t handle) {
     return s_scopes[handle].m_datambrs.size();
 }
 
-RPY_EXTERN
 char* cppyy_datamember_name(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_name);
 }
 
-RPY_EXTERN
 char* cppyy_datamember_type(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_type);
 }
 
-RPY_EXTERN
 ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_offset;
 }
 
 
 /* data member properties ------------------------------------------------  */
-RPY_EXTERN
 int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) {
     return 1;
 }
 
-RPY_EXTERN
 int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
@@ -964,44 +917,37 @@
 
 /* misc helpers ----------------------------------------------------------- */
 #if defined(_MSC_VER)
-RPY_EXTERN
 long long cppyy_strtoll(const char* str) {
     return _strtoi64(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXTERN
 unsigned long long cppyy_strtoull(const char* str) {
     return _strtoui64(str, NULL, 0);
 }
 }
 #else
-RPY_EXTERN
 long long cppyy_strtoll(const char* str) {
     return strtoll(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXTERN
 unsigned long long cppyy_strtoull(const char* str) {
     return strtoull(str, NULL, 0);
 }
 }
 #endif
 
-RPY_EXTERN
 void cppyy_free(void* ptr) {
     free(ptr);
 }
 
-RPY_EXTERN
 cppyy_object_t cppyy_charp2stdstring(const char* str) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(str);
     return (cppyy_object_t)arena;
 }
 
-RPY_EXTERN
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(*(std::string*)ptr);
diff --git a/pypy/module/cpyext/src/pythread.c b/pypy/module/cpyext/src/pythread.c
--- a/pypy/module/cpyext/src/pythread.c
+++ b/pypy/module/cpyext/src/pythread.c
@@ -1,11 +1,18 @@
 #include <Python.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 #include "pythread.h"
 #include "src/thread.h"
 
 long
 PyThread_get_thread_ident(void)
 {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (long)GetCurrentThreadId();
+#else
+    return (long)pthread_self();
+#endif
 }
 
 PyThread_type_lock
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -7,7 +7,7 @@
     # it's necessary to run "import time" at least once before any
     # other cpyext test, otherwise the same statement will fail in
     # test_datetime.py.
-    space = gettestobjspace(usemodules=['rctime'])
+    space = gettestobjspace(usemodules=['time'])
     space.getbuiltinmodule("time")
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -102,7 +102,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
+                                   'itertools', 'time', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -12,7 +12,7 @@
 
 class AppTestFcntl:
     spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios',
-                                   'select', 'rctime'))
+                                   'select', 'time'))
 
     def setup_class(cls):
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -4,7 +4,7 @@
 
 class AppTestImpModule:
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'rctime', 'struct'],
+        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -149,7 +149,7 @@
 
 class AppTestImport:
     spaceconfig = {
-        "usemodules": ['_md5', 'rctime'],
+        "usemodules": ['_md5', 'time'],
     }
 
     def setup_class(cls):
@@ -1044,7 +1044,7 @@
 
 class AppTestImportHooks(object):
     spaceconfig = {
-        "usemodules": ['struct', 'itertools', 'rctime'],
+        "usemodules": ['struct', 'itertools', 'time'],
     }
 
     def setup_class(cls):
@@ -1304,7 +1304,7 @@
 
 
 class AppTestMultithreadedImp(object):
-    spaceconfig = dict(usemodules=['thread', 'rctime'])
+    spaceconfig = dict(usemodules=['thread', 'time'])
 
     def setup_class(cls):
         #if not conftest.option.runappdirect:
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -144,7 +144,6 @@
     atom_int(tc, int)           puts code and int
     atom_int64(tc, int64)       puts code and int64
     atom_str(tc, str)           puts code, len and string
-    atom_strlist(tc, strlist)   puts code, len and list of strings
 
     building blocks for compound types:
 
@@ -198,15 +197,6 @@
         self.atom_int(typecode, len(x))
         self.put(x)
 
-    def atom_strlist(self, typecode, tc2, x):
-        self.atom_int(typecode, len(x))
-        atom_str = self.atom_str
-        for item in x:
-            # type(str) seems to be forbidden
-            #if type(item) is not str:
-            #    self.raise_exc('object with wrong type in strlist')
-            atom_str(tc2, item)
-
     def start(self, typecode):
         # type(char) not supported
         self.put(typecode)
@@ -379,16 +369,6 @@
         self.start(typecode)
         return self.get_lng()
 
-    def atom_strlist(self, typecode, tc2):
-        self.start(typecode)
-        lng = self.get_lng()
-        res = [None] * lng
-        idx = 0
-        while idx < lng:
-            res[idx] = self.atom_str(tc2)
-            idx += 1
-        return res
-
     def start(self, typecode):
         tc = self.get1()
         if tc != typecode:
@@ -436,7 +416,6 @@
 
     def get_w_obj(self, allow_null=False):
         space = self.space
-        w_ret = space.w_None # something not None
         tc = self.get1()
         w_ret = self._dispatch[ord(tc)](space, self, tc)
         if w_ret is None and not allow_null:
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -7,7 +7,7 @@
 
 class AppTestMath:
     spaceconfig = {
-        "usemodules": ['math', 'struct', 'itertools', 'rctime', 'binascii'],
+        "usemodules": ['math', 'struct', 'itertools', 'time', 'binascii'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from rpython.tool.pairtype import extendabletype
-
+from pypy.module.micronumpy import support
 
 def wrap_impl(space, w_cls, w_instance, impl):
     if w_cls is None or space.is_w(w_cls, space.gettypefor(W_NDimArray)):
@@ -44,11 +44,32 @@
         return W_NDimArray(impl)
 
     @staticmethod
-    def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False,
-                               w_subtype=None, w_base=None, writable=True):
+    def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1,
+                               order='C', owning=False, w_subtype=None,
+                               w_base=None, writable=True, strides=None):
         from pypy.module.micronumpy import concrete
-        from pypy.module.micronumpy.strides import calc_strides
-        strides, backstrides = calc_strides(shape, dtype, order)
+        from pypy.module.micronumpy.strides import (calc_strides,
+                                                    calc_backstrides)
+        isize = dtype.elsize
+        if storage_bytes > 0 :
+            totalsize = support.product(shape) * isize
+            if totalsize > storage_bytes:
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "buffer is too small for requested array"))
+        else:
+            storage_bytes = support.product(shape) * isize
+        if strides is None:
+            strides, backstrides = calc_strides(shape, dtype, order)
+        else:
+            if len(strides) != len(shape):
+                raise oefmt(space.w_ValueError,
+                    'strides, if given, must be the same length as shape')
+            for i in range(len(strides)):
+                if strides[i] < 0 or strides[i]*shape[i] > storage_bytes:
+                    raise oefmt(space.w_ValueError,
+                        'strides is incompatible with shape of requested '
+                        'array and size of buffer')
+            backstrides = calc_backstrides(strides, shape)
         if w_base is not None:
             if owning:
                 raise OperationError(space.w_ValueError,
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -59,9 +59,9 @@
     _mixin_ = True
 
     def reduce(self, space):
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
+        _numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(_numpypy, MixedModule)
+        multiarray = _numpypy.get("multiarray")
         assert isinstance(multiarray, MixedModule)
         scalar = multiarray.get("scalar")
 
@@ -167,7 +167,7 @@
         if len(args_w) >= 1:
             for w_arg in args_w:
                 try:
-                    idx = support.index_w(space, w_arg)
+                    support.index_w(space, w_arg)
                 except OperationError:
                     raise oefmt(space.w_TypeError, "an integer is required")
             raise oefmt(space.w_ValueError, "axes don't match array")
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -34,8 +34,8 @@
 
 
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
-                        "unegative", "flat", "tostring","count_nonzero",
-                        "argsort"]
+                        "unegative", "flat", "tostring", "count_nonzero",
+                        "argsort", "cumsum", "logical_xor_reduce"]
 TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted']
 TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype']
 THREE_ARG_FUNCTIONS = ['where']
@@ -559,6 +559,11 @@
                 w_res = arr.descr_any(interp.space)
             elif self.name == "all":
                 w_res = arr.descr_all(interp.space)
+            elif self.name == "cumsum":
+                w_res = arr.descr_cumsum(interp.space)
+            elif self.name == "logical_xor_reduce":
+                logical_xor = ufuncs.get(interp.space).logical_xor
+                w_res = logical_xor.reduce(interp.space, arr, None)
             elif self.name == "unegative":
                 neg = ufuncs.get(interp.space).negative
                 w_res = neg.call(interp.space, [arr])
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -11,7 +11,7 @@
 from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
     RecordChunk, calc_strides, calc_new_strides, shape_agreement,
-    calculate_broadcast_strides)
+    calculate_broadcast_strides, calc_backstrides)
 
 
 class BaseConcreteArray(object):
@@ -47,6 +47,7 @@
     def setitem(self, index, value):
         self.dtype.itemtype.store(self, index, 0, value)
 
+    @jit.unroll_safe
     def setslice(self, space, arr):

From noreply at buildbot.pypy.org  Tue Dec  9 17:29:56 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Tue,  9 Dec 2014 17:29:56 +0100 (CET)
Subject: [pypy-commit] pypy recursion_and_inlining: Close to-be-merged
	branch.
Message-ID: <20141209162956.965E81C09B2@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: recursion_and_inlining
Changeset: r74868:dcda54278f74
Date: 2014-12-09 16:26 +0000
http://bitbucket.org/pypy/pypy/changeset/dcda54278f74/

Log:	Close to-be-merged branch.


From noreply at buildbot.pypy.org  Tue Dec  9 17:29:57 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Tue,  9 Dec 2014 17:29:57 +0100 (CET)
Subject: [pypy-commit] pypy default: Merge recursion_and_inlining.
Message-ID: <20141209162957.D13DB1C09B2@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r74869:70d88f23b9bb
Date: 2014-12-09 16:29 +0000
http://bitbucket.org/pypy/pypy/changeset/70d88f23b9bb/

Log:	Merge recursion_and_inlining.

	This branch stops inlining in recursive function calls after N
	levels of (posiblly indirect) recursion in a function (where N is
	configurable; what the best possible value of N might be is still a
	little unclear, and ideally requires testing on a wider range of
	benchmarks). This stops us abusing abort as a way of stopping
	inlining in recursion, and improves performance.

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -964,9 +964,40 @@
         assembler_call = False
         if warmrunnerstate.inlining:
             if warmrunnerstate.can_inline_callable(greenboxes):
+                # We've found a potentially inlinable function; now we need to
+                # see if it's already on the stack. In other words: are we about
+                # to enter recursion? If so, we don't want to inline the
+                # recursion, which would be equivalent to unrolling a while
+                # loop.
                 portal_code = targetjitdriver_sd.mainjitcode
-                return self.metainterp.perform_call(portal_code, allboxes,
-                                                    greenkey=greenboxes)
+                count = 0
+                for f in self.metainterp.framestack:
+                    if f.jitcode is not portal_code:
+                        continue
+                    gk = f.greenkey
+                    if gk is None:
+                        continue
+                    assert len(gk) == len(greenboxes)
+                    i = 0
+                    for i in range(len(gk)):
+                        if not gk[i].same_constant(greenboxes[i]):
+                            break
+                    else:
+                        count += 1
+                memmgr = self.metainterp.staticdata.warmrunnerdesc.memory_manager
+                if count >= memmgr.max_unroll_recursion:
+                    # This function is recursive and has exceeded the
+                    # maximum number of unrollings we allow. We want to stop
+                    # inlining it further and to make sure that, if it
+                    # hasn't happened already, the function is traced
+                    # separately as soon as possible.
+                    if have_debug_prints():
+                        loc = targetjitdriver_sd.warmstate.get_location_str(greenboxes)
+                        debug_print("recursive function (not inlined):", loc)
+                    warmrunnerstate.dont_trace_here(greenboxes)
+                else:
+                    return self.metainterp.perform_call(portal_code, allboxes,
+                                greenkey=greenboxes)
             assembler_call = True
             # verify that we have all green args, needed to make sure
             # that assembler that we call is still correct
diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -1112,6 +1112,37 @@
         assert res == 2095
         self.check_resops(call_assembler=12)
 
+    def test_inline_recursion_limit(self):
+        driver = JitDriver(greens = ["threshold", "loop"], reds=["i"])
+        @dont_look_inside
+        def f():
+            set_param(driver, "max_unroll_recursion", 10)
+        def portal(threshold, loop, i):
+            f()
+            if i > threshold:
+                return i
+            while True:
+                driver.jit_merge_point(threshold=threshold, loop=loop, i=i)
+                if loop:
+                    portal(threshold, False, 0)
+                else:
+                    portal(threshold, False, i + 1)
+                    return i
+                if i > 10:
+                    return 1
+                i += 1
+                driver.can_enter_jit(threshold=threshold, loop=loop, i=i)
+
+        res1 = portal(10, True, 0)
+        res2 = self.meta_interp(portal, [10, True, 0], inline=True)
+        assert res1 == res2
+        self.check_resops(call_assembler=2)
+
+        res1 = portal(9, True, 0)
+        res2 = self.meta_interp(portal, [9, True, 0], inline=True)
+        assert res1 == res2
+        self.check_resops(call_assembler=0)
+
     def test_handle_jitexception_in_portal(self):
         # a test for _handle_jitexception_in_portal in blackhole.py
         driver = JitDriver(greens = ['codeno'], reds = ['i', 'str'],
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -69,7 +69,8 @@
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
                     function_threshold=4,
-                    enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, **kwds):
+                    enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, 
+                    max_unroll_recursion=7, **kwds):
     from rpython.config.config import ConfigError
     translator = interp.typer.annotator.translator
     try:
@@ -91,6 +92,7 @@
         jd.warmstate.set_param_retrace_limit(retrace_limit)
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
+        jd.warmstate.set_param_max_unroll_recursion(max_unroll_recursion)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
         return interp, graph
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -291,6 +291,11 @@
             if self.warmrunnerdesc.memory_manager:
                 self.warmrunnerdesc.memory_manager.max_unroll_loops = value
 
+    def set_param_max_unroll_recursion(self, value):
+        if self.warmrunnerdesc:
+            if self.warmrunnerdesc.memory_manager:
+                self.warmrunnerdesc.memory_manager.max_unroll_recursion = value
+
     def disable_noninlinable_function(self, greenkey):
         cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
         cell.flags |= JC_DONT_TRACE_HERE
@@ -567,19 +572,26 @@
         jd = self.jitdriver_sd
         cpu = self.cpu
 
-        def can_inline_greenargs(*greenargs):
+        def can_inline_callable(greenkey):
+            greenargs = unwrap_greenkey(greenkey)
             if can_never_inline(*greenargs):
                 return False
             cell = JitCell.get_jitcell(*greenargs)
             if cell is not None and (cell.flags & JC_DONT_TRACE_HERE) != 0:
                 return False
             return True
-        def can_inline_callable(greenkey):
-            greenargs = unwrap_greenkey(greenkey)
-            return can_inline_greenargs(*greenargs)
-        self.can_inline_greenargs = can_inline_greenargs
         self.can_inline_callable = can_inline_callable
 
+        def dont_trace_here(greenkey):
+            # Set greenkey as somewhere that tracing should not occur into;
+            # notice that, as per the description of JC_DONT_TRACE_HERE earlier,
+            # if greenkey hasn't been traced separately, setting
+            # JC_DONT_TRACE_HERE will force tracing the next time the function
+            # is encountered.
+            cell = JitCell.ensure_jit_cell_at_key(greenkey)
+            cell.flags |= JC_DONT_TRACE_HERE
+        self.dont_trace_here = dont_trace_here
+
         if jd._should_unroll_one_iteration_ptr is None:
             def should_unroll_one_iteration(greenkey):
                 return False
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -463,6 +463,7 @@
     'max_unroll_loops': 'number of extra unrollings a loop can cause',
     'enable_opts': 'INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): '
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
+    'max_unroll_recursion': 'how many levels deep to unroll a recursive function'
     }
 
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
@@ -476,6 +477,7 @@
               'max_retrace_guards': 15,
               'max_unroll_loops': 0,
               'enable_opts': 'all',
+              'max_unroll_recursion': 7,
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
 

From noreply at buildbot.pypy.org  Tue Dec  9 17:30:58 2014
From: noreply at buildbot.pypy.org (vext01)
Date: Tue,  9 Dec 2014 17:30:58 +0100 (CET)
Subject: [pypy-commit] benchmarks min_5_secs: Ensure that all benchmarks run
 for at least 5 secs on a fast machine.
Message-ID: <20141209163058.F38451C09B2@cobra.cs.uni-duesseldorf.de>

Author: Edd Barrett <vext01 at gmail.com>
Branch: min_5_secs
Changeset: r285:507b827a733d
Date: 2014-12-09 16:31 +0000
http://bitbucket.org/pypy/benchmarks/changeset/507b827a733d/

Log:	Ensure that all benchmarks run for at least 5 secs on a fast
	machine.

	(I actually aimed for 5<=t<=6, but some are more)

	Note that some benchmarks (spitfire, twisted) do not report how long
	an iteration took, but rather how many of x happened in t seconds.
	So the output may not necessarily be >=5 secs.

diff --git a/lib/pypy/rpython/translator/c/src/signals.o b/lib/pypy/rpython/translator/c/src/signals.o
index 166016f96874014a34535131bd7cfed3aabe09eb..978377c7573fccb667802c8cb75d56c1a1ce01b9
GIT binary patch

[cut]

diff --git a/lib/pypy/rpython/translator/c/src/stacklet/stacklet.o b/lib/pypy/rpython/translator/c/src/stacklet/stacklet.o
index e42497399f1f975b42e62fbf236498014c21f654..f9610bf7b82769ca1eeeac7f2eca92de8d120802
GIT binary patch

[cut]

diff --git a/lib/pypy/rpython/translator/c/src/thread.o b/lib/pypy/rpython/translator/c/src/thread.o
index 9b7dcc7cb6e9c6eb94216acf121fbcd5b32297e5..a0f49075af33414be1dca504e743a537a7f175f6
GIT binary patch

[cut]

diff --git a/own/bm_chameleon.py b/own/bm_chameleon.py
--- a/own/bm_chameleon.py
+++ b/own/bm_chameleon.py
@@ -13,15 +13,20 @@
 </tr>
 </table>"""
 
+INNER_ITERS = 500
+
 def main(n):
     tmpl = PageTemplate(BIGTABLE_ZPT)
     options = {'table': [dict(a=1, b=2, c=3, d=4, e=5, f=6, g=7, h=8, i=9, j=10)
                          for x in range(1000)]}
     import time
     l = []
-    for k in range(n):
+    for k in xrange(n):
         t0 = time.time()
-        tmpl(options=options)
+
+        for i in xrange(INNER_ITERS):
+            tmpl(options=options)
+
         l.append(time.time() - t0)
     return l
 
diff --git a/own/bm_dulwich_log.py b/own/bm_dulwich_log.py
--- a/own/bm_dulwich_log.py
+++ b/own/bm_dulwich_log.py
@@ -4,13 +4,18 @@
 
 import dulwich.repo
 
+INNER_ITERS = 50
+
 def test_dulwich(n):
     l = []
     r = dulwich.repo.Repo(os.path.join(os.path.dirname(__file__), 'git-demo'))
     import time
-    for i in range(20):
+    for i in xrange(n):
         t0 = time.time()
-        r.revision_history(r.head())
+
+        for j in xrange(INNER_ITERS):
+            r.revision_history(r.head())
+
         l.append(time.time() - t0)
     return l
 
diff --git a/own/bm_genshi.py b/own/bm_genshi.py
--- a/own/bm_genshi.py
+++ b/own/bm_genshi.py
@@ -22,21 +22,31 @@
 </table>
 """
 
-
 def main(n, bench):
     tmpl_cls, tmpl_str = {
         'xml': (MarkupTemplate, BIGTABLE_XML),
         'text': (NewTextTemplate, BIGTABLE_TEXT),
         }[bench]
+
+    if bench == "text":
+        inner_iters = 620
+    elif bench == "xml":
+        inner_iters = 220
+    else:
+        assert(False)
+
     tmpl = tmpl_cls(tmpl_str)
     context = {'table':
                [dict(a=1, b=2, c=3, d=4, e=5, f=6, g=7, h=8, i=9, j=10)
                 for x in range(1000)]}
     l = []
-    for k in range(n):
+    for k in xrange(n):
         t0 = time.time()
-        stream = tmpl.generate(**context)
-        stream.render()
+
+        for i in xrange(inner_iters):
+            stream = tmpl.generate(**context)
+            stream.render()
+
         l.append(time.time() - t0)
     return l
 
diff --git a/own/bm_mako.py b/own/bm_mako.py
--- a/own/bm_mako.py
+++ b/own/bm_mako.py
@@ -120,6 +120,7 @@
 ${fun6()}
 """
 
+INNER_ITERS = 550
 
 def test_mako(count):
 
@@ -128,7 +129,7 @@
     lookup.put_string('page.mako', PAGE_TEMPLATE)
 
     template = Template(CONTENT_TEMPLATE, lookup=lookup)
-    
+
     table = [xrange(150) for i in xrange(150)]
     paragraphs = xrange(50)
     title = 'Hello world!'
@@ -136,9 +137,11 @@
     times = []
     for i in range(count):
         t0 = time.time()
-        data = template.render(table=table, paragraphs=paragraphs,
-                               lorem=LOREM_IPSUM, title=title,
-                               img_count=50)
+
+        for j in xrange(INNER_ITERS):
+            data = template.render(table=table, paragraphs=paragraphs,
+                                   lorem=LOREM_IPSUM, title=title,
+                                   img_count=50)
         t1 = time.time()
         times.append(t1-t0)
     return times
diff --git a/own/bm_sympy.py b/own/bm_sympy.py
--- a/own/bm_sympy.py
+++ b/own/bm_sympy.py
@@ -20,13 +20,25 @@
     x, y, z = symbols('x y z')
     str(expand((x+2*y+3*z)**30))
 
+INNER_ITERS_D = {
+    'expand' : 650000,
+    'integrate' : 10,
+    'str' : 500,
+    'sum' : 500,
+}
+
 def main(n, bench):
+
+    inner_iters = INNER_ITERS_D.get(bench, 1)
     func = globals()['bench_' + bench]
     l = []
     for i in range(n):
         clear_cache()
         t0 = time.time()
-        func()
+
+        for j in xrange(inner_iters):
+            func()
+
         l.append(time.time() - t0)
     return l
 
diff --git a/own/chaos.py b/own/chaos.py
--- a/own/chaos.py
+++ b/own/chaos.py
@@ -13,6 +13,8 @@
 import sys
 import time
 
+INNER_ITERS = 5000 * 1300
+
 class GVector(object):
     def __init__(self, x = 0, y = 0, z = 0):
         self.x = x
@@ -219,9 +221,9 @@
                         (self.maxy + self.miny) / 2, 0)
         colored = 0
         times = []
-        for _ in range(n):
+        for _ in xrange(n):
             t1 = time.time()
-            for i in xrange(5000):
+            for i in xrange(INNER_ITERS):
                 point = self.transform_point(point)
                 x = (point.x - self.minx) / self.width * w
                 y = (point.y - self.miny) / self.height * h
diff --git a/own/crypto_pyaes.py b/own/crypto_pyaes.py
--- a/own/crypto_pyaes.py
+++ b/own/crypto_pyaes.py
@@ -22,13 +22,18 @@
 
     assert plaintext == cleartext
 
+INNER_ITERS = 197
+
 def main(arg):
     # XXX warmup
 
     times = []
     for i in xrange(arg):
         t0 = time.time()
-        o = benchmark()
+
+        for j in xrange(INNER_ITERS):
+            o = benchmark()
+
         tk = time.time()
         times.append(tk - t0)
     return times
diff --git a/own/deltablue.py b/own/deltablue.py
--- a/own/deltablue.py
+++ b/own/deltablue.py
@@ -614,15 +614,19 @@
     chain_test(100)
     projection_test(100)
 
+INNER_ITERS = 17000
 
 # Specific to the PyPy implementation, to run within the main harnass.
 def main(n):
     import time
     times = []
 
-    for i in range(n):
+    for i in xrange(n):
         t1 = time.time()
-        delta_blue()
+
+        for j in xrange(INNER_ITERS):
+            delta_blue()
+
         t2 = time.time()
         times.append(t2 - t1)
 
diff --git a/own/eparse.py b/own/eparse.py
--- a/own/eparse.py
+++ b/own/eparse.py
@@ -4,13 +4,18 @@
 ometa.FAST = True
 from monte.eparser import EParser
 
+INNER_ITERS = 24
+
 def main(n):
     l = []
     data = open(os.path.join(os.path.dirname(__file__), 'test.e')).read()
     for _ in range(n):
         t0 = time.time()
-        p = EParser(data)
-        v, e = p.apply('start')
+
+        for i in xrange(INNER_ITERS):
+            p = EParser(data)
+            v, e = p.apply('start')
+
         l.append(time.time() - t0)
     return l
 
diff --git a/own/fannkuch.py b/own/fannkuch.py
--- a/own/fannkuch.py
+++ b/own/fannkuch.py
@@ -52,15 +52,20 @@
 
 DEFAULT_ARG = 9
 
+INNER_ITERS = 60
+
 def main(n):
     times = []
-    for i in range(n):
+    for i in xrange(n):
         t0 = time.time()
-        fannkuch(DEFAULT_ARG)
+
+        for j in xrange(INNER_ITERS):
+            fannkuch(DEFAULT_ARG)
+
         tk = time.time()
         times.append(tk - t0)
     return times
-    
+
 if __name__ == "__main__":
     parser = optparse.OptionParser(
         usage="%prog [options]",
diff --git a/own/float.py b/own/float.py
--- a/own/float.py
+++ b/own/float.py
@@ -47,17 +47,22 @@
 
 POINTS = 100000
 
+INNER_ITERS = 240
+
 def main(arg):
     # XXX warmup
-    
+
     times = []
     for i in xrange(arg):
         t0 = time.time()
-        o = benchmark(POINTS)
+
+        for j in xrange(INNER_ITERS):
+            o = benchmark(POINTS)
+
         tk = time.time()
         times.append(tk - t0)
     return times
-    
+
 if __name__ == "__main__":
     parser = optparse.OptionParser(
         usage="%prog [options]",
diff --git a/own/go.py b/own/go.py
--- a/own/go.py
+++ b/own/go.py
@@ -423,13 +423,18 @@
     board = Board()
     pos = computer_move(board)
 
+INNER_ITERS = 89
+
 def main(n):
     times = []
-    for i in range(5):
+    for i in xrange(5):
         versus_cpu() # warmup
-    for i in range(n):
+    for i in xrange(n):
         t1 = time.time()
-        versus_cpu()
+
+        for j in xrange(INNER_ITERS):
+            versus_cpu()
+
         t2 = time.time()
         times.append(t2 - t1)
     return times
diff --git a/own/json_bench.py b/own/json_bench.py
--- a/own/json_bench.py
+++ b/own/json_bench.py
@@ -11,14 +11,20 @@
 
 cases = ['EMPTY', 'SIMPLE', 'NESTED', 'HUGE']
 
+INNER_ITERS = 13
+
 def main(n):
     l = []
-    for i in range(n):
+    for i in xrange(n):
         t0 = time.time()
-        for case in cases:
-            data, count = globals()[case]
-            for i in range(count):
-                json.dumps(data)
+
+        for m in xrange(INNER_ITERS):
+
+            for case in cases:
+                data, count = globals()[case]
+                for i in range(count):
+                    json.dumps(data)
+
         l.append(time.time() - t0)
     return l
 
diff --git a/own/meteor-contest.py b/own/meteor-contest.py
--- a/own/meteor-contest.py
+++ b/own/meteor-contest.py
@@ -135,21 +135,25 @@
 
 SOLVE_ARG = 60
 
+INNER_ITERS = 92
+
 def main(n):
     times = []
-    for i in range(n):
+    for i in xrange(n):
+
         t0 = time.time()
-        free = frozenset(xrange(len(board)))
-        curr_board = [-1] * len(board)
-        pieces_left = range(len(pieces))
-        solutions = []
-        solve(SOLVE_ARG, 0, free, curr_board, pieces_left, solutions)
-        #print len(solutions),  'solutions found\n'
-        #for i in (0, -1): print_board(solutions[i])
-        tk = time.time()
+        for j in xrange(INNER_ITERS):
+            free = frozenset(xrange(len(board)))
+            curr_board = [-1] * len(board)
+            pieces_left = range(len(pieces))
+            solutions = []
+            solve(SOLVE_ARG, 0, free, curr_board, pieces_left, solutions)
+            #print len(solutions),  'solutions found\n'
+            #for i in (0, -1): print_board(solutions[i])
+            tk = time.time()
         times.append(tk - t0)
     return times
-    
+
 if __name__ == "__main__":
     parser = optparse.OptionParser(
         usage="%prog [options]",
diff --git a/own/nbody_modified.py b/own/nbody_modified.py
--- a/own/nbody_modified.py
+++ b/own/nbody_modified.py
@@ -109,17 +109,22 @@
 
 NUMBER_OF_ITERATIONS = 20000
 
+INNER_ITERATIONS = 290
+
 def main(n, ref='sun'):
     # XXX warmup
-    
+
     times = []
-    for i in range(n):
+    for i in xrange(n):
         t0 = time.time()
-        offset_momentum(BODIES[ref])
-        report_energy()
-        advance(0.01, NUMBER_OF_ITERATIONS)
-        report_energy()
-        tk = time.time()
+
+        for j in xrange(INNER_ITERATIONS):
+            offset_momentum(BODIES[ref])
+            report_energy()
+            advance(0.01, NUMBER_OF_ITERATIONS)
+            report_energy()
+            tk = time.time()
+
         times.append(tk - t0)
     return times
 
diff --git a/own/pyflate-fast.py b/own/pyflate-fast.py
--- a/own/pyflate-fast.py
+++ b/own/pyflate-fast.py
@@ -660,6 +660,8 @@
     assert md5.md5(out).hexdigest() == "afa004a630fe072901b1d9628b960974"
     input.close()
 
+INNER_ITERS = 26
+
 def main(n):
     import time
     times = []
@@ -667,7 +669,10 @@
         _main() # warmup
     for i in range(n):
         t1 = time.time()
-        _main()
+
+        for j in xrange(INNER_ITERS):
+            _main()
+
         t2 = time.time()
         times.append(t2 - t1)
     return times
diff --git a/own/raytrace-simple.py b/own/raytrace-simple.py
--- a/own/raytrace-simple.py
+++ b/own/raytrace-simple.py
@@ -353,6 +353,8 @@
     s.addObject(Halfspace(Point(0,0,0), Vector.UP), CheckerboardSurface())
     s.render(c)
 
+INNER_ITERS = 300
+
 def main(n):
     import time
     times = []
@@ -360,7 +362,10 @@
         _main() # warmup
     for i in range(n):
         t1 = time.time()
-        _main()
+
+        for j in xrange(INNER_ITERS):
+            _main()
+
         t2 = time.time()
         times.append(t2 - t1)
     return times
diff --git a/own/spectral-norm.py b/own/spectral-norm.py
--- a/own/spectral-norm.py
+++ b/own/spectral-norm.py
@@ -42,25 +42,30 @@
 
 DEFAULT_N = 130
 
+INNER_ITERS = 600
+
 def main(n):
     times = []
-    for i in range(n):
+    for i in xrange(n):
         t0 = time.time()
-        u = [1] * DEFAULT_N
 
-        for dummy in xrange (10):
-            v = eval_AtA_times_u (u)
-            u = eval_AtA_times_u (v)
+        for j in xrange(INNER_ITERS):
+            u = [1] * DEFAULT_N
 
-        vBv = vv = 0
+            for dummy in xrange (10):
+                v = eval_AtA_times_u (u)
+                u = eval_AtA_times_u (v)
 
-        for ue, ve in izip (u, v):
-            vBv += ue * ve
-            vv  += ve * ve
+            vBv = vv = 0
+
+            for ue, ve in izip (u, v):
+                vBv += ue * ve
+                vv  += ve * ve
+
         tk = time.time()
         times.append(tk - t0)
     return times
-    
+
 if __name__ == "__main__":
     parser = optparse.OptionParser(
         usage="%prog [options]",
diff --git a/own/spitfire.py b/own/spitfire.py
--- a/own/spitfire.py
+++ b/own/spitfire.py
@@ -21,11 +21,17 @@
 # bummer, timeit module is stupid
 from bigtable import test_python_cstringio, test_spitfire_o4, test_spitfire
 
+INNER_ITERS_D = {
+    "python_cstringio" :  12000,
+    "spitfire_o4" : 4000,
+}
+
 def runtest(n, benchmark):
     times = []
+    inner_iters = INNER_ITERS_D.get(benchmark, 100)
     for i in range(n):
         sys.stdout = StringIO()
-        bigtable.run([benchmark], 100)
+        bigtable.run([benchmark], inner_iters)
         times.append(float(sys.stdout.getvalue().split(" ")[-2]))
         sys.stdout = sys.__stdout__
     return times
diff --git a/own/telco.py b/own/telco.py
--- a/own/telco.py
+++ b/own/telco.py
@@ -75,11 +75,16 @@
     end = time()
     return end - start
 
+INNER_ITERS = 450
+
 def main(n):
     run() # warmup
     times = []
     for i in range(n):
-        times.append(run())
+        all_inner_runs = 0.0
+        for j in xrange(INNER_ITERS):
+            all_inner_runs += run()
+        times.append(all_inner_runs)
     return times
 
 
diff --git a/unladen_swallow/perf.py b/unladen_swallow/perf.py
--- a/unladen_swallow/perf.py
+++ b/unladen_swallow/perf.py
@@ -809,7 +809,7 @@
     if options.rigorous:
         trials = 100
     elif options.fast:
-        trials = 5
+        trials = 1 # XXX
     trials = max(1, int(trials * iteration_scaling))
 
     RemovePycs()
diff --git a/unladen_swallow/performance/bm_ai.py b/unladen_swallow/performance/bm_ai.py
--- a/unladen_swallow/performance/bm_ai.py
+++ b/unladen_swallow/performance/bm_ai.py
@@ -69,6 +69,7 @@
             yield vec
 
 
+INNER_ITERS = 230
 def test_n_queens(iterations):
     # Warm-up runs.
     list(n_queens(8))
@@ -77,7 +78,10 @@
     times = []
     for _ in xrange(iterations):
         t0 = time.time()
-        list(n_queens(8))
+
+        for i in xrange(INNER_ITERS):
+            list(n_queens(8))
+
         t1 = time.time()
         times.append(t1 - t0)
     return times
diff --git a/unladen_swallow/performance/bm_django.py b/unladen_swallow/performance/bm_django.py
--- a/unladen_swallow/performance/bm_django.py
+++ b/unladen_swallow/performance/bm_django.py
@@ -32,6 +32,7 @@
 </table>
 """)
 
+INNER_ITERS = 300
 def test_django(count):
     table = [xrange(150) for _ in xrange(150)]
     context = Context({"table": table})
@@ -43,7 +44,10 @@
     times = []
     for _ in xrange(count):
         t0 = time.time()
-        data = DJANGO_TMPL.render(context)
+
+        for i in xrange(INNER_ITERS):
+            data = DJANGO_TMPL.render(context)
+
         t1 = time.time()
         times.append(t1 - t0)
     return times
diff --git a/unladen_swallow/performance/bm_html5lib.py b/unladen_swallow/performance/bm_html5lib.py
--- a/unladen_swallow/performance/bm_html5lib.py
+++ b/unladen_swallow/performance/bm_html5lib.py
@@ -23,6 +23,8 @@
 import html5lib
 
 
+INNER_ITERS = 4
+
 def test_html5lib(count, spec_data):
     # No warm-up runs for this benchmark; in real life, the parser doesn't get
     # to warm up (this isn't a daemon process).
@@ -31,7 +33,10 @@
     for _ in xrange(count):
         spec_data.seek(0)
         t0 = time.time()
-        html5lib.parse(spec_data)
+
+        for i in xrange(INNER_ITERS):
+            html5lib.parse(spec_data)
+
         t1 = time.time()
         times.append(t1 - t0)
     return times
diff --git a/unladen_swallow/performance/bm_richards.py b/unladen_swallow/performance/bm_richards.py
--- a/unladen_swallow/performance/bm_richards.py
+++ b/unladen_swallow/performance/bm_richards.py
@@ -18,6 +18,8 @@
 import util
 
 
+INNER_ITERS = 2750
+
 def test_richards(iterations):
     # Warm-up
     r = richards.Richards()
@@ -26,7 +28,7 @@
     times = []
     for _ in xrange(iterations):
         t0 = time.time()
-        r.run(iterations=1)
+        r.run(iterations=INNER_ITERS)
         t1 = time.time()
         times.append(t1 - t0)
     return times
diff --git a/unladen_swallow/performance/bm_rietveld.py b/unladen_swallow/performance/bm_rietveld.py
--- a/unladen_swallow/performance/bm_rietveld.py
+++ b/unladen_swallow/performance/bm_rietveld.py
@@ -86,6 +86,7 @@
     tmpl = loader.get_template(templ_name)
     return tmpl, context
 
+INNER_ITERS = 46
 
 def test_rietveld(count, tmpl, context):
     # Warm up Django.
@@ -95,37 +96,40 @@
     times = []
     for _ in xrange(count):
         t0 = time.time()
-        # 30 calls to render, so that we don't measure loop overhead.
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
-        tmpl.render(context)
+
+        for i in xrange(INNER_ITERS):
+            # 30 calls to render, so that we don't measure loop overhead.
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+            tmpl.render(context)
+
         t1 = time.time()
         times.append(t1 - t0)
     return times
diff --git a/unladen_swallow/performance/bm_spambayes.py b/unladen_swallow/performance/bm_spambayes.py
--- a/unladen_swallow/performance/bm_spambayes.py
+++ b/unladen_swallow/performance/bm_spambayes.py
@@ -20,6 +20,7 @@
 # Local imports
 import util
 
+INNER_ITERS = 150
 
 def test_spambayes(iterations, messages, ham_classifier):
     # Prime the pump. This still leaves some hot functions uncompiled; these
@@ -30,8 +31,9 @@
     times = []
     for _ in xrange(iterations):
         t0 = time.time()
-        for msg in messages:
-            ham_classifier.score(msg)
+        for i in xrange(INNER_ITERS):
+            for msg in messages:
+                ham_classifier.score(msg)
         t1 = time.time()
         times.append(t1 - t0)
     return times
diff --git a/unladen_swallow/performance/bm_spitfire.py b/unladen_swallow/performance/bm_spitfire.py
--- a/unladen_swallow/performance/bm_spitfire.py
+++ b/unladen_swallow/performance/bm_spitfire.py
@@ -40,6 +40,8 @@
 </table>
 """
 
+INNER_ITERS = 31
+
 def test_spitfire(count):
     # Activate the most aggressive Spitfire optimizations. While it might
     # conceivably be interesting to stress Spitfire's lower optimization
@@ -60,7 +62,10 @@
     times = []
     for _ in xrange(count):
         t0 = time.time()
-        data = spitfire_tmpl_o4(search_list=[{"table": table}]).main()
+
+        for i in xrange(INNER_ITERS):
+            data = spitfire_tmpl_o4(search_list=[{"table": table}]).main()
+
         t1 = time.time()
         times.append(t1 - t0)
     return times

From noreply at buildbot.pypy.org  Tue Dec  9 17:40:25 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Dec 2014 17:40:25 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Uh, don't check in the .o files
Message-ID: <20141209164025.DD2D41C0936@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r286:680e6f36aa42
Date: 2014-12-09 16:40 +0000
http://bitbucket.org/pypy/benchmarks/changeset/680e6f36aa42/

Log:	Uh, don't check in the .o files

diff --git a/lib/pypy/rpython/translator/c/src/signals.o b/lib/pypy/rpython/translator/c/src/signals.o
deleted file mode 100644
Binary file lib/pypy/rpython/translator/c/src/signals.o has changed
diff --git a/lib/pypy/rpython/translator/c/src/stacklet/stacklet.o b/lib/pypy/rpython/translator/c/src/stacklet/stacklet.o
deleted file mode 100644
Binary file lib/pypy/rpython/translator/c/src/stacklet/stacklet.o has changed
diff --git a/lib/pypy/rpython/translator/c/src/thread.o b/lib/pypy/rpython/translator/c/src/thread.o
deleted file mode 100644
Binary file lib/pypy/rpython/translator/c/src/thread.o has changed

From noreply at buildbot.pypy.org  Tue Dec  9 17:40:27 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Dec 2014 17:40:27 +0100 (CET)
Subject: [pypy-commit] benchmarks min_5_secs: don't check in the .o files
Message-ID: <20141209164027.53E501C0936@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: min_5_secs
Changeset: r287:4f82cc579716
Date: 2014-12-09 16:40 +0000
http://bitbucket.org/pypy/benchmarks/changeset/4f82cc579716/

Log:	don't check in the .o files

diff --git a/lib/pypy/rpython/translator/c/src/signals.o b/lib/pypy/rpython/translator/c/src/signals.o
deleted file mode 100644
Binary file lib/pypy/rpython/translator/c/src/signals.o has changed
diff --git a/lib/pypy/rpython/translator/c/src/stacklet/stacklet.o b/lib/pypy/rpython/translator/c/src/stacklet/stacklet.o
deleted file mode 100644
Binary file lib/pypy/rpython/translator/c/src/stacklet/stacklet.o has changed
diff --git a/lib/pypy/rpython/translator/c/src/thread.o b/lib/pypy/rpython/translator/c/src/thread.o
deleted file mode 100644
Binary file lib/pypy/rpython/translator/c/src/thread.o has changed

From noreply at buildbot.pypy.org  Tue Dec  9 17:49:30 2014
From: noreply at buildbot.pypy.org (vext01)
Date: Tue,  9 Dec 2014 17:49:30 +0100 (CET)
Subject: [pypy-commit] benchmarks min_5_secs: Put back --fast to 5
	iterations.
Message-ID: <20141209164930.0E7511D22D0@cobra.cs.uni-duesseldorf.de>

Author: Edd Barrett <vext01 at gmail.com>
Branch: min_5_secs
Changeset: r288:3926c2bd88df
Date: 2014-12-09 16:47 +0000
http://bitbucket.org/pypy/benchmarks/changeset/3926c2bd88df/

Log:	Put back --fast to 5 iterations.

diff --git a/unladen_swallow/perf.py b/unladen_swallow/perf.py
--- a/unladen_swallow/perf.py
+++ b/unladen_swallow/perf.py
@@ -809,7 +809,7 @@
     if options.rigorous:
         trials = 100
     elif options.fast:
-        trials = 1 # XXX
+        trials = 5
     trials = max(1, int(trials * iteration_scaling))
 
     RemovePycs()

From noreply at buildbot.pypy.org  Tue Dec  9 17:49:31 2014
From: noreply at buildbot.pypy.org (vext01)
Date: Tue,  9 Dec 2014 17:49:31 +0100 (CET)
Subject: [pypy-commit] benchmarks min_5_secs: merge
Message-ID: <20141209164931.7026E1D22D0@cobra.cs.uni-duesseldorf.de>

Author: Edd Barrett <vext01 at gmail.com>
Branch: min_5_secs
Changeset: r289:8c2f3584ae9a
Date: 2014-12-09 16:49 +0000
http://bitbucket.org/pypy/benchmarks/changeset/8c2f3584ae9a/

Log:	merge

diff --git a/lib/pypy/rpython/translator/c/src/signals.o b/lib/pypy/rpython/translator/c/src/signals.o
deleted file mode 100644
Binary file lib/pypy/rpython/translator/c/src/signals.o has changed
diff --git a/lib/pypy/rpython/translator/c/src/stacklet/stacklet.o b/lib/pypy/rpython/translator/c/src/stacklet/stacklet.o
deleted file mode 100644
Binary file lib/pypy/rpython/translator/c/src/stacklet/stacklet.o has changed
diff --git a/lib/pypy/rpython/translator/c/src/thread.o b/lib/pypy/rpython/translator/c/src/thread.o
deleted file mode 100644
Binary file lib/pypy/rpython/translator/c/src/thread.o has changed

From noreply at buildbot.pypy.org  Tue Dec  9 17:52:49 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Dec 2014 17:52:49 +0100 (CET)
Subject: [pypy-commit] benchmarks min_5_secs: (cfbolz, arigo)
Message-ID: <20141209165249.A62591C09B2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: min_5_secs
Changeset: r290:7acb20388ca1
Date: 2014-12-09 16:53 +0000
http://bitbucket.org/pypy/benchmarks/changeset/7acb20388ca1/

Log:	(cfbolz, arigo)

	Un-unroll that loop

diff --git a/unladen_swallow/performance/bm_rietveld.py b/unladen_swallow/performance/bm_rietveld.py
--- a/unladen_swallow/performance/bm_rietveld.py
+++ b/unladen_swallow/performance/bm_rietveld.py
@@ -86,7 +86,7 @@
     tmpl = loader.get_template(templ_name)
     return tmpl, context
 
-INNER_ITERS = 46
+INNER_ITERS = 46 * 30
 
 def test_rietveld(count, tmpl, context):
     # Warm up Django.
@@ -98,36 +98,6 @@
         t0 = time.time()
 
         for i in xrange(INNER_ITERS):
-            # 30 calls to render, so that we don't measure loop overhead.
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
-            tmpl.render(context)
             tmpl.render(context)
 
         t1 = time.time()

From noreply at buildbot.pypy.org  Tue Dec  9 18:00:08 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Dec 2014 18:00:08 +0100 (CET)
Subject: [pypy-commit] pypy default: (cfbolz, arigo)
Message-ID: <20141209170008.4DE431D37CE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74870:d5298ea685c7
Date: 2014-12-09 16:59 +0000
http://bitbucket.org/pypy/pypy/changeset/d5298ea685c7/

Log:	(cfbolz, arigo)

	Clarify what error we get from calling getattr() with 3 args in
	RPython

diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -528,6 +528,10 @@
     pyfunc = staticmethod(getattr)
 
     def constfold(self):
+        from rpython.flowspace.flowcontext import FlowingError
+        if len(self.args) == 3:
+            raise FlowingError(
+                "getattr() with three arguments not supported: %s" % (self,))
         w_obj, w_name = self.args
         # handling special things like sys
         if (w_obj in NOT_REALLY_CONST and
@@ -538,7 +542,6 @@
             try:
                 result = getattr(obj, name)
             except Exception as e:
-                from rpython.flowspace.flowcontext import FlowingError
                 etype = e.__class__
                 msg = "getattr(%s, %s) always raises %s: %s" % (
                     obj, name, etype, e)

From noreply at buildbot.pypy.org  Tue Dec  9 18:59:55 2014
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue,  9 Dec 2014 18:59:55 +0100 (CET)
Subject: [pypy-commit] benchmarks default: (arigo, cfbolz)
Message-ID: <20141209175955.6301A1D236C@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r291:ded8521a022d
Date: 2014-12-09 18:00 +0000
http://bitbucket.org/pypy/benchmarks/changeset/ded8521a022d/

Log:	(arigo, cfbolz)

	add a pypy_interp benchmark that runs a loop in the pypy interpreter

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -82,7 +82,7 @@
              'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast',
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
              'json_bench', 'pidigits', 'hexiom2', 'eparse', 'deltablue',
-             'bm_dulwich_log', 'bm_krakatau', 'bm_mdp']:
+             'bm_dulwich_log', 'bm_krakatau', 'bm_mdp', 'pypy_interp']:
     _register_new_bm(name, name, globals(), **opts.get(name, {}))
 
 for name in ['names', 'iteration', 'tcp', 'pb', ]:#'web']:#, 'accepts']:
diff --git a/own/pypy_interp.py b/own/pypy_interp.py
new file mode 100644
--- /dev/null
+++ b/own/pypy_interp.py
@@ -0,0 +1,45 @@
+import sys, os, cStringIO
+import time
+import util, optparse
+
+pardir = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
+pypypath = os.path.join(pardir, 'lib', 'pypy')
+sys.path.insert(0, pypypath)
+
+from pypy.tool.pytest.objspace import gettestobjspace
+
+def bench(space):
+    w_l = space.appexec([], """():
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+        glob_a = A(1)
+        l = []
+        a, b = 1, 1
+        for i in range(100):
+            l.append((i, a, str(i), float(i), {i: a}, A(b), glob_a.x))
+            a, b = b, a + b
+        return l
+    """)
+
+def main(n):
+    l = []
+    space = gettestobjspace()
+    # warmup
+    bench(space)
+    bench(space)
+    for i in range(n):
+        t0 = time.time()
+        bench(space)
+        time_elapsed = time.time() - t0
+        l.append(time_elapsed)
+    return l
+
+if __name__ == "__main__":
+    parser = optparse.OptionParser(
+        usage="%prog [options]",
+        description="Test the performance of the pypy-interp benchmark")
+    util.add_standard_options_to(parser)
+    options, args = parser.parse_args()
+
+    util.run_benchmark(options, options.num_runs, main)

From noreply at buildbot.pypy.org  Tue Dec  9 19:35:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  9 Dec 2014 19:35:02 +0100 (CET)
Subject: [pypy-commit] pypy default: Test get_errno() and get_last_error()
	here
Message-ID: <20141209183502.3806D1C09B2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74871:05a286a4b5e0
Date: 2014-12-09 18:34 +0000
http://bitbucket.org/pypy/pypy/changeset/05a286a4b5e0/

Log:	Test get_errno() and get_last_error() here

diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -16,6 +16,7 @@
         #include "src/precommondefs.h"
         #include <stdlib.h>
         #include <stdio.h>
+        #include <errno.h>
 
         struct x
         {
@@ -204,6 +205,24 @@
             return inp;
         }
 
+        RPY_EXPORTED
+        int check_errno(int incoming)
+        {
+            int old_errno = errno;
+            errno = incoming;
+            return old_errno;
+        }
+
+        #ifdef _WIN32
+        #include <Windows.h>
+        RPY_EXPORTED
+        int check_last_error(int incoming)
+        {
+            int old_errno = GetLastError();
+            SetLastError(incoming);
+            return old_errno;
+        }
+        #endif
         '''))
         eci = ExternalCompilationInfo(include_dirs=[cdir])
         return str(platform.compile([c_file], eci, 'x', standalone=False))
@@ -1150,6 +1169,37 @@
         raises(OverflowError, "arg1[0] = 10**900")
         arg1.free()
 
+    def test_errno(self):
+        import _rawffi
+        lib = _rawffi.CDLL(self.lib_name)
+        A = _rawffi.Array('i')
+        f = lib.ptr('check_errno', ['i'], 'i')
+        _rawffi.set_errno(42)
+        arg = A(1)
+        arg[0] = 43
+        res = f(arg)
+        assert res[0] == 42
+        z = _rawffi.get_errno()
+        assert z == 43
+        arg.free()
+
+    def test_last_error(self):
+        import sys
+        if sys.platform != 'win32':
+            skip("Windows test")
+        import _rawffi
+        lib = _rawffi.CDLL(self.lib_name)
+        A = _rawffi.Array('i')
+        f = lib.ptr('check_last_error', ['i'], 'i')
+        _rawffi.set_last_error(42)
+        arg = A(1)
+        arg[0] = 43
+        res = f(arg)
+        assert res[0] == 42
+        z = _rawffi.get_last_error()
+        assert z == 43
+        arg.free()
+
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])

From noreply at buildbot.pypy.org  Wed Dec 10 00:01:24 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Dec 2014 00:01:24 +0100 (CET)
Subject: [pypy-commit] pypy default: Attempt to fix issue #1944
Message-ID: <20141209230124.67F321D236C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74872:e4a49ff75f6d
Date: 2014-12-09 23:01 +0000
http://bitbucket.org/pypy/pypy/changeset/e4a49ff75f6d/

Log:	Attempt to fix issue #1944

diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -14,6 +14,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.module._rawffi.alt.type_converter import FromAppLevelConverter, ToAppLevelConverter
 from pypy.module._rawffi.interp_rawffi import got_libffi_error, wrap_dlopenerror
+from pypy.module._rawffi import lasterror
 
 import os
 if os.name == 'nt':
@@ -201,11 +202,23 @@
         self.func = func
         self.argchain = argchain
 
+    def before(self):
+        lasterror.restore_last_error(self.space)
+
+    def after(self):
+        lasterror.save_last_error(self.space)
+
     def get_longlong(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.LONGLONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.LONGLONG)
+        self.after()
+        return x
 
     def get_ulonglong(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.ULONGLONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.ULONGLONG)
+        self.after()
+        return x
 
     def get_signed(self, w_ffitype):
         # if the declared return type of the function is smaller than LONG,
@@ -216,64 +229,94 @@
         # to space.wrap in order to get a nice applevel <int>.
         #
         restype = w_ffitype.get_ffitype()
+        self.before()
         call = self.func.call
         if restype is libffi.types.slong:
-            return call(self.argchain, rffi.LONG)
+            x = call(self.argchain, rffi.LONG)
         elif restype is libffi.types.sint:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.INT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.INT))
         elif restype is libffi.types.sshort:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT))
         elif restype is libffi.types.schar:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR))
         else:
-            self.error(w_ffitype)
+            raise self.error(w_ffitype)
+        self.after()
+        return x
 
     def get_unsigned(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.ULONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.ULONG)
+        self.after()
+        return x
 
     def get_unsigned_which_fits_into_a_signed(self, w_ffitype):
         # the same comment as get_signed apply
         restype = w_ffitype.get_ffitype()
+        self.before()
         call = self.func.call
         if restype is libffi.types.uint:
             assert not libffi.IS_32_BIT
             # on 32bit machines, we should never get here, because it's a case
             # which has already been handled by get_unsigned above.
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT))
         elif restype is libffi.types.ushort:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT))
         elif restype is libffi.types.uchar:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR))
         else:
-            self.error(w_ffitype)
+            raise self.error(w_ffitype)
+        self.after()
+        return x
 
 
     def get_pointer(self, w_ffitype):
+        self.before()
         ptrres = self.func.call(self.argchain, rffi.VOIDP)
+        self.after()
         return rffi.cast(rffi.ULONG, ptrres)
 
     def get_char(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.UCHAR)
+        self.before()
+        x = self.func.call(self.argchain, rffi.UCHAR)
+        self.after()
+        return x
 
     def get_unichar(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.WCHAR_T)
+        self.before()
+        x = self.func.call(self.argchain, rffi.WCHAR_T)
+        self.after()
+        return x
 
     def get_float(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.DOUBLE)
+        self.before()
+        x = self.func.call(self.argchain, rffi.DOUBLE)
+        self.after()
+        return x
 
     def get_singlefloat(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.FLOAT)
+        self.before()
+        x = self.func.call(self.argchain, rffi.FLOAT)
+        self.after()
+        return x
 
     def get_struct(self, w_ffitype, w_structdescr):
+        self.before()
         addr = self.func.call(self.argchain, rffi.LONG, is_struct=True)
+        self.after()
         return w_structdescr.fromaddress(self.space, addr)
 
     def get_struct_rawffi(self, w_ffitype, w_structdescr):
+        self.before()
         uintval = self.func.call(self.argchain, rffi.ULONG, is_struct=True)
+        self.after()
         return w_structdescr.fromaddress(self.space, uintval)
 
     def get_void(self, w_ffitype):
-        return self.func.call(self.argchain, lltype.Void)
+        self.before()
+        x = self.func.call(self.argchain, lltype.Void)
+        self.after()
+        return x
 
 
 def unpack_argtypes(space, w_argtypes, w_restype):
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -18,6 +18,7 @@
 from rpython.rlib.rarithmetic import intmask, r_uint
 from pypy.module._rawffi.buffer import RawFFIBuffer
 from pypy.module._rawffi.tracker import tracker
+from pypy.module._rawffi import lasterror
 
 TYPEMAP = {
     # XXX A mess with unsigned/signed/normal chars :-/
@@ -495,10 +496,14 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                lasterror.restore_last_error(space)
                 self.ptr.call(args_ll, result.ll_buffer)
+                lasterror.save_last_error(space)
                 return space.wrap(result)
             else:
+                lasterror.restore_last_error(space)
                 self.ptr.call(args_ll, lltype.nullptr(rffi.VOIDP.TO))
+                lasterror.save_last_error(space)
                 return space.w_None
         except StackCheckError, e:
             raise OperationError(space.w_ValueError, space.wrap(e.message))

From noreply at buildbot.pypy.org  Wed Dec 10 00:02:46 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Dec 2014 00:02:46 +0100 (CET)
Subject: [pypy-commit] pypy default: add missing file
Message-ID: <20141209230246.9B3B51D28EE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74873:33de11afa62a
Date: 2014-12-09 23:02 +0000
http://bitbucket.org/pypy/pypy/changeset/33de11afa62a/

Log:	add missing file

diff --git a/pypy/module/_rawffi/lasterror.py b/pypy/module/_rawffi/lasterror.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_rawffi/lasterror.py
@@ -0,0 +1,32 @@
+# For Windows only.
+# https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
+
+import os
+
+_MS_WINDOWS = os.name == "nt"
+
+
+if _MS_WINDOWS:
+    from rpython.rlib import rwin32
+    from pypy.interpreter.executioncontext import Executioncontext
+
+
+    ExecutionContext._rawffi_last_error = 0
+
+    def restore_last_error(space):
+        ec = space.getexecutioncontext()
+        lasterror = ec._rawffi_last_error
+        rwin32.SetLastError(lasterror)
+
+    def save_last_error(space):
+        lasterror = rwin32.GetLastError()
+        ec = space.getexecutioncontext()
+        ec._rawffi_last_error = lasterror
+
+else:
+
+    def restore_last_error(space):
+        pass
+
+    def save_last_error(space):
+        pass

From noreply at buildbot.pypy.org  Wed Dec 10 00:17:31 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Dec 2014 00:17:31 +0100 (CET)
Subject: [pypy-commit] pypy default: fixes
Message-ID: <20141209231731.251F21C09B2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74874:99234fd4cade
Date: 2014-12-09 23:17 +0000
http://bitbucket.org/pypy/pypy/changeset/99234fd4cade/

Log:	fixes

diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -620,12 +620,10 @@
 
 if sys.platform == 'win32':
     def get_last_error(space):
-        from rpython.rlib.rwin32 import GetLastError
-        return space.wrap(GetLastError())
+        return space.wrap(lasterror.fetch_last_error(space))
     @unwrap_spec(error=int)
     def set_last_error(space, error):
-        from rpython.rlib.rwin32 import SetLastError
-        SetLastError(error)
+        lasterror.store_last_error(space, error)
 else:
     # always have at least a dummy version of these functions
     # (https://bugs.pypy.org/issue1242)
diff --git a/pypy/module/_rawffi/lasterror.py b/pypy/module/_rawffi/lasterror.py
--- a/pypy/module/_rawffi/lasterror.py
+++ b/pypy/module/_rawffi/lasterror.py
@@ -8,11 +8,19 @@
 
 if _MS_WINDOWS:
     from rpython.rlib import rwin32
-    from pypy.interpreter.executioncontext import Executioncontext
+    from pypy.interpreter.executioncontext import ExecutionContext
 
 
     ExecutionContext._rawffi_last_error = 0
 
+    def fetch_last_error(space):
+        ec = space.getexecutioncontext()
+        return ec._rawffi_last_error
+
+    def store_last_error(space, last_error):
+        ec = space.getexecutioncontext()
+        ec._rawffi_last_error = last_error
+
     def restore_last_error(space):
         ec = space.getexecutioncontext()
         lasterror = ec._rawffi_last_error

From noreply at buildbot.pypy.org  Wed Dec 10 01:16:49 2014
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 10 Dec 2014 01:16:49 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: make sure that the overridden
 PyFrame.execute_frame is seen before we start translation
Message-ID: <20141210001649.087F51C09B2@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: vmprof
Changeset: r74875:794e9137937b
Date: 2014-12-10 00:16 +0000
http://bitbucket.org/pypy/pypy/changeset/794e9137937b/

Log:	make sure that the overridden PyFrame.execute_frame is seen before
	we start translation

diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -12,3 +12,7 @@
         'enable': 'interp_vmprof.enable',
         'disable': 'interp_vmprof.disable',
     }
+
+    def setup_after_space_initialization(self):
+        # force the __extend__ hacks to occur early
+        import pypy.module._vmprof.interp_vmprof

From noreply at buildbot.pypy.org  Wed Dec 10 10:26:58 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Dec 2014 10:26:58 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Add ICBD as a benchmark. This
 comes from Kevin Modzelewski on pypy-dev.
Message-ID: <20141210092658.58C541D3816@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r292:67706c915134
Date: 2014-12-10 09:15 +0000
http://bitbucket.org/pypy/benchmarks/changeset/67706c915134/

Log:	Add ICBD as a benchmark. This comes from Kevin Modzelewski on pypy-
	dev.

diff too long, truncating to 2000 out of 90238 lines

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -82,7 +82,8 @@
              'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast',
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
              'json_bench', 'pidigits', 'hexiom2', 'eparse', 'deltablue',
-             'bm_dulwich_log', 'bm_krakatau', 'bm_mdp', 'pypy_interp']:
+             'bm_dulwich_log', 'bm_krakatau', 'bm_mdp', 'pypy_interp',
+             'bm_icbd']:
     _register_new_bm(name, name, globals(), **opts.get(name, {}))
 
 for name in ['names', 'iteration', 'tcp', 'pb', ]:#'web']:#, 'accepts']:
diff --git a/own/bm_icbd.py b/own/bm_icbd.py
new file mode 100644
--- /dev/null
+++ b/own/bm_icbd.py
@@ -0,0 +1,35 @@
+import sys, os, subprocess, time
+this_dir = os.path.abspath(os.path.dirname(__file__))
+
+def main(n):
+    # HAAAAAACK with subprocess because I can't get the thing
+    # to run in-process :-(  As a result, it doesn't make
+    # sense to run it more than once.
+
+    d = os.environ.copy()
+    d['PYTHONPATH'] = 'icbd'
+    t0 = time.time()
+    popen = subprocess.Popen(
+                     [sys.executable,
+                      '-m', 'icbd.type_analyzer.analyze_all',
+                      '-I', 'stdlib/python2.5_tiny',
+                      '-I', '.',
+                      '-E', 'icbd/type_analyzer/tests',
+                      '-E', 'icbd/compiler/benchmarks',
+                      '-E', 'icbd/compiler/tests',
+                      '-I', 'stdlib/type_mocks',
+                      '-n',
+                      'icbd'], cwd=this_dir, env=d, stdout=subprocess.PIPE)
+    popen.communicate()
+    time_elapsed = time.time() - t0
+    return [time_elapsed] * n
+
+if __name__ == "__main__":
+    import util, optparse
+    parser = optparse.OptionParser(
+        usage="%prog [options]",
+        description="Test the performance of the ICBD benchmark")
+    util.add_standard_options_to(parser)
+    options, args = parser.parse_args()
+
+    util.run_benchmark(options, options.num_runs, main)
diff --git a/own/icbd/.gitignore b/own/icbd/.gitignore
new file mode 100644
--- /dev/null
+++ b/own/icbd/.gitignore
@@ -0,0 +1,24 @@
+*.pyc
+*.d
+*.sw?
+*.out
+*.out[0-9]
+*.o
+*.s
+*.s[0-9]
+*.gen.bc
+*.gen.c
+*.comb.ll
+*.gen.ll
+*opt.ll
+*.so
+icbd/compiler/tests/[0-9]
+icbd/compiler/tests/[0-9][0-9]
+icbd/compiler/tests/*.hpp
+icbd/compiler/tests/*[0-9].cpp
+icbd/compiler/tests/Makefile
+*.shed
+*.ppm
+icbd/compiler/dist
+*.pyicbd
+*.pyd
diff --git a/own/icbd/LICENSE b/own/icbd/LICENSE
new file mode 100644
--- /dev/null
+++ b/own/icbd/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014, Kevin Modzelewski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/own/icbd/README.md b/own/icbd/README.md
new file mode 100644
--- /dev/null
+++ b/own/icbd/README.md
@@ -0,0 +1,109 @@
+# icbd
+
+> ICBD is a defunct project, but I'm releasing the source code in case any one is interested.
+
+ICBD is a static type analyzer and static compiler for Python.  It originated
+from the desire at Dropbox to have additional type information for our growing
+codebases, without requiring programmers to add type annotations.  There are
+other static type analyzers for Python out there; the goal of ICBD was to
+produce one that would work well on the Dropbox codebase, which meant
+concretely:
+
+- Handling untypable code as gracefully as possible
+- "Acceptable" running times for multi-100k LOC codebases
+- A plugin system for handling code that is inherently not statically analyzable
+
+These goals were somewhat met, perhaps to the limits of using whole-program type
+inference; the running time for the Dropbox codebase was about 15 minutes, which
+I believe is better than alternatives but is not good enough for easy use.
+
+The plugin system was able to get anecdotally get ICBD to successfully analyze a
+sufficient fraction of the Dropbox codebase, though when typing failed, it was
+difficult to tell why.
+
+### Testing it out
+
+You need to install pygments and simplejson to run ICBD; once you do, run
+
+```
+bash icbd/type_analyzer/run.sh icbd
+```
+
+to have ICBD analyze its own codebase (it doesn't do particularly well since
+there are no plugins for its getattr usage).  You can see the output at
+file:///tmp/icbd_icbd/icbd/type_analyzer/type_checker.html
+
+### Technical approach
+
+ICBD treats every Python expression as a constraint on the allowable types of
+the program: `a = o.x` encodes the constraint that "a will have the type of o's
+x attribute", `f(x)` encodes the constaint that "f accepts a type argument of
+type type(x)", etc.  At a high level, ICBD then does a graph fixed-point
+analysis to determine a set of types that satisfy the constraints, and emits
+errors when types could not be found (ex: `a, b, c = 1, 2`).
+
+The guessed types start at the BOTTOM type, and ICBD iteratively finds
+unsatisfied constraints and raises the involved types to attempt to satisfy the
+constraints.  In theory, if ICBD always raises types monotonically and to the
+minimum satisfying type, this process will converge on a deterministic set of
+lowest (or in some sense, most descriptive) types.
+
+In practice, this is somewhat harder because the type constraints are dependent
+on other types -- for example, attribute lookup dependencies can change based on
+learning that an attribute may actually be a instance attribute.  This doesn't
+seem to be that big a deal, though.
+
+### Comparison to Python type analyzers
+
+ICBD only analyzes a single version of each function, and assumes that any
+combination of seen argument types could occur together.  For example, in this
+snippet:
+
+```python
+def add_int_or_str(a, b):
+    return a + b
+
+add_int_or_str(1, 2)
+add_int_or_str("hello ", "world")
+```
+
+ICBD will report an error, saying that an int cannot be added to a string.
+Other analyzers will notice that add_int_or_str does not actually receive an
+(int, str) combination of arguments and handle this correctly, but at an
+exponentially-large cost.
+
+This means that ICBD does not handle polymorphic functions very gracefully,
+but in practice this seems to net be a common thing for applications code, and
+the plugin system makes it possible to handle on a one-off basis.
+
+#### Quick note about HM
+
+People often ask "why don't you just apply HM (Hindley-Milner) which computes
+the types ideally".  Yes, it's true that HM has some nice ideality properties,
+but it only applies to a fairly-restrictive type system.  I'm not a type
+theorist, but my understanding is that to properly represent the Python type
+system you need
+
+- Dependent types to represent attributes, and
+- Rank-n polymorphism to represent global variables
+
+both of which I believe make the inference problem undecidable.  (Is there
+really any Haskell program that uses 0 type annotations?)
+
+Of course in theory it's possible to translate Python code into, for example,
+Haskell, but I don't see any reason to believe this can be done in an automated
+way such that the generated Haskell has any meaningful relation to the original
+Python.  For example, one could build an X86 emulator in Haskell, and use that
+to run the Python interpreter, but the types on the Haskell program have no
+usefulness to the original Python programmer.
+
+# Compiler
+
+ICBD also includes a static compiler, in icbd/compiler.  Assuming that the type
+analyzer can come up with a complete set of types for your program (a big
+assumption), in theory it should be possible to compile the Python as if it were
+a more traditionally-typed static language.
+
+In practice I doubt that real code can really be 100% typed; the ICBD compiler
+served as the starting point for the [Pyston](https://github.com/dropbox/pyston)
+project, which replaces it.
diff --git a/own/icbd/demos/demo.py b/own/icbd/demos/demo.py
new file mode 100644
--- /dev/null
+++ b/own/icbd/demos/demo.py
@@ -0,0 +1,291 @@
+if 0:
+                                                        ""
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        # Basic type analysis:
+                                                        a = 2
+                                                        b = '3'
+                                                        d = {a:b}
+                                                        ch = ord(b[2].lower()[0])
+                                                        print ch
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+
+
+
+
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        # Name errors:
+                                                        ch *= e
+                                                        
+
+
+
+
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        # Nested-type manipulation:
+                                                        d = {1:[1], 3:[4]}
+                                                        d[1].append(2)
+                                                        
+                                                        l = [range(i) for i in xrange(5)]
+                                                        l.append([-1])
+                                                        l2 = [k[0] for k in l]
+                                                        l2.append(1)
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+
+
+
+
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        # Function type inference:
+                                                        def func(x, y):
+                                                            return b * (x + y)
+                                                        r = func(3, 4)
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+
+
+
+
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        # Higher-level types:
+                                                        def f(x):
+                                                            def g():
+                                                                return x
+                                                            return g
+                                                        
+                                                        inner = f(2)
+                                                        print f(1)() + inner()
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+
+
+
+
+                                                        
+                                                        
+                                                        # Control-flow analysis:
+                                                        if inner():
+                                                            x = 3
+                                                        else:
+                                                            x = [0]
+                                                        print l2[x]
+                                                        
+                                                        def fib(x):
+                                                            if x <= 1:
+                                                                return x
+                                                            return fib(x-1) + fib(x-2)
+                                                        fib(2)
+                                                        
+                                                        
+                                                        
+
+
+
+
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        def foo():
+                                                            return foo()
+                                                        if 0:
+                                                            x = 0
+                                                        else:
+                                                            x = foo()
+                                                        print x
+                                                        
+                                                        
+                                                        
+                                                        
+
+
+
+
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        # Classes:
+                                                        class C(object):
+                                                            def __init__(self, x):
+                                                                self.x = x
+                                                            def bar(self, z):
+                                                                return self.x * z
+                                                            def baz(self, k):
+                                                                return k**2
+                                                        
+                                                        c = C(2)
+                                                        c.baz(3)
+                                                        
+                                                        
+                                                        
+                                                        
+
+
+
+
+                                                        
+                                                        
+                                                        # attributes
+                                                        print c.x
+                                                        z = c.bar(3)
+                                                        c.y = 3
+                                                        z = c.x * c.y
+                                                        z *= c.z
+                                                        
+                                                        
+                                                        
+
+
+
+
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        
+                                                        # Complex type analysis:
+                                                        def f1(x):
+                                                            def g(y):
+                                                                return x * y
+                                                            return g
+                                                        
+                                                        a = f1('')
+                                                        b = a(2)
+                                                        
+                                                        def f2(x):
+                                                            def g(y):
+                                                                return x * y
+                                                            return g
+                                                        
+                                                        a = f2(1)
+                                                        b = a(2)
+
+
+
+
+
+
+                                                        ""
diff --git a/own/icbd/demos/demo_compact.py b/own/icbd/demos/demo_compact.py
new file mode 100644
--- /dev/null
+++ b/own/icbd/demos/demo_compact.py
@@ -0,0 +1,105 @@
+# Basic type analysis:
+a = 2
+b = '3'
+d = {a:b}
+c = ord(b[2].lower()[0])
+print c
+
+
+# Name errors:
+c *= e
+
+# Nested-type manipulation:
+d = {1:[1], 3:[4]}
+d[1].append(2)
+
+l = [range(i) for i in xrange(5)]
+l.append([-1])
+l2 = [k[0] for k in l]
+l2.append(1)
+
+
+# Argument-type checking, based on inferred types
+l2.append('')
+
+
+
+# Function type inference:
+def f(x, y):
+    return b * (x + y)
+r = f(3, 4)
+
+
+
+# Higher-level types:
+def f(x):
+    def g():
+        return x
+    return g
+
+f1 = f(2)
+print f(1)() + f1()
+
+
+
+
+# Control-flow analysis:
+if f1():
+    x = 3
+else:
+    x = [0]
+print l2[x]
+
+def fib(x):
+    if x <= 1:
+        return x
+    return fib(x-1) + fib(x-2)
+fib(2)
+
+
+def foo():
+    return foo()
+if 0:
+    x = 0
+else:
+    x = foo()
+print x
+
+
+# Classes:
+class Foo(object):
+    def __init__(self, x):
+        self.x = x
+    def bar(self, z):
+        return self.x * z
+    def baz(self, k):
+        return k**2
+
+f = Foo(2)
+f.baz(3)
+
+# attributes
+print f.x
+z = f.bar(3)
+f.y = 3
+z = f.x * f.y
+z *= f.z
+
+
+# Complex type analysis:
+def f1(x):
+    def g(y):
+        return x * y
+    return g
+
+a = f1('')
+b = a(2)
+
+def f2(x):
+    def g(y):
+        return x * y
+    return g
+
+a = f2(1)
+b = a(2)
+
diff --git a/own/icbd/demos/ray.html b/own/icbd/demos/ray.html
new file mode 100644
--- /dev/null
+++ b/own/icbd/demos/ray.html
@@ -0,0 +1,15385 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <title>Emscripten-Generated Code</title>
+    <style>
+      .emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; }
+      canvas.emscripten { border: 1px solid black; }
+      textarea.emscripten { font-family: monospace; width: 80%; }
+      div.emscripten { text-align: center; }
+    </style>
+  </head>
+  <body>
+    <hr/>
+    <div class="emscripten" id="status">Downloading...</div>
+    <canvas class="emscripten" id="canvas" oncontextmenu="event.preventDefault()"></canvas>
+    <hr/>
+    <div class="emscripten"><input type="button" value="fullscreen" onclick="Module.requestFullScreen()"></div>
+    <hr/>
+    <textarea class="emscripten" id="output" rows="8"></textarea>
+    <hr>
+    <script type='text/javascript'>
+      // connect to canvas
+      var Module = {
+        preRun: [],
+        postRun: [],
+        print: (function() {
+          var element = document.getElementById('output');
+          element.value = ''; // clear browser cache
+          return function(text) {
+            // These replacements are necessary if you render to raw HTML
+            //text = text.replace(/&/g, "&amp;");
+            //text = text.replace(/</g, "&lt;");
+            //text = text.replace(/>/g, "&gt;");
+            //text = text.replace('\n', '<br>', 'g');
+            element.value += text + "\n";
+            element.scrollTop = 99999; // focus on bottom
+          };
+        })(),
+        printErr: function(text) {
+          if (0) { // XXX disabled for safety typeof dump == 'function') {
+            dump(text + '\n'); // fast, straight to the real console
+          } else {
+            console.log(text);
+          }
+        },
+        canvas: document.getElementById('canvas'),
+        setStatus: function(text) {
+          if (Module.setStatus.interval) clearInterval(Module.setStatus.interval);
+          document.getElementById('status').innerHTML = text;
+          if (text) {
+            var counter = 0;
+            Module.setStatus.interval = setInterval(function() {
+              counter++;
+              counter %= 3;
+              var dots = ' ';
+              for (var i = 0; i < counter; i++) dots += '.';
+              dots += '*';
+              for (var i = counter; i < 2; i++) dots += '.';
+              document.getElementById('status').innerHTML = text.replace('...', dots);
+            }, 300);
+          }
+        },
+        totalDependencies: 0,
+        monitorRunDependencies: function(left) {
+          this.totalDependencies = Math.max(this.totalDependencies, left);
+          Module.setStatus(left ? 'Preparing: ' + (this.totalDependencies-left) + '/' + this.totalDependencies + '...' : 'All downloads complete.');
+        }
+      };
+      Module.setStatus('Downloading...');
+    </script>      
+    <script type='text/javascript'>
+
+      // Note: Some Emscripten settings will significantly limit the speed of the generated code.
+// Note: Some Emscripten settings may limit the speed of the generated code.
+// TODO: " u s e   s t r i c t ";
+
+try {
+  this['Module'] = Module;
+} catch(e) {
+  this['Module'] = Module = {};
+}
+
+// The environment setup code below is customized to use Module.
+// *** Environment setup code ***
+var ENVIRONMENT_IS_NODE = typeof process === 'object';
+var ENVIRONMENT_IS_WEB = typeof window === 'object';
+var ENVIRONMENT_IS_WORKER = typeof importScripts === 'function';
+var ENVIRONMENT_IS_SHELL = !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER;
+
+if (ENVIRONMENT_IS_NODE) {
+  // Expose functionality in the same simple way that the shells work
+  // Note that we pollute the global namespace here, otherwise we break in node
+  Module['print'] = function(x) {
+    process['stdout'].write(x + '\n');
+  };
+  Module['printErr'] = function(x) {
+    process['stderr'].write(x + '\n');
+  };
+
+  var nodeFS = require('fs');
+  var nodePath = require('path');
+
+  Module['read'] = function(filename) {
+    filename = nodePath['normalize'](filename);
+    var ret = nodeFS['readFileSync'](filename).toString();
+    // The path is absolute if the normalized version is the same as the resolved.
+    if (!ret && filename != nodePath['resolve'](filename)) {
+      filename = path.join(__dirname, '..', 'src', filename);
+      ret = nodeFS['readFileSync'](filename).toString();
+    }
+    return ret;
+  };
+
+  Module['load'] = function(f) {
+    globalEval(read(f));
+  };
+
+  if (!Module['arguments']) {
+    Module['arguments'] = process['argv'].slice(2);
+  }
+} else if (ENVIRONMENT_IS_SHELL) {
+  Module['print'] = print;
+  if (typeof printErr != 'undefined') Module['printErr'] = printErr; // not present in v8 or older sm
+
+  // Polyfill over SpiderMonkey/V8 differences
+  if (typeof read != 'undefined') {
+    Module['read'] = read;
+  } else {
+    Module['read'] = function(f) { snarf(f) };
+  }
+
+  if (!Module['arguments']) {
+    if (typeof scriptArgs != 'undefined') {
+      Module['arguments'] = scriptArgs;
+    } else if (typeof arguments != 'undefined') {
+      Module['arguments'] = arguments;
+    }
+  }
+} else if (ENVIRONMENT_IS_WEB) {
+  if (!Module['print']) {
+    Module['print'] = function(x) {
+      console.log(x);
+    };
+  }
+
+  if (!Module['printErr']) {
+    Module['printErr'] = function(x) {
+      console.log(x);
+    };
+  }
+
+  Module['read'] = function(url) {
+    var xhr = new XMLHttpRequest();
+    xhr.open('GET', url, false);
+    xhr.send(null);
+    return xhr.responseText;
+  };
+
+  if (!Module['arguments']) {
+    if (typeof arguments != 'undefined') {
+      Module['arguments'] = arguments;
+    }
+  }
+} else if (ENVIRONMENT_IS_WORKER) {
+  // We can do very little here...
+
+  Module['load'] = importScripts;
+
+} else {
+  throw 'Unknown runtime environment. Where are we?';
+}
+
+function globalEval(x) {
+  eval.call(null, x);
+}
+if (!Module['load'] == 'undefined' && Module['read']) {
+  Module['load'] = function(f) {
+    globalEval(Module['read'](f));
+  };
+}
+if (!Module['print']) {
+  Module['print'] = function(){};
+}
+if (!Module['printErr']) {
+  Module['printErr'] = Module['print'];
+}
+if (!Module['arguments']) {
+  Module['arguments'] = [];
+}
+// *** Environment setup code ***
+
+// Closure helpers
+Module.print = Module['print'];
+Module.printErr = Module['printErr'];
+
+// Callbacks
+if (!Module['preRun']) Module['preRun'] = [];
+if (!Module['postRun']) Module['postRun'] = [];
+
+  
+// === Auto-generated preamble library stuff ===
+
+//========================================
+// Runtime code shared with compiler
+//========================================
+
+var Runtime = {
+  stackSave: function () {
+    return STACKTOP;
+  },
+  stackRestore: function (stackTop) {
+    STACKTOP = stackTop;
+  },
+  forceAlign: function (target, quantum) {
+    quantum = quantum || 4;
+    if (quantum == 1) return target;
+    if (isNumber(target) && isNumber(quantum)) {
+      return Math.ceil(target/quantum)*quantum;
+    } else if (isNumber(quantum) && isPowerOfTwo(quantum)) {
+      var logg = log2(quantum);
+      return '((((' +target + ')+' + (quantum-1) + ')>>' + logg + ')<<' + logg + ')';
+    }
+    return 'Math.ceil((' + target + ')/' + quantum + ')*' + quantum;
+  },
+  isNumberType: function (type) {
+    return type in Runtime.INT_TYPES || type in Runtime.FLOAT_TYPES;
+  },
+  isPointerType: function isPointerType(type) {
+  return type[type.length-1] == '*';
+},
+  isStructType: function isStructType(type) {
+  if (isPointerType(type)) return false;
+  if (/^\[\d+\ x\ (.*)\]/.test(type)) return true; // [15 x ?] blocks. Like structs
+  if (/<?{ ?[^}]* ?}>?/.test(type)) return true; // { i32, i8 } etc. - anonymous struct types
+  // See comment in isStructPointerType()
+  return type[0] == '%';
+},
+  INT_TYPES: {"i1":0,"i8":0,"i16":0,"i32":0,"i64":0},
+  FLOAT_TYPES: {"float":0,"double":0},
+  bitshift64: function (low, high, op, bits) {
+    var ander = Math.pow(2, bits)-1;
+    if (bits < 32) {
+      switch (op) {
+        case 'shl':
+          return [low << bits, (high << bits) | ((low&(ander << (32 - bits))) >>> (32 - bits))];
+        case 'ashr':
+          return [(((low >>> bits ) | ((high&ander) << (32 - bits))) >> 0) >>> 0, (high >> bits) >>> 0];
+        case 'lshr':
+          return [((low >>> bits) | ((high&ander) << (32 - bits))) >>> 0, high >>> bits];
+      }
+    } else if (bits == 32) {
+      switch (op) {
+        case 'shl':
+          return [0, low];
+        case 'ashr':
+          return [high, (high|0) < 0 ? ander : 0];
+        case 'lshr':
+          return [high, 0];
+      }
+    } else { // bits > 32
+      switch (op) {
+        case 'shl':
+          return [0, low << (bits - 32)];
+        case 'ashr':
+          return [(high >> (bits - 32)) >>> 0, (high|0) < 0 ? ander : 0];
+        case 'lshr':
+          return [high >>>  (bits - 32) , 0];
+      }
+    }
+    abort('unknown bitshift64 op: ' + [value, op, bits]);
+  },
+  or64: function (x, y) {
+    var l = (x | 0) | (y | 0);
+    var h = (Math.round(x / 4294967296) | Math.round(y / 4294967296)) * 4294967296;
+    return l + h;
+  },
+  and64: function (x, y) {
+    var l = (x | 0) & (y | 0);
+    var h = (Math.round(x / 4294967296) & Math.round(y / 4294967296)) * 4294967296;
+    return l + h;
+  },
+  xor64: function (x, y) {
+    var l = (x | 0) ^ (y | 0);
+    var h = (Math.round(x / 4294967296) ^ Math.round(y / 4294967296)) * 4294967296;
+    return l + h;
+  },
+  getNativeTypeSize: function (type, quantumSize) {
+    if (Runtime.QUANTUM_SIZE == 1) return 1;
+    var size = {
+      '%i1': 1,
+      '%i8': 1,
+      '%i16': 2,
+      '%i32': 4,
+      '%i64': 8,
+      "%float": 4,
+      "%double": 8
+    }['%'+type]; // add '%' since float and double confuse Closure compiler as keys, and also spidermonkey as a compiler will remove 's from '_i8' etc
+    if (!size) {
+      if (type[type.length-1] == '*') {
+        size = Runtime.QUANTUM_SIZE; // A pointer
+      } else if (type[0] == 'i') {
+        var bits = parseInt(type.substr(1));
+        assert(bits % 8 == 0);
+        size = bits/8;
+      }
+    }
+    return size;
+  },
+  getNativeFieldSize: function (type) {
+    return Math.max(Runtime.getNativeTypeSize(type), Runtime.QUANTUM_SIZE);
+  },
+  dedup: function dedup(items, ident) {
+  var seen = {};
+  if (ident) {
+    return items.filter(function(item) {
+      if (seen[item[ident]]) return false;
+      seen[item[ident]] = true;
+      return true;
+    });
+  } else {
+    return items.filter(function(item) {
+      if (seen[item]) return false;
+      seen[item] = true;
+      return true;
+    });
+  }
+},
+  set: function set() {
+  var args = typeof arguments[0] === 'object' ? arguments[0] : arguments;
+  var ret = {};
+  for (var i = 0; i < args.length; i++) {
+    ret[args[i]] = 0;
+  }
+  return ret;
+},
+  calculateStructAlignment: function calculateStructAlignment(type) {
+    type.flatSize = 0;
+    type.alignSize = 0;
+    var diffs = [];
+    var prev = -1;
+    type.flatIndexes = type.fields.map(function(field) {
+      var size, alignSize;
+      if (Runtime.isNumberType(field) || Runtime.isPointerType(field)) {
+        size = Runtime.getNativeTypeSize(field); // pack char; char; in structs, also char[X]s.
+        alignSize = size;
+      } else if (Runtime.isStructType(field)) {
+        size = Types.types[field].flatSize;
+        alignSize = Types.types[field].alignSize;
+      } else {
+        throw 'Unclear type in struct: ' + field + ', in ' + type.name_ + ' :: ' + dump(Types.types[type.name_]);
+      }
+      alignSize = type.packed ? 1 : Math.min(alignSize, Runtime.QUANTUM_SIZE);
+      type.alignSize = Math.max(type.alignSize, alignSize);
+      var curr = Runtime.alignMemory(type.flatSize, alignSize); // if necessary, place this on aligned memory
+      type.flatSize = curr + size;
+      if (prev >= 0) {
+        diffs.push(curr-prev);
+      }
+      prev = curr;
+      return curr;
+    });
+    type.flatSize = Runtime.alignMemory(type.flatSize, type.alignSize);
+    if (diffs.length == 0) {
+      type.flatFactor = type.flatSize;
+    } else if (Runtime.dedup(diffs).length == 1) {
+      type.flatFactor = diffs[0];
+    }
+    type.needsFlattening = (type.flatFactor != 1);
+    return type.flatIndexes;
+  },
+  generateStructInfo: function (struct, typeName, offset) {
+    var type, alignment;
+    if (typeName) {
+      offset = offset || 0;
+      type = (typeof Types === 'undefined' ? Runtime.typeInfo : Types.types)[typeName];
+      if (!type) return null;
+      assert(type.fields.length === struct.length, 'Number of named fields must match the type for ' + typeName);
+      alignment = type.flatIndexes;
+    } else {
+      var type = { fields: struct.map(function(item) { return item[0] }) };
+      alignment = Runtime.calculateStructAlignment(type);
+    }
+    var ret = {
+      __size__: type.flatSize
+    };
+    if (typeName) {
+      struct.forEach(function(item, i) {
+        if (typeof item === 'string') {
+          ret[item] = alignment[i] + offset;
+        } else {
+          // embedded struct
+          var key;
+          for (var k in item) key = k;
+          ret[key] = Runtime.generateStructInfo(item[key], type.fields[i], alignment[i]);
+        }
+      });
+    } else {
+      struct.forEach(function(item, i) {
+        ret[item[1]] = alignment[i];
+      });
+    }
+    return ret;
+  },
+  addFunction: function (func) {
+    var ret = FUNCTION_TABLE.length;
+    FUNCTION_TABLE.push(func);
+    FUNCTION_TABLE.push(0);
+    return ret;
+  },
+  warnOnce: function (text) {
+    if (!Runtime.warnOnce.shown) Runtime.warnOnce.shown = {};
+    if (!Runtime.warnOnce.shown[text]) {
+      Runtime.warnOnce.shown[text] = 1;
+      Module.printErr(text);
+    }
+  },
+  funcWrappers: {},
+  getFuncWrapper: function (func) {
+    if (!Runtime.funcWrappers[func]) {
+      Runtime.funcWrappers[func] = function() {
+        FUNCTION_TABLE[func].apply(null, arguments);
+      };
+    }
+    return Runtime.funcWrappers[func];
+  },
+  UTF8Processor: function () {
+    var buffer = [];
+    var needed = 0;
+    this.processCChar = function (code) {
+      code = code & 0xff;
+      if (needed) {
+        buffer.push(code);
+        needed--;
+      }
+      if (buffer.length == 0) {
+        if (code < 128) return String.fromCharCode(code);
+        buffer.push(code);
+        if (code > 191 && code < 224) {
+          needed = 1;
+        } else {
+          needed = 2;
+        }
+        return '';
+      }
+      if (needed > 0) return '';
+      var c1 = buffer[0];
+      var c2 = buffer[1];
+      var c3 = buffer[2];
+      var ret;
+      if (c1 > 191 && c1 < 224) {
+        ret = String.fromCharCode(((c1 & 31) << 6) | (c2 & 63));
+      } else {
+        ret = String.fromCharCode(((c1 & 15) << 12) | ((c2 & 63) << 6) | (c3 & 63));
+      }
+      buffer.length = 0;
+      return ret;
+    }
+    this.processJSString = function(string) {
+      string = unescape(encodeURIComponent(string));
+      var ret = [];
+      for (var i = 0; i < string.length; i++) {
+        ret.push(string.charCodeAt(i));
+      }
+      return ret;
+    }
+  },
+  stackAlloc: function stackAlloc(size) { var ret = STACKTOP;STACKTOP += size;STACKTOP = ((((STACKTOP)+3)>>2)<<2);assert(STACKTOP < STACK_ROOT + STACK_MAX, "Ran out of stack"); return ret; },
+  staticAlloc: function staticAlloc(size) { var ret = STATICTOP;STATICTOP += size;STATICTOP = ((((STATICTOP)+3)>>2)<<2); if (STATICTOP >= TOTAL_MEMORY) enlargeMemory();; return ret; },
+  alignMemory: function alignMemory(size,quantum) { var ret = size = Math.ceil((size)/(quantum ? quantum : 4))*(quantum ? quantum : 4); return ret; },
+  makeBigInt: function makeBigInt(low,high,unsigned) { var ret = (unsigned ? (((low)>>>0)+(((high)>>>0)*4294967296)) : (((low)>>>0)+(((high)|0)*4294967296))); return ret; },
+  QUANTUM_SIZE: 4,
+  __dummy__: 0
+}
+
+
+
+var CorrectionsMonitor = {
+  MAX_ALLOWED: 0, // XXX
+  corrections: 0,
+  sigs: {},
+
+  note: function(type, succeed, sig) {
+    if (!succeed) {
+      this.corrections++;
+      if (this.corrections >= this.MAX_ALLOWED) abort('\n\nToo many corrections!');
+    }
+  },
+
+  print: function() {
+  }
+};
+
+
+
+
+
+//========================================
+// Runtime essentials
+//========================================
+
+var __THREW__ = false; // Used in checking for thrown exceptions.
+
+var ABORT = false;
+
+var undef = 0;
+// tempInt is used for 32-bit signed values or smaller. tempBigInt is used
+// for 32-bit unsigned values or more than 32 bits. TODO: audit all uses of tempInt
+var tempValue, tempInt, tempBigInt, tempInt2, tempBigInt2, tempPair, tempBigIntI, tempBigIntR, tempBigIntS, tempBigIntP, tempBigIntD;
+var tempI64, tempI64b;
+
+function abort(text) {
+  Module.print(text + ':\n' + (new Error).stack);
+  ABORT = true;
+  throw "Assertion: " + text;
+}
+
+function assert(condition, text) {
+  if (!condition) {
+    abort('Assertion failed: ' + text);
+  }
+}
+
+var globalScope = this;
+
+// C calling interface. A convenient way to call C functions (in C files, or
+// defined with extern "C").
+//
+// Note: LLVM optimizations can inline and remove functions, after which you will not be
+//       able to call them. Adding
+//
+//         __attribute__((used))
+//
+//       to the function definition will prevent that.
+//
+// Note: Closure optimizations will minify function names, making
+//       functions no longer callable. If you run closure (on by default
+//       in -O2 and above), you should export the functions you will call
+//       by calling emcc with something like
+//
+//         -s EXPORTED_FUNCTIONS='["_func1","_func2"]'
+//
+// @param ident      The name of the C function (note that C++ functions will be name-mangled - use extern "C")
+// @param returnType The return type of the function, one of the JS types 'number', 'string' or 'array' (use 'number' for any C pointer, and
+//                   'array' for JavaScript arrays and typed arrays).
+// @param argTypes   An array of the types of arguments for the function (if there are no arguments, this can be ommitted). Types are as in returnType,
+//                   except that 'array' is not possible (there is no way for us to know the length of the array)
+// @param args       An array of the arguments to the function, as native JS values (as in returnType)
+//                   Note that string arguments will be stored on the stack (the JS string will become a C string on the stack).
+// @return           The return value, as a native JS value (as in returnType)
+function ccall(ident, returnType, argTypes, args) {
+  var stack = 0;
+  function toC(value, type) {
+    if (type == 'string') {
+      if (value === null || value === undefined || value === 0) return 0; // null string
+      if (!stack) stack = Runtime.stackSave();
+      var ret = Runtime.stackAlloc(value.length+1);
+      writeStringToMemory(value, ret);
+      return ret;
+    } else if (type == 'array') {
+      if (!stack) stack = Runtime.stackSave();
+      var ret = Runtime.stackAlloc(value.length);
+      writeArrayToMemory(value, ret);
+      return ret;
+    }
+    return value;
+  }
+  function fromC(value, type) {
+    if (type == 'string') {
+      return Pointer_stringify(value);
+    }
+    assert(type != 'array');
+    return value;
+  }
+  try {
+    var func = eval('_' + ident);
+  } catch(e) {
+    try {
+      func = globalScope['Module']['_' + ident]; // closure exported function
+    } catch(e) {}
+  }
+  assert(func, 'Cannot call unknown function ' + ident + ' (perhaps LLVM optimizations or closure removed it?)');
+  var i = 0;
+  var cArgs = args ? args.map(function(arg) {
+    return toC(arg, argTypes[i++]);
+  }) : [];
+  var ret = fromC(func.apply(null, cArgs), returnType);
+  if (stack) Runtime.stackRestore(stack);
+  return ret;
+}
+Module["ccall"] = ccall;
+
+// Returns a native JS wrapper for a C function. This is similar to ccall, but
+// returns a function you can call repeatedly in a normal way. For example:
+//
+//   var my_function = cwrap('my_c_function', 'number', ['number', 'number']);
+//   alert(my_function(5, 22));
+//   alert(my_function(99, 12));
+//
+function cwrap(ident, returnType, argTypes) {
+  // TODO: optimize this, eval the whole function once instead of going through ccall each time
+  return function() {
+    return ccall(ident, returnType, argTypes, Array.prototype.slice.call(arguments));
+  }
+}
+Module["cwrap"] = cwrap;
+
+// Sets a value in memory in a dynamic way at run-time. Uses the
+// type data. This is the same as makeSetValue, except that
+// makeSetValue is done at compile-time and generates the needed
+// code then, whereas this function picks the right code at
+// run-time.
+// Note that setValue and getValue only do *aligned* writes and reads!
+// Note that ccall uses JS types as for defining types, while setValue and
+// getValue need LLVM types ('i8', 'i32') - this is a lower-level operation
+function setValue(ptr, value, type, noSafe) {
+  type = type || 'i8';
+  if (type[type.length-1] === '*') type = 'i32'; // pointers are 32-bit
+    switch(type) {
+      case 'i1': HEAP8[(ptr)]=value; break;
+      case 'i8': HEAP8[(ptr)]=value; break;
+      case 'i16': HEAP16[((ptr)>>1)]=value; break;
+      case 'i32': HEAP32[((ptr)>>2)]=value; break;
+      case 'i64': HEAP32[((ptr)>>2)]=value; break;
+      case 'float': HEAPF32[((ptr)>>2)]=value; break;
+      case 'double': (tempDoubleF64[0]=value,HEAP32[((ptr)>>2)]=tempDoubleI32[0],HEAP32[(((ptr)+(4))>>2)]=tempDoubleI32[1]); break;
+      default: abort('invalid type for setValue: ' + type);
+    }
+}
+Module['setValue'] = setValue;
+
+// Parallel to setValue.
+function getValue(ptr, type, noSafe) {
+  type = type || 'i8';
+  if (type[type.length-1] === '*') type = 'i32'; // pointers are 32-bit
+    switch(type) {
+      case 'i1': return HEAP8[(ptr)];
+      case 'i8': return HEAP8[(ptr)];
+      case 'i16': return HEAP16[((ptr)>>1)];
+      case 'i32': return HEAP32[((ptr)>>2)];
+      case 'i64': return HEAP32[((ptr)>>2)];
+      case 'float': return HEAPF32[((ptr)>>2)];
+      case 'double': return (tempDoubleI32[0]=HEAP32[((ptr)>>2)],tempDoubleI32[1]=HEAP32[(((ptr)+(4))>>2)],tempDoubleF64[0]);
+      default: abort('invalid type for setValue: ' + type);
+    }
+  return null;
+}
+Module['getValue'] = getValue;
+
+// Allocates memory for some data and initializes it properly.
+
+var ALLOC_NORMAL = 0; // Tries to use _malloc()
+var ALLOC_STACK = 1; // Lives for the duration of the current function call
+var ALLOC_STATIC = 2; // Cannot be freed
+Module['ALLOC_NORMAL'] = ALLOC_NORMAL;
+Module['ALLOC_STACK'] = ALLOC_STACK;
+Module['ALLOC_STATIC'] = ALLOC_STATIC;
+
+function allocate(slab, types, allocator) {
+  var zeroinit, size;
+  if (typeof slab === 'number') {
+    zeroinit = true;
+    size = slab;
+  } else {
+    zeroinit = false;
+    size = slab.length;
+  }
+
+  var singleType = typeof types === 'string' ? types : null;
+
+  var ret = [_malloc, Runtime.stackAlloc, Runtime.staticAlloc][allocator === undefined ? ALLOC_STATIC : allocator](Math.max(size, singleType ? 1 : types.length));
+
+  if (zeroinit) {
+      _memset(ret, 0, size);
+      return ret;
+  }
+  
+  var i = 0, type;
+  while (i < size) {
+    var curr = slab[i];
+
+    if (typeof curr === 'function') {
+      curr = Runtime.getFunctionIndex(curr);
+    }
+
+    type = singleType || types[i];
+    if (type === 0) {
+      i++;
+      continue;
+    }
+    assert(type, 'Must know what type to store in allocate!');
+
+    if (type == 'i64') type = 'i32'; // special case: we have one i32 here, and one i32 later
+
+    setValue(ret+i, curr, type);
+    i += Runtime.getNativeTypeSize(type);
+  }
+
+  return ret;
+}
+Module['allocate'] = allocate;
+
+function Pointer_stringify(ptr, /* optional */ length) {
+  var utf8 = new Runtime.UTF8Processor();
+  var nullTerminated = typeof(length) == "undefined";
+  var ret = "";
+  var i = 0;
+  var t;
+  while (1) {
+    t = HEAPU8[((ptr)+(i))];
+    if (nullTerminated && t == 0) break;
+    ret += utf8.processCChar(t);
+    i += 1;
+    if (!nullTerminated && i == length) break;
+  }
+  return ret;
+}
+Module['Pointer_stringify'] = Pointer_stringify;
+
+function Array_stringify(array) {
+  var ret = "";
+  for (var i = 0; i < array.length; i++) {
+    ret += String.fromCharCode(array[i]);
+  }
+  return ret;
+}
+Module['Array_stringify'] = Array_stringify;
+
+// Memory management
+
+var FUNCTION_TABLE; // XXX: In theory the indexes here can be equal to pointers to stacked or malloced memory. Such comparisons should
+                    //      be false, but can turn out true. We should probably set the top bit to prevent such issues.
+
+var PAGE_SIZE = 4096;
+function alignMemoryPage(x) {
+  return ((x+4095)>>12)<<12;
+}
+
+var HEAP;
+var HEAP8, HEAPU8, HEAP16, HEAPU16, HEAP32, HEAPU32, HEAPF32, HEAPF64;
+
+var STACK_ROOT, STACKTOP, STACK_MAX;
+var STATICTOP;
+function enlargeMemory() {
+  // TOTAL_MEMORY is the current size of the actual array, and STATICTOP is the new top.
+  Module.printErr('Warning: Enlarging memory arrays, this is not fast! ' + [STATICTOP, TOTAL_MEMORY]);
+  assert(STATICTOP >= TOTAL_MEMORY);
+  assert(TOTAL_MEMORY > 4); // So the loop below will not be infinite
+  while (TOTAL_MEMORY <= STATICTOP) { // Simple heuristic. Override enlargeMemory() if your program has something more optimal for it
+    TOTAL_MEMORY = alignMemoryPage(2*TOTAL_MEMORY);
+  }
+  var oldHEAP8 = HEAP8;
+  var buffer = new ArrayBuffer(TOTAL_MEMORY);
+  HEAP8 = new Int8Array(buffer);
+  HEAP16 = new Int16Array(buffer);
+  HEAP32 = new Int32Array(buffer);
+  HEAPU8 = new Uint8Array(buffer);
+  HEAPU16 = new Uint16Array(buffer);
+  HEAPU32 = new Uint32Array(buffer);
+  HEAPF32 = new Float32Array(buffer);
+  HEAPF64 = new Float64Array(buffer);
+  HEAP8.set(oldHEAP8);
+}
+
+var TOTAL_STACK = Module['TOTAL_STACK'] || 5242880;
+var TOTAL_MEMORY = Module['TOTAL_MEMORY'] || 10485760;
+var FAST_MEMORY = Module['FAST_MEMORY'] || 2097152;
+
+// Initialize the runtime's memory
+// check for full engine support (use string 'subarray' to avoid closure compiler confusion)
+  assert(!!Int32Array && !!Float64Array && !!(new Int32Array(1)['subarray']) && !!(new Int32Array(1)['set']),
+         'Cannot fallback to non-typed array case: Code is too specialized');
+
+  var buffer = new ArrayBuffer(TOTAL_MEMORY);
+  HEAP8 = new Int8Array(buffer);
+  HEAP16 = new Int16Array(buffer);
+  HEAP32 = new Int32Array(buffer);
+  HEAPU8 = new Uint8Array(buffer);
+  HEAPU16 = new Uint16Array(buffer);
+  HEAPU32 = new Uint32Array(buffer);
+  HEAPF32 = new Float32Array(buffer);
+  HEAPF64 = new Float64Array(buffer);
+
+  // Endianness check (note: assumes compiler arch was little-endian)
+  HEAP32[0] = 255;
+  assert(HEAPU8[0] === 255 && HEAPU8[3] === 0, 'Typed arrays 2 must be run on a little-endian system');
+
+var base = intArrayFromString('(null)'); // So printing %s of NULL gives '(null)'
+                                         // Also this ensures we leave 0 as an invalid address, 'NULL'
+STATICTOP = base.length;
+for (var i = 0; i < base.length; i++) {
+  HEAP8[(i)]=base[i]
+}
+
+Module['HEAP'] = HEAP;
+Module['HEAP8'] = HEAP8;
+Module['HEAP16'] = HEAP16;
+Module['HEAP32'] = HEAP32;
+Module['HEAPU8'] = HEAPU8;
+Module['HEAPU16'] = HEAPU16;
+Module['HEAPU32'] = HEAPU32;
+Module['HEAPF32'] = HEAPF32;
+Module['HEAPF64'] = HEAPF64;
+
+STACK_ROOT = STACKTOP = Runtime.alignMemory(STATICTOP);
+STACK_MAX = STACK_ROOT + TOTAL_STACK;
+
+var tempDoublePtr = Runtime.alignMemory(STACK_MAX, 8);
+var tempDoubleI8  = HEAP8.subarray(tempDoublePtr);
+var tempDoubleI32 = HEAP32.subarray(tempDoublePtr >> 2);
+var tempDoubleF32 = HEAPF32.subarray(tempDoublePtr >> 2);
+var tempDoubleF64 = HEAPF64.subarray(tempDoublePtr >> 3);
+function copyTempFloat(ptr) { // functions, because inlining this code is increases code size too much
+  tempDoubleI8[0] = HEAP8[ptr];
+  tempDoubleI8[1] = HEAP8[ptr+1];
+  tempDoubleI8[2] = HEAP8[ptr+2];
+  tempDoubleI8[3] = HEAP8[ptr+3];
+}
+function copyTempDouble(ptr) {
+  tempDoubleI8[0] = HEAP8[ptr];
+  tempDoubleI8[1] = HEAP8[ptr+1];
+  tempDoubleI8[2] = HEAP8[ptr+2];
+  tempDoubleI8[3] = HEAP8[ptr+3];
+  tempDoubleI8[4] = HEAP8[ptr+4];
+  tempDoubleI8[5] = HEAP8[ptr+5];
+  tempDoubleI8[6] = HEAP8[ptr+6];
+  tempDoubleI8[7] = HEAP8[ptr+7];
+}
+STACK_MAX = tempDoublePtr + 8;
+
+STATICTOP = alignMemoryPage(STACK_MAX);
+
+function callRuntimeCallbacks(callbacks) {
+  while(callbacks.length > 0) {
+    var callback = callbacks.shift();
+    var func = callback.func;
+    if (typeof func === 'number') {
+      func = FUNCTION_TABLE[func];
+    }
+    func(callback.arg === undefined ? null : callback.arg);
+  }
+}
+
+var __ATINIT__ = []; // functions called during startup
+var __ATMAIN__ = []; // functions called when main() is to be run
+var __ATEXIT__ = []; // functions called during shutdown
+
+function initRuntime() {
+  callRuntimeCallbacks(__ATINIT__);
+}
+function preMain() {
+  callRuntimeCallbacks(__ATMAIN__);
+}
+function exitRuntime() {
+  callRuntimeCallbacks(__ATEXIT__);
+
+  // Print summary of correction activity
+  CorrectionsMonitor.print();
+}
+
+function String_len(ptr) {
+  var i = 0;
+  while (HEAP8[((ptr)+(i))]) i++; // Note: should be |!= 0|, technically. But this helps catch bugs with undefineds
+  return i;
+}
+Module['String_len'] = String_len;
+
+// Tools
+
+// This processes a JS string into a C-line array of numbers, 0-terminated.
+// For LLVM-originating strings, see parser.js:parseLLVMString function
+function intArrayFromString(stringy, dontAddNull, length /* optional */) {
+  var ret = (new Runtime.UTF8Processor()).processJSString(stringy);
+  if (length) {
+    ret.length = length;
+  }
+  if (!dontAddNull) {
+    ret.push(0);
+  }
+  return ret;
+}
+Module['intArrayFromString'] = intArrayFromString;
+
+function intArrayToString(array) {
+  var ret = [];
+  for (var i = 0; i < array.length; i++) {
+    var chr = array[i];
+    if (chr > 0xFF) {
+        assert(false, 'Character code ' + chr + ' (' + String.fromCharCode(chr) + ')  at offset ' + i + ' not in 0x00-0xFF.');
+      chr &= 0xFF;
+    }
+    ret.push(String.fromCharCode(chr));
+  }
+  return ret.join('');
+}
+Module['intArrayToString'] = intArrayToString;
+
+// Write a Javascript array to somewhere in the heap
+function writeStringToMemory(string, buffer, dontAddNull) {
+  var array = intArrayFromString(string, dontAddNull);
+  var i = 0;
+  while (i < array.length) {
+    var chr = array[i];
+    HEAP8[((buffer)+(i))]=chr
+    i = i + 1;
+  }
+}
+Module['writeStringToMemory'] = writeStringToMemory;
+
+function writeArrayToMemory(array, buffer) {
+  for (var i = 0; i < array.length; i++) {
+    HEAP8[((buffer)+(i))]=array[i];
+  }
+}
+Module['writeArrayToMemory'] = writeArrayToMemory;
+
+var STRING_TABLE = [];
+
+function unSign(value, bits, ignore, sig) {
+  if (value >= 0) {
+    return value;
+  }
+  return bits <= 32 ? 2*Math.abs(1 << (bits-1)) + value // Need some trickery, since if bits == 32, we are right at the limit of the bits JS uses in bitshifts
+                    : Math.pow(2, bits)         + value;
+  // TODO: clean up previous line
+}
+function reSign(value, bits, ignore, sig) {
+  if (value <= 0) {
+    return value;
+  }
+  var half = bits <= 32 ? Math.abs(1 << (bits-1)) // abs is needed if bits == 32
+                        : Math.pow(2, bits-1);
+  if (value >= half && (bits <= 32 || value > half)) { // for huge values, we can hit the precision limit and always get true here. so don't do that
+                                                       // but, in general there is no perfect solution here. With 64-bit ints, we get rounding and errors
+                                                       // TODO: In i64 mode 1, resign the two parts separately and safely
+    value = -2*half + value; // Cannot bitshift half, as it may be at the limit of the bits JS uses in bitshifts
+  }
+  return value;
+}
+
+// A counter of dependencies for calling run(). If we need to
+// do asynchronous work before running, increment this and
+// decrement it. Incrementing must happen in a place like
+// PRE_RUN_ADDITIONS (used by emcc to add file preloading).
+// Note that you can add dependencies in preRun, even though
+// it happens right before run - run will be postponed until
+// the dependencies are met.
+var runDependencies = 0;
+function addRunDependency() {
+  runDependencies++;
+  if (Module['monitorRunDependencies']) {
+    Module['monitorRunDependencies'](runDependencies);
+  }
+}
+Module['addRunDependency'] = addRunDependency;
+function removeRunDependency() {
+  runDependencies--;
+  if (Module['monitorRunDependencies']) {
+    Module['monitorRunDependencies'](runDependencies);
+  }
+  if (runDependencies == 0) run();
+}
+Module['removeRunDependency'] = removeRunDependency;
+
+// === Body ===
+
+
+
+
+function ____init___0($self, $initx, $inity, $initz) {
+  ;
+  var __label__;
+  __label__ = 2; 
+  while(1) switch(__label__) {
+    case 2: 
+      __label__ = 3; break;
+    case 3: 
+      __label__ = 4; break;
+    case 4: 
+      var $_tmp_70=(($self+4)|0);
+      (tempDoubleF64[0]=$initx,HEAP32[(($_tmp_70)>>2)]=tempDoubleI32[0],HEAP32[((($_tmp_70)+(4))>>2)]=tempDoubleI32[1]);
+      var $_tmp_71=(($self+20)|0);
+      (tempDoubleF64[0]=$inity,HEAP32[(($_tmp_71)>>2)]=tempDoubleI32[0],HEAP32[((($_tmp_71)+(4))>>2)]=tempDoubleI32[1]);
+      var $_tmp_72=(($self+12)|0);
+      (tempDoubleF64[0]=$initz,HEAP32[(($_tmp_72)>>2)]=tempDoubleI32[0],HEAP32[((($_tmp_72)+(4))>>2)]=tempDoubleI32[1]);
+      __label__ = 5; break;
+    case 5: 
+      ;
+      return;
+    default: assert(0, "bad label: " + __label__);
+  }
+}
+
+
+function ____init___1($self, $initx, $inity, $initz) {
+  ;
+  var __label__;
+  __label__ = 2; 
+  while(1) switch(__label__) {
+    case 2: 
+      __label__ = 3; break;
+    case 3: 
+      __label__ = 4; break;
+    case 4: 
+      var $_tmp_218=(($self+4)|0);
+      (tempDoubleF64[0]=$initx,HEAP32[(($_tmp_218)>>2)]=tempDoubleI32[0],HEAP32[((($_tmp_218)+(4))>>2)]=tempDoubleI32[1]);
+      var $_tmp_219=(($self+20)|0);
+      (tempDoubleF64[0]=$inity,HEAP32[(($_tmp_219)>>2)]=tempDoubleI32[0],HEAP32[((($_tmp_219)+(4))>>2)]=tempDoubleI32[1]);
+      var $_tmp_220=(($self+12)|0);
+      (tempDoubleF64[0]=$initz,HEAP32[(($_tmp_220)>>2)]=tempDoubleI32[0],HEAP32[((($_tmp_220)+(4))>>2)]=tempDoubleI32[1]);
+      __label__ = 5; break;
+    case 5: 
+      ;
+      return;
+    default: assert(0, "bad label: " + __label__);
+  }
+}
+
+
+function _main($argc, $argv) {
+  ;
+  var __label__;
+  __label__ = 2; 
+  while(1) switch(__label__) {
+    case 2: 
+      var $call1=_SDL_Init(32);
+      var $w=HEAP32[((_MYW)>>2)];
+      var $h=HEAP32[((_MYH)>>2)];
+      var $call2=_SDL_SetVideoMode(2 * $w, 2 * $h, 32, 0);
+      HEAP32[((_screen)>>2)]=$call2;
+      __label__ = 3; break;
+    case 3: 
+      __label__ = 4; break;
+    case 4: 
+      var $_tmp_8=((_global_closure+4)|0);
+      (tempDoubleF64[0]=0.00001,HEAP32[(($_tmp_8)>>2)]=tempDoubleI32[0],HEAP32[((($_tmp_8)+(4))>>2)]=tempDoubleI32[1]);
+      var $_tmp_9=((_global_closure+12)|0);
+      (tempDoubleF64[0]=1000000000,HEAP32[(($_tmp_9)>>2)]=tempDoubleI32[0],HEAP32[((($_tmp_9)+(4))>>2)]=tempDoubleI32[1]);
+      var $_tmp_10=_cls1_init();
+      var $_tmp_11=0;
+      var $_tmp_12=0;
+      var $_tmp_13=0;
+      ____init___0($_tmp_10, $_tmp_11, $_tmp_12, $_tmp_13);
+      var $_tmp_14=_cls1_init();
+      var $_tmp_15=1;
+      var $_tmp_16=0;
+      var $_tmp_17=0;
+      ____init___0($_tmp_14, $_tmp_15, $_tmp_16, $_tmp_17);
+      var $_tmp_18=_cls1_init();
+      var $_tmp_19=0;
+      var $_tmp_20=1;
+      var $_tmp_21=0;
+      ____init___0($_tmp_18, $_tmp_19, $_tmp_20, $_tmp_21);
+      var $_tmp_22=((_global_closure+24)|0);
+      HEAP32[(($_tmp_22)>>2)]=$_tmp_18;
+      _cls1_incref($_tmp_18);
+      var $_tmp_23=_cls1_init();
+      var $_tmp_24=0;
+      var $_tmp_25=0;
+      var $_tmp_26=1;
+      ____init___0($_tmp_23, $_tmp_24, $_tmp_25, $_tmp_26);
+      var $_tmp_27=__reflectThrough_0($_tmp_14, $_tmp_18);
+      var $_tmp_28=____eq___0($_tmp_27, $_tmp_14);
+      _cls1_decref($_tmp_27);
+      var $_tmp_29=_bool_nonzero($_tmp_28);
+      var $_tmp_30=$_tmp_29 ^ 1;
+      var $_tmp_31=_bool_nonzero($_tmp_30);
+      _cls1_decref($_tmp_23);
+      _cls1_decref($_tmp_10);
+      _cls1_decref($_tmp_14);
+      if ($_tmp_31) { __label__ = 5; break; } else { __label__ = 6; break; }
+    case 5: 
+      __label__ = 7; break;
+    case 6: 
+      __label__ = 8; break;
+    case 7: 
+      var $_tmp_32=_int_div(1, 0);
+      var $_tmp_33=_int_str($_tmp_32);
+      var $_tmp_34=_str_buf($_tmp_33);
+      var $_tmp_35=_str_len($_tmp_33);
+      var $tmp4_0=((STRING_TABLE._str_0)|0);
+      var $0=_printf($tmp4_0, (tempInt=STACKTOP,STACKTOP += 8,assert(STACKTOP < STACK_ROOT + STACK_MAX, "Ran out of stack"),HEAP32[((tempInt)>>2)]=$_tmp_35,HEAP32[(((tempInt)+(4))>>2)]=$_tmp_34,tempInt));
+      _str_decref($_tmp_33);
+      __label__ = 8; break;
+    case 8: 
+      var $_tmp_36=_cls1_init();
+      var $_tmp_37=-1;
+      var $_tmp_38=-1;
+      var $_tmp_39=0;
+      ____init___0($_tmp_36, $_tmp_37, $_tmp_38, $_tmp_39);
+      var $_tmp_40=__reflectThrough_0($_tmp_36, $_tmp_18);
+      _cls1_decref($_tmp_36);
+      var $_tmp_41=_cls1_init();
+      var $_tmp_42=-1;
+      var $_tmp_43=1;
+      var $_tmp_44=0;
+      ____init___0($_tmp_41, $_tmp_42, $_tmp_43, $_tmp_44);
+      var $_tmp_45=____eq___0($_tmp_40, $_tmp_41);
+      _cls1_decref($_tmp_40);
+      _cls1_decref($_tmp_41);
+      var $_tmp_46=_bool_nonzero($_tmp_45);
+      var $_tmp_47=$_tmp_46 ^ 1;
+      var $_tmp_48=_bool_nonzero($_tmp_47);
+      _cls1_decref($_tmp_18);
+      if ($_tmp_48) { __label__ = 9; break; } else { __label__ = 10; break; }
+    case 9: 
+      __label__ = 11; break;
+    case 10: 
+      __label__ = 12; break;
+    case 11: 
+      var $_tmp_49=_int_div(1, 0);
+      var $_tmp_50=_int_str($_tmp_49);
+      var $_tmp_51=_str_buf($_tmp_50);
+      var $_tmp_52=_str_len($_tmp_50);
+      var $tmp8_0=((STRING_TABLE._str_0)|0);
+      var $1=_printf($tmp8_0, (tempInt=STACKTOP,STACKTOP += 8,assert(STACKTOP < STACK_ROOT + STACK_MAX, "Ran out of stack"),HEAP32[((tempInt)>>2)]=$_tmp_52,HEAP32[(((tempInt)+(4))>>2)]=$_tmp_51,tempInt));
+      _str_decref($_tmp_50);
+      __label__ = 12; break;
+    case 12: 
+      var $_tmp_53=_cls0_init();
+      var $_tmp_54=0;
+      var $_tmp_55=0;
+      var $_tmp_56=0;
+      ____init___1($_tmp_53, $_tmp_54, $_tmp_55, $_tmp_56);
+      var $_tmp_57=((_global_closure+20)|0);
+      HEAP32[(($_tmp_57)>>2)]=$_tmp_53;
+      _cls0_incref($_tmp_53);
+      var $_tmp_58=_cls1_init();
+      var $_tmp_59=3;
+      var $_tmp_60=4;
+      var $_tmp_61=12;
+      ____init___0($_tmp_58, $_tmp_59, $_tmp_60, $_tmp_61);
+      var $_tmp_62=_cls1_init();
+      var $_tmp_63=1;
+      var $_tmp_64=1;
+      var $_tmp_65=1;
+      ____init___0($_tmp_62, $_tmp_63, $_tmp_64, $_tmp_65);
+      __main_0();
+      _cls1_decref($_tmp_58);
+      _cls0_decref($_tmp_53);
+      _cls1_decref($_tmp_62);
+      __label__ = 13; break;
+    case 13: 
+      var $_tmp_68=HEAP32[((((_global_closure+20)|0))>>2)];
+      _cls0_decref($_tmp_68);
+      var $_tmp_69=HEAP32[((((_global_closure+24)|0))>>2)];
+      _cls1_decref($_tmp_69);
+      _teardown_runtime();
+      ;
+      return 0;
+    default: assert(0, "bad label: " + __label__);
+  }
+}
+Module["_main"] = _main;_main["X"]=1;
+
+function __magnitude_0($self) {
+  ;
+  var __label__;
+  __label__ = 2; 
+  while(1) switch(__label__) {
+    case 2: 
+      __label__ = 3; break;
+    case 3: 
+      __label__ = 4; break;
+    case 4: 
+      var $_tmp_97=__dot_0($self, $self);
+      var $_tmp_98=_math_sqrt($_tmp_97);
+      ;
+      return $_tmp_98;
+    default: assert(0, "bad label: " + __label__);
+  }
+}
+
+
+function ____add___0($self, $other) {
+  ;
+  var __label__;
+  __label__ = 2; 
+  while(1) switch(__label__) {
+    case 2: 
+      __label__ = 3; break;
+    case 3: 
+      __label__ = 4; break;
+    case 4: 
+      var $_tmp_99=(($self+4)|0);
+      var $_tmp_100=(tempDoubleI32[0]=HEAP32[(($_tmp_99)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_99)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_101=(($other+4)|0);
+      var $_tmp_102=(tempDoubleI32[0]=HEAP32[(($_tmp_101)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_101)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_103=_float_add($_tmp_100, $_tmp_102);
+      var $_tmp_104=(($self+20)|0);
+      var $_tmp_105=(tempDoubleI32[0]=HEAP32[(($_tmp_104)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_104)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_106=(($other+20)|0);
+      var $_tmp_107=(tempDoubleI32[0]=HEAP32[(($_tmp_106)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_106)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_108=_float_add($_tmp_105, $_tmp_107);
+      var $_tmp_109=(($self+12)|0);
+      var $_tmp_110=(tempDoubleI32[0]=HEAP32[(($_tmp_109)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_109)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_111=(($other+12)|0);
+      var $_tmp_112=(tempDoubleI32[0]=HEAP32[(($_tmp_111)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_111)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_113=_float_add($_tmp_110, $_tmp_112);
+      var $_tmp_114=_cls1_init();
+      ____init___0($_tmp_114, $_tmp_103, $_tmp_108, $_tmp_113);
+      ;
+      return $_tmp_114;
+    default: assert(0, "bad label: " + __label__);
+  }
+}
+
+
+function ____sub___0($self, $other) {
+  ;
+  var __label__;
+  __label__ = 2; 
+  while(1) switch(__label__) {
+    case 2: 
+      __label__ = 3; break;
+    case 3: 
+      __label__ = 4; break;
+    case 4: 
+      var $_tmp_115=(($self+4)|0);
+      var $_tmp_116=(tempDoubleI32[0]=HEAP32[(($_tmp_115)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_115)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_117=(($other+4)|0);
+      var $_tmp_118=(tempDoubleI32[0]=HEAP32[(($_tmp_117)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_117)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_119=_float_sub($_tmp_116, $_tmp_118);
+      var $_tmp_120=(($self+20)|0);
+      var $_tmp_121=(tempDoubleI32[0]=HEAP32[(($_tmp_120)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_120)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_122=(($other+20)|0);
+      var $_tmp_123=(tempDoubleI32[0]=HEAP32[(($_tmp_122)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_122)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_124=_float_sub($_tmp_121, $_tmp_123);
+      var $_tmp_125=(($self+12)|0);
+      var $_tmp_126=(tempDoubleI32[0]=HEAP32[(($_tmp_125)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_125)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_127=(($other+12)|0);
+      var $_tmp_128=(tempDoubleI32[0]=HEAP32[(($_tmp_127)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_127)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_129=_float_sub($_tmp_126, $_tmp_128);
+      var $_tmp_130=_cls1_init();
+      ____init___0($_tmp_130, $_tmp_119, $_tmp_124, $_tmp_129);
+      ;
+      return $_tmp_130;
+    default: assert(0, "bad label: " + __label__);
+  }
+}
+
+
+function __scale_0($self, $factor) {
+  ;
+  var __label__;
+  __label__ = 2; 
+  while(1) switch(__label__) {
+    case 2: 
+      __label__ = 3; break;
+    case 3: 
+      __label__ = 4; break;
+    case 4: 
+      var $_tmp_131=(($self+4)|0);
+      var $_tmp_132=(tempDoubleI32[0]=HEAP32[(($_tmp_131)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_131)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_133=_float_mul($factor, $_tmp_132);
+      var $_tmp_134=(($self+20)|0);
+      var $_tmp_135=(tempDoubleI32[0]=HEAP32[(($_tmp_134)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_134)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_136=_float_mul($factor, $_tmp_135);
+      var $_tmp_137=(($self+12)|0);
+      var $_tmp_138=(tempDoubleI32[0]=HEAP32[(($_tmp_137)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_137)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_139=_float_mul($factor, $_tmp_138);
+      var $_tmp_140=_cls1_init();
+      ____init___0($_tmp_140, $_tmp_133, $_tmp_136, $_tmp_139);
+      ;
+      return $_tmp_140;
+    default: assert(0, "bad label: " + __label__);
+  }
+}
+
+
+function __dot_0($self, $other) {
+  ;
+  var __label__;
+  __label__ = 2; 
+  while(1) switch(__label__) {
+    case 2: 
+      __label__ = 3; break;
+    case 3: 
+      __label__ = 4; break;
+    case 4: 
+      var $_tmp_141=(($self+4)|0);
+      var $_tmp_142=(tempDoubleI32[0]=HEAP32[(($_tmp_141)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_141)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_143=(($other+4)|0);
+      var $_tmp_144=(tempDoubleI32[0]=HEAP32[(($_tmp_143)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_143)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_145=_float_mul($_tmp_142, $_tmp_144);
+      var $_tmp_146=(($self+20)|0);
+      var $_tmp_147=(tempDoubleI32[0]=HEAP32[(($_tmp_146)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_146)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_148=(($other+20)|0);
+      var $_tmp_149=(tempDoubleI32[0]=HEAP32[(($_tmp_148)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_148)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_150=_float_mul($_tmp_147, $_tmp_149);
+      var $_tmp_151=_float_add($_tmp_145, $_tmp_150);
+      var $_tmp_152=(($self+12)|0);
+      var $_tmp_153=(tempDoubleI32[0]=HEAP32[(($_tmp_152)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_152)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_154=(($other+12)|0);
+      var $_tmp_155=(tempDoubleI32[0]=HEAP32[(($_tmp_154)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_154)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_156=_float_mul($_tmp_153, $_tmp_155);
+      var $_tmp_157=_float_add($_tmp_151, $_tmp_156);
+      ;
+      return $_tmp_157;
+    default: assert(0, "bad label: " + __label__);
+  }
+}
+
+
+function __cross_0($self, $other) {
+  ;
+  var __label__;
+  __label__ = 2; 
+  while(1) switch(__label__) {
+    case 2: 
+      __label__ = 3; break;
+    case 3: 
+      __label__ = 4; break;
+    case 4: 
+      var $_tmp_158=(($self+20)|0);
+      var $_tmp_159=(tempDoubleI32[0]=HEAP32[(($_tmp_158)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_158)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_160=(($other+12)|0);
+      var $_tmp_161=(tempDoubleI32[0]=HEAP32[(($_tmp_160)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_160)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_162=_float_mul($_tmp_159, $_tmp_161);
+      var $_tmp_163=(($self+12)|0);
+      var $_tmp_164=(tempDoubleI32[0]=HEAP32[(($_tmp_163)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_163)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_165=(($other+20)|0);
+      var $_tmp_166=(tempDoubleI32[0]=HEAP32[(($_tmp_165)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_165)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_167=_float_mul($_tmp_164, $_tmp_166);
+      var $_tmp_168=_float_sub($_tmp_162, $_tmp_167);
+      var $_tmp_169=(($self+12)|0);
+      var $_tmp_170=(tempDoubleI32[0]=HEAP32[(($_tmp_169)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_169)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_171=(($other+4)|0);
+      var $_tmp_172=(tempDoubleI32[0]=HEAP32[(($_tmp_171)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_171)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_173=_float_mul($_tmp_170, $_tmp_172);
+      var $_tmp_174=(($self+4)|0);
+      var $_tmp_175=(tempDoubleI32[0]=HEAP32[(($_tmp_174)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_174)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_176=(($other+12)|0);
+      var $_tmp_177=(tempDoubleI32[0]=HEAP32[(($_tmp_176)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_176)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_178=_float_mul($_tmp_175, $_tmp_177);
+      var $_tmp_179=_float_sub($_tmp_173, $_tmp_178);
+      var $_tmp_180=(($self+4)|0);
+      var $_tmp_181=(tempDoubleI32[0]=HEAP32[(($_tmp_180)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_180)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_182=(($other+20)|0);
+      var $_tmp_183=(tempDoubleI32[0]=HEAP32[(($_tmp_182)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_182)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_184=_float_mul($_tmp_181, $_tmp_183);
+      var $_tmp_185=(($self+20)|0);
+      var $_tmp_186=(tempDoubleI32[0]=HEAP32[(($_tmp_185)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_185)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_187=(($other+4)|0);
+      var $_tmp_188=(tempDoubleI32[0]=HEAP32[(($_tmp_187)>>2)],tempDoubleI32[1]=HEAP32[((($_tmp_187)+(4))>>2)],tempDoubleF64[0]);
+      var $_tmp_189=_float_mul($_tmp_186, $_tmp_188);
+      var $_tmp_190=_float_sub($_tmp_184, $_tmp_189);
+      var $_tmp_191=_cls1_init();
+      ____init___0($_tmp_191, $_tmp_168, $_tmp_179, $_tmp_190);
+      ;
+      return $_tmp_191;
+    default: assert(0, "bad label: " + __label__);
+  }
+}
+
+
+function __normalized_0($self) {
+  ;
+  var __label__;
+  __label__ = 2; 
+  while(1) switch(__label__) {
+    case 2: 
+      __label__ = 3; break;
+    case 3: 
+      __label__ = 4; break;
+    case 4: 

From noreply at buildbot.pypy.org  Wed Dec 10 10:26:59 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Dec 2014 10:26:59 +0100 (CET)
Subject: [pypy-commit] benchmarks default: fixes
Message-ID: <20141210092659.E910A1D3816@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r293:24b6c21ba3f0
Date: 2014-12-10 09:27 +0000
http://bitbucket.org/pypy/benchmarks/changeset/24b6c21ba3f0/

Log:	fixes

diff --git a/own/bm_icbd.py b/own/bm_icbd.py
--- a/own/bm_icbd.py
+++ b/own/bm_icbd.py
@@ -3,26 +3,33 @@
 
 def main(n):
     # HAAAAAACK with subprocess because I can't get the thing
-    # to run in-process :-(  As a result, it doesn't make
-    # sense to run it more than once.
-
-    d = os.environ.copy()
-    d['PYTHONPATH'] = 'icbd'
-    t0 = time.time()
-    popen = subprocess.Popen(
-                     [sys.executable,
-                      '-m', 'icbd.type_analyzer.analyze_all',
-                      '-I', 'stdlib/python2.5_tiny',
-                      '-I', '.',
-                      '-E', 'icbd/type_analyzer/tests',
-                      '-E', 'icbd/compiler/benchmarks',
-                      '-E', 'icbd/compiler/tests',
-                      '-I', 'stdlib/type_mocks',
-                      '-n',
-                      'icbd'], cwd=this_dir, env=d, stdout=subprocess.PIPE)
-    popen.communicate()
-    time_elapsed = time.time() - t0
-    return [time_elapsed] * n
+    # to run in-process :-(  As a result, it doesn't make much
+    # sense to run it more than once.  We'll run it n/25 times,
+    # like hexiom2.  All runs suffer from warm-up.
+    l = []
+    for i in range(n):
+        if (i % 25) == 0:
+            d = os.environ.copy()
+            d['PYTHONPATH'] = 'icbd'
+            t0 = time.time()
+            popen = subprocess.Popen(
+                 [sys.executable,
+                  '-m', 'icbd.type_analyzer.analyze_all',
+                  '-I', 'stdlib/python2.5_tiny',
+                  '-I', '.',
+                  '-E', 'icbd/type_analyzer/tests',
+                  '-E', 'icbd/compiler/benchmarks',
+                  '-E', 'icbd/compiler/tests',
+                  '-I', 'stdlib/type_mocks',
+                  '-n',
+                  'icbd'],
+                 cwd=this_dir, env=d, stdout=subprocess.PIPE)
+            popen.communicate()
+            if popen.returncode != 0:
+                raise OSError("the subprocess failed, see above")
+            time_elapsed = time.time() - t0
+        l.append(time_elapsed)
+    return l
 
 if __name__ == "__main__":
     import util, optparse

From noreply at buildbot.pypy.org  Wed Dec 10 12:16:22 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Dec 2014 12:16:22 +0100 (CET)
Subject: [pypy-commit] pypy default: Try to clean up the construction of
 GcStructs with inlined fields.
Message-ID: <20141210111622.651261C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74876:1a2817c44b28
Date: 2014-12-10 11:16 +0000
http://bitbucket.org/pypy/pypy/changeset/1a2817c44b28/

Log:	Try to clean up the construction of GcStructs with inlined fields.
	Complains e.g. on GcStructs containing inlined no-length arrays.

diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -182,8 +182,8 @@
     def _freeze_(self):
         return True
 
-    def _inline_is_varsize(self, last):
-        return False
+    def _note_inlined_into(self, parent, first, last):
+        """Called when this type is being used inline in a container."""
 
     def _is_atomic(self):
         return False
@@ -201,8 +201,9 @@
 class ContainerType(LowLevelType):
     _adtmeths = {}
 
-    def _inline_is_varsize(self, last):
-        raise TypeError("%r cannot be inlined in structure" % self)
+    def _note_inlined_into(self, parent, first, last):
+        raise TypeError("%r cannot be inlined in %r" % (
+            self.__class__.__name__, parent.__class__.__name__))
 
     def _install_extras(self, adtmeths={}, hints={}):
         self._adtmeths = frozendict(adtmeths)
@@ -279,11 +280,13 @@
 
         # look if we have an inlined variable-sized array as the last field
         if fields:
+            first = True
             for name, typ in fields[:-1]:
-                typ._inline_is_varsize(False)
+                typ._note_inlined_into(self, first=first, last=False)
                 first = False
             name, typ = fields[-1]
-            if typ._inline_is_varsize(True):
+            typ._note_inlined_into(self, first=first, last=True)
+            if typ._is_varsize():
                 self._arrayfld = name
         self._flds = frozendict(flds)
         self._names = tuple(names)
@@ -299,11 +302,14 @@
                 return first, FIRSTTYPE
         return None, None
 
-    def _inline_is_varsize(self, last):
-        if self._arrayfld:
+    def _note_inlined_into(self, parent, first, last):
+        if self._arrayfld is not None:
             raise TypeError("cannot inline a var-sized struct "
                             "inside another container")
-        return False
+        if self._gckind == 'gc':
+            if not first or not isinstance(parent, GcStruct):
+                raise TypeError("a GcStruct can only be inlined as the first "
+                                "field of another GcStruct")
 
     def _is_atomic(self):
         for typ in self._flds.values():
@@ -428,15 +434,18 @@
         if isinstance(self.OF, ContainerType) and self.OF._gckind != 'raw':
             raise TypeError("cannot have a %s container as array item type"
                             % (self.OF._gckind,))
-        self.OF._inline_is_varsize(False)
+        self.OF._note_inlined_into(self, first=False, last=False)
 
         self._install_extras(**kwds)
 
-    def _inline_is_varsize(self, last):
-        if not last:
+    def _note_inlined_into(self, parent, first, last):
+        if not last or not isinstance(parent, Struct):
             raise TypeError("cannot inline an array in another container"
                             " unless as the last field of a structure")
-        return True
+        if self._gckind == 'gc':
+            raise TypeError("cannot inline a GC array inside a structure")
+        if parent._gckind == 'gc' and self._hints.get('nolength', False):
+            raise TypeError("cannot inline a no-length array inside a GcStruct")
 
     def _is_atomic(self):
         return self.OF._is_atomic()
@@ -474,9 +483,6 @@
 
 class GcArray(Array):
     _gckind = 'gc'
-    def _inline_is_varsize(self, last):
-        raise TypeError("cannot inline a GC array inside a structure")
-
 
 class FixedSizeArray(Struct):
     # behaves more or less like a Struct with fields item0, item1, ...
@@ -508,7 +514,7 @@
         if isinstance(self.OF, ContainerType) and self.OF._gckind != 'raw':
             raise TypeError("cannot have a %s container as array item type"
                             % (self.OF._gckind,))
-        self.OF._inline_is_varsize(False)
+        self.OF._note_inlined_into(self, first=False, last=False)
 
     def _str_fields(self):
         return str(self.OF)
@@ -579,8 +585,11 @@
     def __str__(self):
         return "%s (opaque)" % self.tag
 
-    def _inline_is_varsize(self, last):
-        return False    # OpaqueType can be inlined
+    def _note_inlined_into(self, parent, first, last):
+        # OpaqueType can be inlined, but not GcOpaqueType
+        if self._gckind == 'gc':
+            raise TypeError("%r cannot be inlined in %r" % (
+                self.__class__.__name__, parent.__class__.__name__))
 
     def _container_example(self):
         return _opaque(self)
@@ -599,10 +608,6 @@
     def __str__(self):
         return "%s (gcopaque)" % self.tag
 
-    def _inline_is_varsize(self, last):
-        raise TypeError("%r cannot be inlined in structure" % self)
-
-
 class ForwardReference(ContainerType):
     _gckind = 'raw'
     def become(self, realcontainertype):
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -808,6 +808,20 @@
     assert F.RESULT == Signed
     assert F.ARGS == (Signed,)
 
+def test_cannot_inline_random_stuff_in_gcstruct():
+    S = GcStruct('S')
+    GcStruct('X', ('a', S))    # works
+    py.test.raises(TypeError, GcStruct, 'X', ('a', Signed), ('b', S))
+    GcStruct('X', ('a', Array(Signed)))   # works
+    py.test.raises(TypeError, GcStruct, 'X', ('a', Array(Signed)),
+                                             ('b', Signed))
+    Struct('X', ('a', Array(Signed, hints={'nolength': True})))   # works
+    py.test.raises(TypeError, GcStruct, 'X',
+                   ('a', Array(Signed, hints={'nolength': True})))
+    GcStruct('X', ('a', OpaqueType('foo')))   # works
+    py.test.raises(TypeError, GcStruct, 'X', ('a', GcOpaqueType('foo')))
+
+
 class TestTrackAllocation:
     def test_automatic_tracking(self):
         # calls to start_tracking_allocations/stop_tracking_allocations

From noreply at buildbot.pypy.org  Wed Dec 10 15:27:12 2014
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 10 Dec 2014 15:27:12 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: hg merge default
Message-ID: <20141210142712.DA01B1D38E8@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: vmprof
Changeset: r74877:b53169916578
Date: 2014-12-10 14:26 +0000
http://bitbucket.org/pypy/pypy/changeset/b53169916578/

Log:	hg merge default

diff too long, truncating to 2000 out of 52817 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -89,6 +89,7 @@
 ^pypy/doc/image/lattice3\.png$
 ^pypy/doc/image/stackless_informal\.png$
 ^pypy/doc/image/parsing_example.+\.png$
+^rpython/doc/_build/.*$
 ^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
 ^compiled
 ^.git/
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -23,6 +23,7 @@
 
     the pypy-dev team <pypy-dev at python.org>
 
+
 Building
 ========
 
@@ -36,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html
diff --git a/ctypes_configure/doc/configure.txt b/ctypes_configure/doc/configure.txt
--- a/ctypes_configure/doc/configure.txt
+++ b/ctypes_configure/doc/configure.txt
@@ -19,6 +19,4 @@
 usage
 =====
 
-`sample.py`_ explains in details how to use it.
-
-.. _`sample.py`: http://codespeak.net/svn/pypy/dist/ctypes_configure/doc/sample.py
+:source:`sample.py <ctypes_configure/doc/sample.py>` explains in details how to use it.
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -59,7 +59,7 @@
     def __init__(self, basename, core=False, compiler=None, usemodules='',
                  skip=None):
         self.basename = basename
-        self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket']
+        self._usemodules = usemodules.split() + ['signal', 'time', 'itertools', '_socket']
         self._compiler = compiler
         self.core = core
         self.skip = skip
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -66,11 +66,12 @@
 
 @builtinify
 def getgrnam(name):
-    if not isinstance(name, str):
+    if not isinstance(name, basestring):
         raise TypeError("expected string")
+    name = str(name)
     res = libc.getgrnam(name)
     if not res:
-        raise KeyError(name)
+        raise KeyError("'getgrnam(): name not found: %s'" % name)
     return _group_from_gstruct(res)
 
 @builtinify
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -29,7 +29,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -40,7 +40,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
     "cStringIO", "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -64,19 +64,15 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
@@ -86,9 +82,10 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")],
 }
+if sys.platform == "win32":
+    module_suggests["cpyext"].append(("translation.shared", True))
 
 module_import_dependencies = {
     # no _rawffi if importing rpython.rlib.clibffi raises ImportError
@@ -255,10 +252,6 @@
         BoolOption("optimized_list_getitem",
                    "special case the 'list[integer]' expressions",
                    default=False),
-        BoolOption("builtinshortcut",
-                   "a shortcut for operations between built-in types. XXX: "
-                   "deprecated, not really a shortcut any more.",
-                   default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
                    default=False,
@@ -270,9 +263,6 @@
                    # weakrefs needed, because of get_subclasses()
                    requires=[("translation.rweakref", True)]),
 
-        ChoiceOption("multimethods", "the multimethod implementation to use",
-                     ["doubledispatch", "mrd"],
-                     default="mrd"),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
                    default=False,
diff --git a/pypy/config/test/test_pypyoption.py b/pypy/config/test/test_pypyoption.py
--- a/pypy/config/test/test_pypyoption.py
+++ b/pypy/config/test/test_pypyoption.py
@@ -64,7 +64,7 @@
     def check_file_exists(fn):
         assert configdocdir.join(fn).check()
 
-    from pypy.doc.config.confrest import all_optiondescrs
+    from pypy.doc.config.generate import all_optiondescrs
     configdocdir = thisdir.dirpath().dirpath().join("doc", "config")
     for descr in all_optiondescrs:
         prefix = descr._name
diff --git a/pypy/doc/TODO b/pypy/doc/TODO
new file mode 100644
--- /dev/null
+++ b/pypy/doc/TODO
@@ -0,0 +1,40 @@
+Documentation TODO
+==================
+
+General
+-------
+
+* architecture documents don't really show the separation between PyPy and
+  RPython
+  * architecture.rst is duplicate (both pypy and rpython)
+* Consider moving information from pypy/doc/{build,windows}.rst to rpython/doc
+
+
+Cleanup
+~~~~~~~
+
+* remove documentation on removed features
+  * various object spaces
+* update / remove dead links
+
+
+Meta
+~~~~
+
+* work on configuration/options documentation generation
+
+
+PyPy
+----
+
+* Update coding guide
+* Move links from project-documentation.rst to index.rst and consider killing
+  it.
+
+
+RPython
+-------
+
+* make translation.rst a high-level overview and move details in their own
+  documents
+* redo various outdated pictures in translation.rst
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,20 +1,17 @@
-
 .. comment: this document is very incomplete, should we generate
             it automatically?
 
-=======================
 The ``__pypy__`` module
 =======================
 
 The ``__pypy__`` module is the main entry point to special features provided
-by PyPy's standard interpreter. Its content depends on `configuration options`_ 
-which may add new functionality and functions whose existence or non-existence 
-indicates the presence of such features. 
+by PyPy's standard interpreter. Its content depends on :doc:`configuration options <config/index>`
+which may add new functionality and functions whose existence or non-existence
+indicates the presence of such features.
 
-.. _`configuration options`: config/index.html
 
 Generally available functionality
-=================================
+---------------------------------
 
  - ``internal_repr(obj)``: return the interpreter-level representation of an
    object.
@@ -22,28 +19,22 @@
    It works like a simplified array of characters (actually, depending on the
    configuration the ``array`` module internally uses this).
 
+
 Transparent Proxy Functionality
-===============================
+-------------------------------
 
-If `transparent proxies`_ are enabled (with :config:`objspace.std.withtproxy`)
+If :ref:`transparent proxies <tproxy>` are enabled (with :config:`objspace.std.withtproxy`)
 the following functions are put into ``__pypy__``:
 
  - ``tproxy(typ, controller)``: Return something that looks like it is of type
    typ. Its behaviour is completely controlled by the controller. See the docs
-   about `transparent proxies`_ for detail.
-
+   about :ref:`transparent proxies <tproxy>` for detail.
  - ``get_tproxy_controller(obj)``: If obj is really a transparent proxy, return
    its controller. Otherwise return None.
 
-.. _`transparent proxies`: objspace-proxies.html#tproxy
-
 
 Functionality available on py.py (not after translation)
-========================================================
+--------------------------------------------------------
 
  - ``isfake(obj)``: returns True if ``obj`` is faked.
-
  - ``interp_pdb()``: start a pdb at interpreter-level.
-
-
-
diff --git a/pypy/doc/_ref.txt b/pypy/doc/_ref.txt
deleted file mode 100644
--- a/pypy/doc/_ref.txt
+++ /dev/null
@@ -1,114 +0,0 @@
-.. This file is generated automatically by makeref.py script,
-   which in turn is run manually.
-
-.. _`ctypes_configure/doc/sample.py`: https://bitbucket.org/pypy/pypy/src/default/ctypes_configure/doc/sample.py
-.. _`dotviewer/`: https://bitbucket.org/pypy/pypy/src/default/dotviewer/
-.. _`lib-python/`: https://bitbucket.org/pypy/pypy/src/default/lib-python/
-.. _`lib-python/2.7/dis.py`: https://bitbucket.org/pypy/pypy/src/default/lib-python/2.7/dis.py
-.. _`lib_pypy/`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/
-.. _`lib_pypy/greenlet.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/greenlet.py
-.. _`lib_pypy/tputil.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/tputil.py
-.. _`pypy/bin/`: https://bitbucket.org/pypy/pypy/src/default/pypy/bin/
-.. _`pypy/bin/pyinteractive.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/bin/pyinteractive.py
-.. _`pypy/config/`: https://bitbucket.org/pypy/pypy/src/default/pypy/config/
-.. _`pypy/config/pypyoption.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/config/pypyoption.py
-.. _`pypy/doc/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/
-.. _`pypy/doc/config/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/config/
-.. _`pypy/doc/discussion/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/discussion/
-.. _`pypy/goal/`: https://bitbucket.org/pypy/pypy/src/default/pypy/goal/
-.. _`pypy/interpreter`:
-.. _`pypy/interpreter/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/
-.. _`pypy/interpreter/argument.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/argument.py
-.. _`pypy/interpreter/astcompiler`:
-.. _`pypy/interpreter/astcompiler/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/
-.. _`pypy/interpreter/astcompiler/assemble.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/assemble.py
-.. _`pypy/interpreter/astcompiler/ast.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/ast.py
-.. _`pypy/interpreter/astcompiler/astbuilder.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/astbuilder.py
-.. _`pypy/interpreter/astcompiler/asthelpers.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/asthelpers.py
-.. _`pypy/interpreter/astcompiler/codegen.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/codegen.py
-.. _`pypy/interpreter/astcompiler/optimize.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/optimize.py
-.. _`pypy/interpreter/astcompiler/symtable.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/symtable.py
-.. _`pypy/interpreter/astcompiler/tools/Python.asdl`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/tools/Python.asdl
-.. _`pypy/interpreter/astcompiler/tools/asdl_py.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/tools/asdl_py.py
-.. _`pypy/interpreter/baseobjspace.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/baseobjspace.py
-.. _`pypy/interpreter/eval.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/eval.py
-.. _`pypy/interpreter/executioncontext.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/executioncontext.py
-.. _`pypy/interpreter/function.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/function.py
-.. _`pypy/interpreter/gateway.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/gateway.py
-.. _`pypy/interpreter/mixedmodule.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/mixedmodule.py
-.. _`pypy/interpreter/module.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/module.py
-.. _`pypy/interpreter/pyframe.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyframe.py
-.. _`pypy/interpreter/pyopcode.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyopcode.py
-.. _`pypy/interpreter/pyparser`:
-.. _`pypy/interpreter/pyparser/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/
-.. _`pypy/interpreter/pyparser/future.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/future.py
-.. _`pypy/interpreter/pyparser/metaparser.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/metaparser.py
-.. _`pypy/interpreter/pyparser/parser.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/parser.py
-.. _`pypy/interpreter/pyparser/pyparse.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/pyparse.py
-.. _`pypy/interpreter/pyparser/pytokenizer.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/pytokenizer.py
-.. _`pypy/interpreter/typedef.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/typedef.py
-.. _`pypy/module`:
-.. _`pypy/module/`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/
-.. _`pypy/module/__builtin__/__init__.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/__builtin__/__init__.py
-.. _`pypy/module/cppyy/capi/__init__.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/capi/__init__.py
-.. _`pypy/module/cppyy/capi/builtin_capi.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/capi/builtin_capi.py
-.. _`pypy/module/cppyy/include/capi.h`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/include/capi.h
-.. _`pypy/module/test_lib_pypy/`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/test_lib_pypy/
-.. _`pypy/objspace/`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/
-.. _`pypy/objspace/std`:
-.. _`pypy/objspace/std/`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/
-.. _`pypy/objspace/std/bytesobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/bytesobject.py
-.. _`pypy/objspace/std/multimethod.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/multimethod.py
-.. _`pypy/objspace/std/objspace.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/objspace.py
-.. _`pypy/objspace/std/proxy_helpers.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/proxy_helpers.py
-.. _`pypy/objspace/std/proxyobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/proxyobject.py
-.. _`pypy/objspace/std/strbufobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/strbufobject.py
-.. _`pypy/objspace/std/transparent.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/transparent.py
-.. _`pypy/tool/`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/
-.. _`pypy/tool/pytest/`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/pytest/
-.. _`rpython/annotator`:
-.. _`rpython/annotator/`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/
-.. _`rpython/annotator/annrpython.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/annrpython.py
-.. _`rpython/annotator/binaryop.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/binaryop.py
-.. _`rpython/annotator/builtin.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/builtin.py
-.. _`rpython/bin/translatorshell.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/bin/translatorshell.py
-.. _`rpython/config/`: https://bitbucket.org/pypy/pypy/src/default/rpython/config/
-.. _`rpython/config/translationoption.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/config/translationoption.py
-.. _`rpython/flowspace/`: https://bitbucket.org/pypy/pypy/src/default/rpython/flowspace/
-.. _`rpython/flowspace/model.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/flowspace/model.py
-.. _`rpython/memory/`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/
-.. _`rpython/memory/gc/generation.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/generation.py
-.. _`rpython/memory/gc/hybrid.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/hybrid.py
-.. _`rpython/memory/gc/minimarkpage.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/minimarkpage.py
-.. _`rpython/memory/gc/semispace.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/semispace.py
-.. _`rpython/rlib`:
-.. _`rpython/rlib/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/
-.. _`rpython/rlib/listsort.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/listsort.py
-.. _`rpython/rlib/nonconst.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/nonconst.py
-.. _`rpython/rlib/objectmodel.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/objectmodel.py
-.. _`rpython/rlib/parsing/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/parsing/
-.. _`rpython/rlib/parsing/tree.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/parsing/tree.py
-.. _`rpython/rlib/rarithmetic.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rarithmetic.py
-.. _`rpython/rlib/rbigint.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rbigint.py
-.. _`rpython/rlib/rrandom.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rrandom.py
-.. _`rpython/rlib/rsocket.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rsocket.py
-.. _`rpython/rlib/streamio.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/streamio.py
-.. _`rpython/rlib/test`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/test/
-.. _`rpython/rlib/unroll.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/unroll.py
-.. _`rpython/rtyper`:
-.. _`rpython/rtyper/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/
-.. _`rpython/rtyper/lltypesystem/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/lltypesystem/
-.. _`rpython/rtyper/lltypesystem/lltype.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/lltypesystem/lltype.py
-.. _`rpython/rtyper/rint.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rint.py
-.. _`rpython/rtyper/rlist.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rlist.py
-.. _`rpython/rtyper/rmodel.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rmodel.py
-.. _`rpython/rtyper/rtyper.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rtyper.py
-.. _`rpython/rtyper/test/test_llinterp.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/test/test_llinterp.py
-.. _`rpython/tool/algo/`: https://bitbucket.org/pypy/pypy/src/default/rpython/tool/algo/
-.. _`rpython/translator`:
-.. _`rpython/translator/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/
-.. _`rpython/translator/backendopt/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/backendopt/
-.. _`rpython/translator/c/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/
-.. _`rpython/translator/c/src/stacklet/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/src/stacklet/
-.. _`rpython/translator/c/src/stacklet/stacklet.h`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/src/stacklet/stacklet.h
-.. _`rpython/translator/tool/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/tool/
diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -1,93 +1,29 @@
-==================================================
-Goals and Architecture Overview 
-==================================================
+Goals and Architecture Overview
+===============================
 
 .. contents::
 
+This document gives an overview of the goals and architecture of PyPy. If you're
+interested in :ref:`using PyPy <using-pypy>` or :ref:`hacking on it <developing-pypy>`,
+have a look at our :ref:`getting started <getting-started-index>` section.
 
-This document gives an overview of the goals and architecture of PyPy.
-See `getting started`_ for a practical introduction and starting points. 
 
-Mission statement 
-====================
+Mission statement
+-----------------
 
-We aim to provide:
+We aim to provide a compliant, flexible and fast implementation of the Python_
+Language which uses the RPython toolchain to enable new advanced high-level
+features without having to encode the low-level details.  We call this PyPy.
 
- * a common translation and support framework for producing
-   implementations of dynamic languages, emphasizing a clean
-   separation between language specification and implementation
-   aspects.  We call this the `RPython toolchain`_.
+.. _Python: http://docs.python.org/reference/
 
- * a compliant, flexible and fast implementation of the Python_ Language 
-   which uses the above toolchain to enable new advanced high-level features 
-   without having to encode the low-level details.  We call this PyPy.
-
-By separating concerns in this way, our implementation
-of Python - and other dynamic languages - is able to automatically
-generate a Just-in-Time compiler for any dynamic language.  It also
-allows a mix-and-match approach to implementation decisions, including
-many that have historically been outside of a user's control, such as
-target platform, memory and 
-threading models, garbage collection strategies, and optimizations applied, 
-including whether or not to have a JIT in the first place.
 
 High Level Goals
-=============================
-
-RPython - the Translation Toolchain
------------------------------------------------
-
-Traditionally, language interpreters are written in a target platform language
-such as C/Posix, Java or C#.  Each implementation provides 
-a fundamental mapping between application source code and the target 
-environment.  One of 
-the goals of the "all-encompassing" environments, such as the .NET framework
-and to some extent the Java virtual machine, is to provide standardized
-and higher level functionalities in order to support language implementers
-for writing language implementations. 
-
-PyPy is experimenting with a more ambitious approach.  We are using a
-subset of the high-level language Python, called RPython_, in which we
-write languages as simple interpreters with few references to and
-dependencies on lower level details.  The `RPython toolchain`_
-produces a concrete virtual machine for the platform of our choice by
-inserting appropriate lower level aspects.  The result can be customized
-by selecting other feature and platform configurations.
-
-Our goal is to provide a possible solution to the problem of language
-implementers: having to write ``l * o * p`` interpreters for ``l``
-dynamic languages and ``p`` platforms with ``o`` crucial design
-decisions.  PyPy aims at making it possible to change each of these
-variables independently such that:
-
-* ``l``: the language that we analyze can be evolved or entirely replaced;
-
-* ``o``: we can tweak and optimize the translation process to produce 
-  platform specific code based on different models and trade-offs;
-
-* ``p``: we can write new translator back-ends to target different
-  physical and virtual platforms.
-
-By contrast, a standardized target environment - say .NET -
-enforces ``p=1`` as far as it's concerned.  This helps making ``o`` a
-bit smaller by providing a higher-level base to build upon.  Still,
-we believe that enforcing the use of one common environment 
-is not necessary.  PyPy's goal is to give weight to this claim - at least 
-as far as language implementation is concerned - showing an approach
-to the ``l * o * p`` problem that does not rely on standardization.
-
-The most ambitious part of this goal is to `generate Just-In-Time
-Compilers`_ in a language-independent way, instead of only translating
-the source interpreter into an interpreter for the target platform.
-This is an area of language implementation that is commonly considered
-very challenging because of the involved complexity.
-
-
-PyPy - the Python Interpreter
---------------------------------------------
+----------------
 
 Our main motivation for developing the translation framework is to
-provide a full featured, customizable, fast_ and `very compliant`_ Python
+provide a full featured, customizable, :ref:`fast <how-fast-is-pypy>` and
+:doc:`very compliant <cpython_differences>` Python
 implementation, working on and interacting with a large variety of
 platforms and allowing the quick introduction of new advanced language
 features.
@@ -106,88 +42,22 @@
 and fully orthogonal to the interpreter source code.
 
 
-PyPy Architecture 
-===========================
-
-As you would expect from a project implemented using ideas from the world
-of `Extreme Programming`_, the architecture of PyPy has evolved over time
-and continues to evolve.  Nevertheless, the high level architecture is 
-stable. As described above, there are two rather independent basic
-subsystems: the `PyPy Python Interpreter`_ and the `RPython Translation Toolchain`_.
-
-.. _`translation framework`:
-
-RPython Translation Toolchain
--------------------------
-
-The job of the RPython toolchain is to translate RPython_ programs
-into an efficient version of that program for one of the various target
-platforms, generally one that is considerably lower-level than Python.
-
-The approach we have taken is to reduce the level of abstraction of the
-source RPython program in several steps, from the high level down to the
-level of the target platform, whatever that may be.  Currently we
-support two broad flavours of target platforms: the ones that assume a
-C-like memory model with structures and pointers, and the ones that
-assume an object-oriented model with classes, instances and methods (as,
-for example, the Java and .NET virtual machines do).
-
-The RPython toolchain never sees the RPython source code or syntax
-trees, but rather starts with the *code objects* that define the
-behaviour of the function objects one gives it as input.  It can be
-considered as "freezing" a pre-imported RPython program into an
-executable form suitable for the target platform.
-
-The steps of the translation process can be summarized as follows:
-
-* The code object of each source functions is converted to a `control
-  flow graph` by the `Flow Object Space`_.
-
-* The control flow graphs are processed by the Annotator_, which
-  performs whole-program type inference to annotate each variable of
-  the control flow graph with the types it may take at run-time.
-
-* The information provided by the annotator is used by the RTyper_ to
-  convert the high level operations of the control flow graphs into
-  operations closer to the abstraction level of the target platform.
-
-* Optionally, `various transformations`_ can then be applied which, for
-  example, perform optimizations such as inlining, add capabilities
-  such as stackless-style concurrency, or insert code for the
-  `garbage collector`_.
-
-* Then, the graphs are converted to source code for the target platform
-  and compiled into an executable.
-
-This process is described in much more detail in the `document about
-the RPython toolchain`_ and in the paper `Compiling dynamic language
-implementations`_.
-
-.. _`control flow graph`: translation.html#the-flow-model
-.. _`Flow Object Space`: objspace.html#the-flow-object-space
-.. _Annotator: translation.html#the-annotation-pass
-.. _RTyper: rtyper.html#overview
-.. _`various transformations`: translation.html#the-optional-transformations
-.. _`document about the RPython toolchain`: translation.html
-.. _`garbage collector`: garbage_collection.html
-.. _`RPython toolchain`: translation.html
-.. _`standard interpreter`: 
-.. _`python interpreter`: 
+.. _python-interpreter:
 
 PyPy Python Interpreter
--------------------------------------
+-----------------------
 
 PyPy's *Python Interpreter* is written in RPython and implements the
 full Python language.  This interpreter very closely emulates the
 behavior of CPython.  It contains the following key components:
 
-- a bytecode compiler responsible for producing Python code objects 
+- a bytecode compiler responsible for producing Python code objects
   from the source code of a user application;
 
-- a `bytecode evaluator`_ responsible for interpreting 
+- a :doc:`bytecode evaluator <interpreter>` responsible for interpreting
   Python code objects;
 
-- a `standard object space`_, responsible for creating and manipulating
+- a :ref:`standard object space <standard-object-space>`, responsible for creating and manipulating
   the Python objects seen by the application.
 
 The *bytecode compiler* is the preprocessing phase that produces a
@@ -200,64 +70,6 @@
 lists, as well as the operations between them, like addition or
 truth-value-testing.
 
-This division between bytecode evaluator and object space is very
-important, as it gives a lot of flexibility.  One can plug in 
-different `object spaces`_ to get different or enriched behaviours 
-of the Python objects.  Additionally, a special more abstract object
-space, the `flow object space`_, allows us to reuse the bytecode
-evaluator for our translation framework.
-
-.. _`bytecode evaluator`: interpreter.html
-.. _`standard object space`: objspace.html#the-standard-object-space
-.. _`object spaces`: objspace.html
-.. _`flow object space`: objspace.html#the-flow-object-space
-
-.. _`the translation framework`:
-
-
-Further reading
-===============
-
-All of PyPy's documentation can be reached from the `documentation
-index`_.  Of particular interest after reading this document might be:
-
- * `getting-started`_: a hands-on guide to getting involved with the
-   PyPy source code.
-
- * `PyPy's approach to virtual machine construction`_: a paper
-   presented to the Dynamic Languages Symposium attached to OOPSLA
-   2006.
-
- * `The translation document`_: a detailed description of our
-   translation process.
-
- * `JIT Generation in PyPy`_, describing how we produce a Just-in-time
-   Compiler from an interpreter.
-
- * A tutorial of how to use the `RPython toolchain`_ to `implement your own
-   interpreter`_.
-
-.. _`documentation index`: index.html#project-documentation
-.. _`getting-started`: getting-started-dev.html
-.. _`PyPy's approach to virtual machine construction`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dls2006/pypy-vm-construction.pdf
-.. _`the translation document`: translation.html
-.. _`RPython toolchain`: translation.html
-.. _`Compiling dynamic language implementations`: https://bitbucket.org/pypy/extradoc/raw/tip/eu-report/D05.1_Publish_on_translating_a_very-high-level_description.pdf
-.. _`Technical reports`: index-report.html
-
-.. _`getting started`: getting-started.html
-.. _`Extreme Programming`: http://www.extremeprogramming.org/
-
-.. _fast: faq.html#how-fast-is-pypy
-.. _`very compliant`: cpython_differences.html
-
-.. _`RPython`: coding-guide.html#rpython
-
-.. _Python: http://docs.python.org/reference/
-.. _Psyco: http://psyco.sourceforge.net
-.. _`generate Just-In-Time Compilers`: jit/index.html
-.. _`JIT Generation in PyPy`: jit/index.html
-.. _`implement your own interpreter`: http://morepypy.blogspot.com/2011/04/tutorial-writing-interpreter-with-pypy.html
-
-.. include:: _ref.txt
-
+This division between bytecode evaluator and object space gives a lot of
+flexibility.  One can plug in different :doc:`object spaces <objspace>` to get
+different or enriched behaviours of the Python objects.
diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst
deleted file mode 100644
--- a/pypy/doc/arm.rst
+++ /dev/null
@@ -1,166 +0,0 @@
-=========================
-Cross-translating for ARM
-=========================
-
-
-Here we describe the setup required and the steps needed to follow to translate
-an interpreter using the RPython translator to target ARM using a cross
-compilation toolchain.
-
-To translate an RPython program for ARM we can either
-translate directly on an ARM device following the normal translation steps. Unfortunately this is not really feasible on most ARM machines. The alternative is to cross-translate using a cross-compilation toolchain.
-
-To cross-translate we run the translation on a more powerful (usually
-x86) machine and generate a binary for ARM using a cross-compiler to compile
-the generated C code. There are several constraints when doing this. In
-particular we currently only support Linux as translation host and target
-platforms (tested on Ubuntu). Also we need a 32-bit environment to run the
-translation. This can be done either on a 32bit host or in 32bit chroot.
-
-
-Requirements
-------------
-
-The tools required to cross translate from a Linux based host to an ARM based Linux target are:
-
-- A checkout of PyPy (default branch).
-- The GCC ARM cross compiler (on Ubuntu it is the ``gcc-arm-linux-gnueabi package``) but other toolchains should also work.
-- Scratchbox 2, a cross-compilation engine (``scratchbox2`` Ubuntu package).
-- A 32-bit PyPy or Python.
-- And the following (or corresponding) packages need to be installed to create an ARM based chroot:
-
-  * ``debootstrap`` 
-  * ``schroot``
-  * ``binfmt-support``
-  * ``qemu-system``
-  * ``qemu-user-static``
-
-- The dependencies above are in addition to the ones needed for a regular
-  translation, `listed here`_.
-
-.. _`listed here`: getting-started-python.html#translating-the-pypy-python-interpreter
-
-
-Creating a Qemu based ARM chroot
---------------------------------
-
-First we will need to create a rootfs containing the packages and dependencies
-required in order to translate PyPy or other interpreters. We are going to
-assume, that the files will be placed in ``/srv/chroot/precise_arm``.
-
-Create the rootfs by calling:
-
-::
-
-   mkdir -p /srv/chroot/precise_arm
-   qemu-debootstrap --variant=buildd --arch=armel precise /srv/chroot/precise_arm/  http://ports.ubuntu.com/ubuntu-ports/
-
-Next, copy the qemu-arm-static binary to the rootfs.
-
-:: 
-
-  cp /usr/bin/qemu-arm-static /srv/chroot/precise_arm/usr/bin/qemu-arm-static
-
-For easier configuration and management we will create a schroot pointing to
-the rootfs. We need to add a configuration block (like the one below) to the
-schroot configuration file in /etc/schroot/schroot.conf.
-
-
-::
-
-  [precise_arm]
-  directory=/srv/chroot/precise_arm
-  users=USERNAME
-  root-users=USERNAME
-  groups=users
-  aliases=default
-  type=directory
-
-
-To verify that everything is working in the chroot, running ``schroot -c
-precise_arm`` should start a shell running in the schroot environment using
-qemu-arm to execute the ARM binaries. Running ``uname -m`` in the chroot should 
-yeild a result like ``armv7l``. Showing that we are emulating an ARM system.
-
-Start the schroot as the user root in order to configure the apt sources and
-to install the following packages:
-
-
-::
-
-  schroot -c precise_arm -u root
-  echo "deb http://ports.ubuntu.com/ubuntu-ports/ precise main universe restricted" > /etc/apt/sources.list
-  apt-get update
-  apt-get install libffi-dev libgc-dev python-dev build-essential libncurses5-dev libbz2-dev
-
-
-Now all dependencies should be in place and we can exit the schroot environment.
-
-
-Configuring scratchbox2
------------------------
-
-To configure the scratchbox we need to cd into the root directory of the rootfs
-we created before. From there we can call the sb2 configuration tools which
-will take the current directory as the base directory for the scratchbox2
-environment.
-
-::
-
-  cd /srv/chroot/precise_arm
-  sb2-init -c `which qemu-arm` ARM `which arm-linux-gnueabi-gcc`
-
-This will create a scratchbox2 based environment called ARM that maps calls to
-gcc done within the scratchbox to the arm-linux-gnueabi-gcc outside the
-scratchbox. Now we should have a working cross compilation toolchain in place
-and can start cross-translating programs for ARM.
-
-Translation
------------
-
-Having performed all the preliminary steps we should now be able to cross
-translate a program for ARM.  You can use this_ minimal
-target to test your setup before applying it to a larger project.
-
-Before starting the translator we need to set two environment variables, so the
-translator knows how to use the scratchbox environment. We need to set the
-**SB2** environment variable to point to the rootfs and the **SB2OPT** should
-contain the command line options for the sb2 command. If our rootfs is in the
-folder /srv/chroot/precise_arm and the scratchbox environment is called "ARM",
-the variables would be defined as follows.
-
-
-::
-
-  export SB2=/srv/chroot/precise_arm
-  export SB2OPT='-t ARM'
-
-Once this is set, you can call the translator. For example save this file 
-
-::
-
-  def main(args):
-      print "Hello World"
-      return 0
-
-  def target(*args):
-      return main, None
-
-and call the translator
-
-::
-
-  pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O1 --platform=arm target.py
-
-If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``.
-
-To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call      
-
-::
-
-  pypy <path to rpython>/rpython/bin/rpython -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py
-
-The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_
-
-.. _`issue 1377`: https://bugs.pypy.org/issue1377
-.. _`issue 1376`: https://bugs.pypy.org/issue1376
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/build.rst
@@ -0,0 +1,170 @@
+Building PyPy from Source
+=========================
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Even when using PyPy to build PyPy, translation is time-consuming -- 30
+minutes on a fast machine -- and RAM-hungry.  You will need **at least** 2 GB
+of memory on a 32-bit machine and 4GB on a 64-bit machine.
+
+
+Clone the repository
+--------------------
+
+If you prefer to compile your own PyPy, or if you want to modify it, you
+will need to obtain a copy of the sources.  This can be done either by
+`downloading them from the download page`_ or by checking them out from the
+repository using mercurial.  We suggest using mercurial if you want to access
+the current development.
+
+.. _downloading them from the download page: http://pypy.org/download.html
+
+You must issue the following command on your
+command line, DOS box, or terminal::
+
+    hg clone http://bitbucket.org/pypy/pypy pypy
+
+This will clone the repository and place it into a directory
+named ``pypy``, and will get you the PyPy source in ``pypy/pypy`` and
+documentation files in ``pypy/pypy/doc``.
+We try to ensure that the tip is always stable, but it might
+occasionally be broken.  You may want to check out `our nightly tests`_:
+find a revision (12-chars alphanumeric string, e.g. "963e808156b3")
+that passed at least the
+``{linux32}`` tests (corresponding to a ``+`` sign on the
+line ``success``) and then, in your cloned repository, switch to this revision
+using::
+
+    hg up -r XXXXX
+
+where XXXXX is the revision id.
+
+.. _our nightly tests: http://buildbot.pypy.org/summary?branch=<trunk>
+
+
+Install build-time dependencies
+-------------------------------
+
+To build PyPy on Unix using the C translation backend, you need at least a C
+compiler and ``make`` installed. Further, some optional modules have additional
+dependencies:
+
+cffi, ctypes
+    libffi, pkg-config
+
+zlib
+    libz
+
+bz2
+    libbz2
+
+lzma (PyPy3 only)
+    liblzma
+
+sqlite3
+    libsqlite3
+
+curses
+    libncurses + cffi dependencies from above
+
+pyexpat
+    libexpat1
+
+_ssl
+    libssl
+
+Make sure to have these libraries (with development headers) installed before
+building PyPy, otherwise the resulting binary will not contain these modules.
+
+On Debian, this is the command to install all build-time dependencies::
+
+    apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
+    libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev
+
+For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
+
+On Fedora::
+
+    yum install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
+    lib-sqlite3-devel ncurses-devel expat-devel openssl-devel
+
+For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+
+On SLES11::
+
+    zypper install gcc make python-devel pkg-config \
+    zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
+    libexpat-devel libffi-devel python-curses
+
+For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+
+On Mac OS X, most of these build-time dependencies are installed alongside
+the Developer Tools. However, note that in order for the installation to
+find them you may need to run::
+
+    xcode-select --install
+
+
+Run the translation
+-------------------
+
+Translate with JIT::
+
+    pypy rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone.py
+
+Translate without JIT::
+
+    pypy rpython/bin/rpython --opt=2 pypy/goal/targetpypystandalone.py
+
+(You can use ``python`` instead of ``pypy`` here, which will take longer
+but works too.)
+
+If everything works correctly this will create an executable ``pypy-c`` in the
+current directory. The executable behaves mostly like a normal Python
+interpreter (see :doc:`cpython_differences`).
+
+
+.. _translate-pypy:
+
+Translating with non-standard options
+-------------------------------------
+
+It is possible to have non-standard features enabled for translation,
+but they are not really tested any more.  Look, for example, at the
+:doc:`objspace proxies <objspace-proxies>` document.
+
+
+
+Installation
+------------
+
+PyPy dynamically finds the location of its libraries depending on the location
+of the executable. The directory hierarchy of a typical PyPy installation
+looks like this::
+
+    ./bin/pypy
+    ./include/
+    ./lib_pypy/
+    ./lib-python/2.7
+    ./site-packages/
+
+The hierarchy shown above is relative to a PREFIX directory. PREFIX is
+computed by starting from the directory where the executable resides, and
+"walking up" the filesystem until we find a directory containing ``lib_pypy``
+and ``lib-python/2.7``.
+
+To install PyPy system wide on unix-like systems, it is recommended to put the
+whole hierarchy alone (e.g. in ``/opt/pypy``) and put a symlink to the
+``pypy`` executable into ``/usr/bin`` or ``/usr/local/bin``.
+
+If the executable fails to find suitable libraries, it will report ``debug:
+WARNING: library path not found, using compiled-in sys.path`` and then attempt
+to continue normally. If the default path is usable, most code will be fine.
+However, the ``sys.prefix`` will be unset and some existing libraries assume
+that this is never the case.
+
+
+.. TODO windows
diff --git a/pypy/doc/cleanup.rst b/pypy/doc/cleanup.rst
deleted file mode 100644
--- a/pypy/doc/cleanup.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-Old documentation that needs review
------------------------------------
-
-.. The following stuff is old (and crufty?), and needs further investigation:
-
-.. doc-index: This needs merging somehow
-
-.. toctree::
-
-   distribution.rst
-
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -1,21 +1,17 @@
-====================================
 Coding Guide
-====================================
+============
 
 .. contents::
 
 This document describes coding requirements and conventions for
 working with the PyPy code base.  Please read it carefully and
 ask back any questions you might have. The document does not talk
-very much about coding style issues. We mostly follow `PEP 8`_ though.
+very much about coding style issues. We mostly follow :pep:`8` though.
 If in doubt, follow the style that is already present in the code base.
 
-.. _`PEP 8`: http://www.python.org/dev/peps/pep-0008/
-
-.. _`RPython`:
 
 Overview and motivation
-========================
+------------------------
 
 We are writing a Python interpreter in Python, using Python's well known
 ability to step behind the algorithmic problems as a language. At first glance,
@@ -25,7 +21,7 @@
 
 
 CPython vs. PyPy
--------------------
+~~~~~~~~~~~~~~~~
 
 Compared to the CPython implementation, Python takes the role of the C
 Code. We rewrite the CPython interpreter in Python itself.  We could
@@ -44,20 +40,20 @@
 but let's stick with this somewhat canonical approach.
 
 
-.. _`application-level`:
-.. _`interpreter-level`:
+.. _application-level:
+.. _interpreter-level:
 
 Application-level and interpreter-level execution and objects
--------------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Since Python is used for implementing all of our code base, there is a
-crucial distinction to be aware of: that between *interpreter-level* objects and 
+crucial distinction to be aware of: that between *interpreter-level* objects and
 *application-level* objects.  The latter are the ones that you deal with
 when you write normal python programs.  Interpreter-level code, however,
 cannot invoke operations nor access attributes from application-level
 objects.  You will immediately recognize any interpreter level code in
 PyPy, because half the variable and object names start with a ``w_``, which
-indicates that they are `wrapped`_ application-level values. 
+indicates that they are `wrapped`_ application-level values.
 
 Let's show the difference with a simple example.  To sum the contents of
 two variables ``a`` and ``b``, one would write the simple application-level
@@ -80,10 +76,10 @@
 interpreting.
 
 
-.. _`app-preferable`: 
+.. _app-preferable:
 
-Application level is often preferable 
--------------------------------------
+Application level is often preferable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Application-level code is substantially higher-level, and therefore
 correspondingly easier to write and debug.  For example, suppose we want
@@ -113,11 +109,11 @@
             space.setitem(w_self, w_key, w_value)
 
 This interpreter-level implementation looks much more similar to the C
-source code.  It is still more readable than its C counterpart because 
-it doesn't contain memory management details and can use Python's native 
-exception mechanism. 
+source code.  It is still more readable than its C counterpart because
+it doesn't contain memory management details and can use Python's native
+exception mechanism.
 
-In any case, it should be obvious that the application-level implementation 
+In any case, it should be obvious that the application-level implementation
 is definitely more readable, more elegant and more maintainable than the
 interpreter-level one (and indeed, dict.update is really implemented at
 applevel in PyPy).
@@ -129,10 +125,11 @@
 level code is usually preferable.  We have an abstraction (called the
 'Gateway') which allows the caller of a function to remain ignorant of
 whether a particular function is implemented at application or
-interpreter level. 
+interpreter level.
+
 
 Our runtime interpreter is "RPython"
-----------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In order to make a C code generator feasible all code on interpreter level has
 to restrict itself to a subset of the Python language, and we adhere to some
@@ -159,7 +156,7 @@
 elegant: For the definition of all the opcodes of the Python
 interpreter, the module ``dis`` is imported and used to initialize our
 bytecode interpreter.  (See ``__initclass__`` in
-`pypy/interpreter/pyopcode.py`_).  This
+:source:`pypy/interpreter/pyopcode.py`).  This
 saves us from adding extra modules to PyPy. The import code is run at
 startup time, and we are allowed to use the CPython builtin import
 function.
@@ -172,308 +169,13 @@
 enables the code generator to emit efficient machine level replacements
 for pure integer objects, for instance.
 
-RPython
-=================
-
-RPython Definition
-------------------
-
-RPython is a restricted subset of Python that is amenable to static analysis.
-Although there are additions to the language and some things might surprisingly
-work, this is a rough list of restrictions that should be considered. Note
-that there are tons of special cased restrictions that you'll encounter
-as you go. The exact definition is "RPython is everything that our translation
-toolchain can accept" :)
-
-.. _`wrapped object`: coding-guide.html#wrapping-rules
-
-Flow restrictions
--------------------------
-
-**variables**
-
-  variables should contain values of at most one type as described in
-  `Object restrictions`_ at each control flow point, that means for
-  example that joining control paths using the same variable to
-  contain both a string and a int must be avoided.  It is allowed to
-  mix None (basically with the role of a null pointer) with many other
-  types: `wrapped objects`, class instances, lists, dicts, strings, etc.
-  but *not* with int, floats or tuples.
-
-**constants**
-
-  all module globals are considered constants.  Their binding must not
-  be changed at run-time.  Moreover, global (i.e. prebuilt) lists and
-  dictionaries are supposed to be immutable: modifying e.g. a global
-  list will give inconsistent results.  However, global instances don't
-  have this restriction, so if you need mutable global state, store it
-  in the attributes of some prebuilt singleton instance.
-
-
-
-**control structures**
-
-  all allowed, ``for`` loops restricted to builtin types, generators
-  very restricted.
-
-**range**
-
-  ``range`` and ``xrange`` are identical. ``range`` does not necessarily create an array,
-  only if the result is modified. It is allowed everywhere and completely
-  implemented. The only visible difference to CPython is the inaccessibility
-  of the ``xrange`` fields start, stop and step.
-
-**definitions**
-
-  run-time definition of classes or functions is not allowed.
-
-**generators**
-
-  generators are supported, but their exact scope is very limited. you can't
-  merge two different generator in one control point.
-
-**exceptions**
-
-+ fully supported
-+ see below `Exception rules`_ for restrictions on exceptions raised by built-in operations
-
-
-Object restrictions
--------------------------
-
-We are using
-
-**integer, float, boolean**
-
-  works.
-
-**strings**
-
-  a lot of, but not all string methods are supported and those that are
-  supported, not necesarilly accept all arguments.  Indexes can be
-  negative.  In case they are not, then you get slightly more efficient
-  code if the translator can prove that they are non-negative.  When
-  slicing a string it is necessary to prove that the slice start and
-  stop indexes are non-negative. There is no implicit str-to-unicode cast
-  anywhere. Simple string formatting using the ``%`` operator works, as long
-  as the format string is known at translation time; the only supported
-  formatting specifiers are ``%s``, ``%d``, ``%x``, ``%o``, ``%f``, plus
-  ``%r`` but only for user-defined instances. Modifiers such as conversion
-  flags, precision, length etc. are not supported. Moreover, it is forbidden
-  to mix unicode and strings when formatting.
-
-**tuples**
-
-  no variable-length tuples; use them to store or return pairs or n-tuples of
-  values. Each combination of types for elements and length constitute
-  a separate and not mixable type.
-
-**lists**
-
-  lists are used as an allocated array.  Lists are over-allocated, so list.append()
-  is reasonably fast. However, if you use a fixed-size list, the code
-  is more efficient. Annotator can figure out most of the time that your
-  list is fixed-size, even when you use list comprehension.
-  Negative or out-of-bound indexes are only allowed for the
-  most common operations, as follows:
-
-  - *indexing*:
-    positive and negative indexes are allowed. Indexes are checked when requested
-    by an IndexError exception clause.
-  
-  - *slicing*:
-    the slice start must be within bounds. The stop doesn't need to, but it must
-    not be smaller than the start.  All negative indexes are disallowed, except for
-    the [:-1] special case.  No step.  Slice deletion follows the same rules.
-    
-  - *slice assignment*:
-    only supports ``lst[x:y] = sublist``, if ``len(sublist) == y - x``.
-    In other words, slice assignment cannot change the total length of the list,
-    but just replace items.
-
-  - *other operators*:
-    ``+``, ``+=``, ``in``, ``*``, ``*=``, ``==``, ``!=`` work as expected.
-
-  - *methods*:
-    append, index, insert, extend, reverse, pop.  The index used in pop() follows
-    the same rules as for *indexing* above.  The index used in insert() must be within
-    bounds and not negative.
-
-**dicts**
-
-  dicts with a unique key type only, provided it is hashable. Custom
-  hash functions and custom equality will not be honored.
-  Use ``rpython.rlib.objectmodel.r_dict`` for custom hash functions.
-
-
-**list comprehensions**
-
-  May be used to create allocated, initialized arrays.
-
-**functions**
-
-+ statically called functions may use defaults and a variable number of
-  arguments (which may be passed as a list instead of a tuple, so write code
-  that does not depend on it being a tuple).
-
-+ dynamic dispatch enforces the use of signatures that are equal for all
-  possible called function, or at least "compatible enough".  This
-  concerns mainly method calls, when the method is overridden or in any
-  way given different definitions in different classes.  It also concerns
-  the less common case of explicitly manipulated function objects.
-  Describing the exact compatibility rules is rather involved (but if you
-  break them, you should get explicit errors from the rtyper and not
-  obscure crashes.)
-
-**builtin functions**
-
-  A number of builtin functions can be used.  The precise set can be
-  found in `rpython/annotator/builtin.py`_ (see ``def builtin_xxx()``).
-  Some builtin functions may be limited in what they support, though.
-
-  ``int, float, str, ord, chr``... are available as simple conversion
-  functions.  Note that ``int, float, str``... have a special meaning as
-  a type inside of isinstance only.
-
-**classes**
-
-+ methods and other class attributes do not change after startup
-+ single inheritance is fully supported
-+ use `rpython.rlib.objectmodel.import_from_mixin(M)` in a class
-  body to copy the whole content of a class `M`.  This can be used
-  to implement mixins: functions and staticmethods are duplicated
-  (the other class attributes are just copied unmodified).
-
-+ classes are first-class objects too
-
-**objects**
-
-  Normal rules apply. The only special methods that are honoured are
-  ``__init__``, ``__del__``, ``__len__``, ``__getitem__``, ``__setitem__``,
-  ``__getslice__``, ``__setslice__``, and ``__iter__``. To handle slicing,
-  ``__getslice__`` and ``__setslice__`` must be used; using ``__getitem__`` and
-   ``__setitem__`` for slicing isn't supported. Additionally, using negative
-   indices for slicing is still not support, even when using ``__getslice__``.
-
-This layout makes the number of types to take care about quite limited.
-
-
-Integer Types
--------------------------
-
-While implementing the integer type, we stumbled over the problem that
-integers are quite in flux in CPython right now. Starting with Python 2.4,
-integers mutate into longs on overflow.  In contrast, we need
-a way to perform wrap-around machine-sized arithmetic by default, while still
-being able to check for overflow when we need it explicitly.  Moreover, we need
-a consistent behavior before and after translation.
-
-We use normal integers for signed arithmetic.  It means that before
-translation we get longs in case of overflow, and after translation we get a
-silent wrap-around.  Whenever we need more control, we use the following
-helpers (which live in `rpython/rlib/rarithmetic.py`_):
-
-**ovfcheck()**
-
-  This special function should only be used with a single arithmetic operation
-  as its argument, e.g. ``z = ovfcheck(x+y)``.  Its intended meaning is to
-  perform the given operation in overflow-checking mode.
-
-  At run-time, in Python, the ovfcheck() function itself checks the result
-  and raises OverflowError if it is a ``long``.  But the code generators use
-  ovfcheck() as a hint: they replace the whole ``ovfcheck(x+y)`` expression
-  with a single overflow-checking addition in C.
-
-**intmask()**
-
-  This function is used for wrap-around arithmetic.  It returns the lower bits
-  of its argument, masking away anything that doesn't fit in a C "signed long int".
-  Its purpose is, in Python, to convert from a Python ``long`` that resulted from a
-  previous operation back to a Python ``int``.  The code generators ignore
-  intmask() entirely, as they are doing wrap-around signed arithmetic all the time
-  by default anyway.  (We have no equivalent of the "int" versus "long int"
-  distinction of C at the moment and assume "long ints" everywhere.)
-
-**r_uint**
-
-  In a few cases (e.g. hash table manipulation), we need machine-sized unsigned
-  arithmetic.  For these cases there is the r_uint class, which is a pure
-  Python implementation of word-sized unsigned integers that silently wrap
-  around.  ("word-sized" and "machine-sized" are used equivalently and mean
-  the native size, which you get using "unsigned long" in C.)
-  The purpose of this class (as opposed to helper functions as above)
-  is consistent typing: both Python and the annotator will propagate r_uint
-  instances in the program and interpret all the operations between them as
-  unsigned.  Instances of r_uint are special-cased by the code generators to
-  use the appropriate low-level type and operations.
-  Mixing of (signed) integers and r_uint in operations produces r_uint that
-  means unsigned results.  To convert back from r_uint to signed integers, use
-  intmask().
-
-
-Exception rules
----------------------
-
-Exceptions are by default not generated for simple cases.::
-
-    #!/usr/bin/python
-
-        lst = [1,2,3,4,5]
-        item = lst[i]    # this code is not checked for out-of-bound access
-
-        try:
-            item = lst[i]
-        except IndexError:
-            # complain
-
-Code with no exception handlers does not raise exceptions (after it has been
-translated, that is.  When you run it on top of CPython, it may raise
-exceptions, of course). By supplying an exception handler, you ask for error
-checking. Without, you assure the system that the operation cannot fail.
-This rule does not apply to *function calls*: any called function is
-assumed to be allowed to raise any exception.
-
-For example::
-
-    x = 5.1
-    x = x + 1.2       # not checked for float overflow
-    try:
-        x = x + 1.2
-    except OverflowError:
-        # float result too big
-
-But::
-
-    z = some_function(x, y)    # can raise any exception
-    try:
-        z = some_other_function(x, y)
-    except IndexError:
-        # only catches explicitly-raised IndexErrors in some_other_function()
-        # other exceptions can be raised, too, and will not be caught here.
-
-The ovfcheck() function described above follows the same rule: in case of
-overflow, it explicitly raise OverflowError, which can be caught anywhere.
-
-Exceptions explicitly raised or re-raised will always be generated.
-
-PyPy is debuggable on top of CPython
-------------------------------------
-
-PyPy has the advantage that it is runnable on standard
-CPython.  That means, we can run all of PyPy with all exception
-handling enabled, so we might catch cases where we failed to
-adhere to our implicit assertions.
-
-.. _`wrapping rules`:
-.. _`wrapped`:
-
-
+.. _wrapped:
 
 Wrapping rules
-==============
+--------------
 
 Wrapping
---------- 
+~~~~~~~~
 
 PyPy is made of Python source code at two levels: there is on the one hand
 *application-level code* that looks like normal Python code, and that
@@ -500,7 +202,7 @@
 structure.
 
 For example, an application-level Python ``list``
-is implemented by the `standard object space`_ as an
+is implemented by the :ref:`standard object space <standard-object-space>` as an
 instance of ``W_ListObject``, which has an instance attribute
 ``wrappeditems`` (an interpreter-level list which contains the
 application-level list's items as wrapped objects).
@@ -509,7 +211,7 @@
 
 
 Naming conventions
-------------------
+~~~~~~~~~~~~~~~~~~
 
 * ``space``: the object space is only visible at
   interpreter-level code, where it is by convention passed around by the name
@@ -529,14 +231,14 @@
 
 
 Operations on ``w_xxx``
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 The core bytecode interpreter considers wrapped objects as black boxes.
 It is not allowed to inspect them directly.  The allowed
 operations are all implemented on the object space: they are
 called ``space.xxx()``, where ``xxx`` is a standard operation
 name (``add``, ``getattr``, ``call``, ``eq``...). They are documented in the
-`object space document`_.
+:ref:`object space document <objspace-interface>`.
 
 A short warning: **don't do** ``w_x == w_y`` or ``w_x is w_y``!
 rationale for this rule is that there is no reason that two
@@ -548,12 +250,11 @@
 use ``space.is_true(space.is_(w_x, w_y))`` or better
 ``space.is_w(w_x, w_y)``.
 
-.. _`object space document`: objspace.html#interface
 
-.. _`applevel-exceptions`: 
+.. _applevel-exceptions:
 
 Application-level exceptions
-----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Interpreter-level code can use exceptions freely.  However,
 all application-level exceptions are represented as an
@@ -585,10 +286,10 @@
 instances of subclasses.
 
 
-.. _`modules`:
+.. _modules:
 
 Modules in PyPy
-===============
+---------------
 
 Modules visible from application programs are imported from
 interpreter or application level files.  PyPy reuses almost all python
@@ -597,18 +298,19 @@
 because they rely on implementation details of CPython.
 
 If we don't just modify an original CPython module but need to rewrite
-it from scratch we put it into `lib_pypy/`_ as a pure application level
+it from scratch we put it into :source:`lib_pypy/` as a pure application level
 module.
 
 When we need access to interpreter-level objects we put the module into
-`pypy/module`_.  Such modules use a `mixed module mechanism`_
+:source:`pypy/module`.  Such modules use a `mixed module mechanism`_
 which makes it convenient to use both interpreter- and application-level parts
 for the implementation.  Note that there is no extra facility for
 pure-interpreter level modules, you just write a mixed module and leave the
 application-level part empty.
 
+
 Determining the location of a module implementation
----------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can interactively find out where a module comes from, when running py.py.
 here are examples for the possible locations::
@@ -626,8 +328,9 @@
     '/home/hpk/pypy-dist/lib-python/2.7/os.py'
     >>>>
 
+
 Module directories / Import order
----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Here is the order in which PyPy looks up Python modules:
 
@@ -650,10 +353,11 @@
 
     The modified CPython library.
 
-.. _`modify modules`:
+
+.. _modify modules:
 
 Modifying a CPython library module or regression test
--------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Although PyPy is very compatible with CPython we sometimes need
 to change modules contained in our copy of the standard library,
@@ -665,34 +369,31 @@
 to see what is changed we have a branch called `vendor/stdlib`
 wich contains the unmodified cpython stdlib
 
-.. _`mixed module mechanism`:
-.. _`mixed modules`:
+
+.. _mixed module mechanism:
+.. _mixed-modules:
 
 Implementing a mixed interpreter/application level Module
----------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If a module needs to access PyPy's interpreter level
 then it is implemented as a mixed module.
 
-Mixed modules are directories in `pypy/module`_ with an  `__init__.py`
+Mixed modules are directories in :source:`pypy/module` with an  `__init__.py`
 file containing specifications where each name in a module comes from.
 Only specified names will be exported to a Mixed Module's applevel
 namespace.
 
-Sometimes it is necessary to really write some functions in C (or
-whatever target language). See `rffi`_ and `external functions
-documentation`_ for details. The latter approach is cumbersome and
-being phased out and former has currently quite a few rough edges.
+Sometimes it is necessary to really write some functions in C (or whatever
+target language). See :ref:`rffi <rpython:rffi>` details.
 
-.. _`rffi`: rffi.html
-.. _`external functions documentation`: translation.html#extfunccalls
 
 application level definitions
-.............................
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Application level specifications are found in the `appleveldefs`
 dictionary found in ``__init__.py`` files of directories in ``pypy/module``.
-For example, in `pypy/module/__builtin__/__init__.py`_ you find the following
+For example, in :source:`pypy/module/__builtin__/__init__.py` you find the following
 entry specifying where ``__builtin__.locals`` comes from::
 
      ...
@@ -703,12 +404,13 @@
 interpreted at application level and the wrapped function value for ``locals``
 will be extracted accordingly.
 
+
 interpreter level definitions
-.............................
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Interpreter level specifications are found in the ``interpleveldefs``
 dictionary found in ``__init__.py`` files of directories in ``pypy/module``.
-For example, in `pypy/module/__builtin__/__init__.py`_ the following
+For example, in :source:`pypy/module/__builtin__/__init__.py` the following
 entry specifies where ``__builtin__.len`` comes from::
 
      ...
@@ -724,7 +426,7 @@
         return space.len(w_obj)
 
 Exposed interpreter level functions usually take a ``space`` argument
-and some wrapped values (see `wrapping rules`_) .
+and some wrapped values (see `Wrapping rules`_) .
 
 You can also use a convenient shortcut in ``interpleveldefs`` dictionaries:
 namely an expression in parentheses to specify an interpreter level
@@ -743,37 +445,40 @@
 --withoutmod-mymodule (the latter being the default)) for py.py and
 translate.py.
 
+
 Testing modules in ``lib_pypy/``
---------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can go to the `pypy/module/test_lib_pypy/`_ directory and invoke the testing tool
+You can go to the :source:`pypy/module/test_lib_pypy/` directory and invoke the testing tool
 ("py.test" or "python ../../pypy/test_all.py") to run tests against the
-lib_pypy hierarchy.  Note, that tests in `pypy/module/test_lib_pypy/`_ are allowed
+lib_pypy hierarchy.  Note, that tests in :source:`pypy/module/test_lib_pypy/` are allowed
 and encouraged to let their tests run at interpreter level although
-`lib_pypy/`_ modules eventually live at PyPy's application level.
+:source:`lib_pypy/` modules eventually live at PyPy's application level.
 This allows us to quickly test our python-coded reimplementations
 against CPython.
 
+
 Testing modules in ``pypy/module``
-----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Simply change to ``pypy/module`` or to a subdirectory and `run the
 tests as usual`_.
 
 
 Testing modules in ``lib-python``
------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In order to let CPython's regression tests run against PyPy
-you can switch to the `lib-python/`_ directory and run
+you can switch to the :source:`lib-python/` directory and run
 the testing tool in order to start compliance tests.
 (XXX check windows compatibility for producing test reports).
 
+
 Naming conventions and directory layout
-===========================================
+---------------------------------------
 
 Directory and File Naming
--------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - directories/modules/namespaces are always **lowercase**
 
@@ -786,8 +491,9 @@
 
 - keep filenames concise and completion-friendly.
 
+
 Naming of python objects
-------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~
 
 - class names are **CamelCase**
 
@@ -803,8 +509,9 @@
   includes w_self.  Don't use ``w_`` in application level
   python only code.
 
+
 Committing & Branching to the repository
------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - write good log messages because several people
   are reading the diffs.
@@ -815,22 +522,23 @@
   ``try1`` doesn't already exists) you should do::
 
     hg branch try1
-    
+
   The branch will be recorded in the repository only after a commit. To switch
   back to the default branch::
-  
+
     hg update default
-    
+
   For further details use the help or refer to the `official wiki`_::
-  
+
     hg help branch
 
-.. _`official wiki`: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: http://mercurial.selenic.com/wiki/Branch
 
-.. _`using development tracker`:
+
+.. _using-development-tracker:
 
 Using the development bug/feature tracker
-=========================================
+-----------------------------------------
 
 We have a `development tracker`_, based on Richard Jones'
 `roundup`_ application.  You can file bugs,
@@ -838,15 +546,14 @@
 for the next milestone, both from an E-Mail and from a
 web interface.
 
-.. _`development tracker`: https://bugs.pypy.org/
-.. _`roundup`: http://roundup.sourceforge.net/
+.. _development tracker: https://bugs.pypy.org/
+.. _roundup: http://roundup.sourceforge.net/
 
 
-.. _`testing in PyPy`:
-.. _`test-design`: 
+.. _testing:
 
 Testing in PyPy
-===============
+---------------
 
 Our tests are based on the `py.test`_ tool which lets you write
 unittests without boilerplate.  All tests of modules
@@ -859,12 +566,11 @@
 - **Application Level tests**. They run at application level which means
   that they look like straight python code but they are interpreted by PyPy.
 
-.. _`standard object space`: objspace.html#standard-object-space
-.. _`objectspace`: objspace.html
-.. _`py.test`: http://pytest.org/
+.. _py.test: http://pytest.org/
+
 
 Interpreter level tests
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 You can write test functions and methods like this::
 
@@ -880,8 +586,9 @@
 module global level and use plain 'assert' statements thanks to the
 usage of the `py.test`_ tool.
 
+
 Application Level tests
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 For testing the conformance and well-behavedness of PyPy it
 is often sufficient to write "normal" application-level
@@ -916,7 +623,8 @@
             assert self.d["a"] == 1
             assert self.d["b"] == 2
 
-.. _`run the tests as usual`:
+
+.. _run the tests as usual:
 
 Another possibility is to use cls.space.appexec, for example::
 
@@ -959,7 +667,7 @@
 
 
 Command line tool test_all
---------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can run almost all of PyPy's tests by invoking::
 
@@ -969,8 +677,9 @@
 located in the ``py/bin/`` directory.  For switches to
 modify test execution pass the ``-h`` option.
 
+
 Coverage reports
-----------------
+~~~~~~~~~~~~~~~~
 
 In order to get coverage reports the `pytest-cov`_ plugin is included.
 it adds some extra requirements ( coverage_ and `cov-core`_ )
@@ -978,12 +687,13 @@
 
   python test_all.py --cov file_or_direcory_to_cover file_or_directory
 
-.. _`pytest-cov`: http://pypi.python.org/pypi/pytest-cov
-.. _`coverage`: http://pypi.python.org/pypi/coverage
-.. _`cov-core`: http://pypi.python.org/pypi/cov-core
+.. _pytest-cov: http://pypi.python.org/pypi/pytest-cov
+.. _coverage: http://pypi.python.org/pypi/coverage
+.. _cov-core: http://pypi.python.org/pypi/cov-core
+
 
 Test conventions
-----------------
+~~~~~~~~~~~~~~~~
 
 - adding features requires adding appropriate tests.  (It often even
   makes sense to first write the tests so that you are sure that they
@@ -993,29 +703,28 @@
   which contain unit tests.  Such scripts can usually be executed
   directly or are collectively run by pypy/test_all.py
 
-.. _`change documentation and website`:
+
+.. _change documentation and website:
 
 Changing documentation and website
-==================================
+----------------------------------
 
 documentation/website files in your local checkout
----------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Most of the PyPy's documentation is kept in `pypy/doc`.
 You can simply edit or add '.rst' files which contain ReST-markuped
 files.  Here is a `ReST quickstart`_ but you can also just look
 at the existing documentation and see how things work.
 
-.. _`ReST quickstart`: http://docutils.sourceforge.net/docs/user/rst/quickref.html
-
 Note that the web site of http://pypy.org/ is maintained separately.
 For now it is in the repository https://bitbucket.org/pypy/pypy.org
 
+.. _ReST quickstart: http://docutils.sourceforge.net/docs/user/rst/quickref.html
+
+
 Automatically test documentation/website changes
-------------------------------------------------
-
-.. _`sphinx home page`:
-.. _`sphinx`: http://sphinx.pocoo.org/
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 We automatically check referential integrity and ReST-conformance.  In order to
 run the tests you need sphinx_ installed.  Then go to the local checkout
@@ -1035,5 +744,4 @@
 
 which will check that remote URLs are reachable.
 
-
-.. include:: _ref.txt
+.. _sphinx: http://sphinx.pocoo.org/
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -40,9 +40,9 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx',
-              'sphinx.ext.todo', 'sphinx.ext.ifconfig', 'sphinx.ext.graphviz',
-              'pypyconfig']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.extlinks',
+              'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.ifconfig',
+              'sphinx.ext.graphviz', 'pypyconfig']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -215,7 +215,8 @@
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None),
+                       'rpython': ('http://rpython.readthedocs.org/en/latest/', None)}
 
 # -- Options for manpage output-------------------------------------------------
 
@@ -225,3 +226,4 @@
    u'The PyPy Project', 1)
 ]
 
+extlinks = {'source': ('https://bitbucket.org/pypy/pypy/src/default/%s', '')}
diff --git a/pypy/doc/config/confrest.py b/pypy/doc/config/confrest.py
deleted file mode 100644
--- a/pypy/doc/config/confrest.py
+++ /dev/null
@@ -1,62 +0,0 @@
-from pypy.doc.confrest import *
-from pypy.config.makerestdoc import make_cmdline_overview
-from pypy.config import pypyoption
-from rpython.config.config import Config
-from rpython.config import translationoption
-
-
-all_optiondescrs = [pypyoption.pypy_optiondescription,
-                    translationoption.translation_optiondescription,
-                   ]
-start_to_descr = dict([(descr._name, descr) for descr in all_optiondescrs])
-
-class PyPyPage(PyPyPage): 
-    def fill(self):
-        super(PyPyPage, self).fill()
-        self.menubar[:] = html.div(
-            html.a("general documentation", href="../index.html",
-                   class_="menu"), " ",
-            html.a("config index", href="index.html",
-                   class_="menu"), " ",
-            html.a("command-line overview", href="commandline.html",
-                   class_="menu"), " ",
-            " ", id="menubar")
-
-class Project(Project): 
-    stylesheet = "../style.css"
-    title = "PyPy Configuration"
-    prefix_title = "PyPy Configuration"
-    Page = PyPyPage 
-
-    def get_content(self, txtpath, encoding):
-        if txtpath.basename == "commandline.rst":
-            result = []
-            for line in txtpath.read().splitlines():
-                if line.startswith('.. GENERATE:'):
-                    start = line[len('.. GENERATE:'):].strip()
-                    descr = start_to_descr[start]
-                    line = make_cmdline_overview(descr, title=False).text()
-                result.append(line)
-            return "\n".join(result)
-        fullpath = txtpath.purebasename
-        start = fullpath.split(".")[0]

From noreply at buildbot.pypy.org  Wed Dec 10 16:36:21 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Dec 2014 16:36:21 +0100 (CET)
Subject: [pypy-commit] pypy default: (cfbolz, tanzim, arigo)
Message-ID: <20141210153621.74B3D1D236C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74878:e56dfa6f9309
Date: 2014-12-10 15:32 +0000
http://bitbucket.org/pypy/pypy/changeset/e56dfa6f9309/

Log:	(cfbolz, tanzim, arigo)

	Essential optimization (<=cfbolz).

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -17,9 +17,13 @@
         self.w_value = w_value
 
 
+def needs_to_unwrap_cell(space, w_value):
+    return (space.config.objspace.std.withtypeversion and
+            isinstance(w_value, TypeCell))
+
+
 def unwrap_cell(space, w_value):
-    if (space.config.objspace.std.withtypeversion and
-            isinstance(w_value, TypeCell)):
+    if needs_to_unwrap_cell(space, w_value):
         return w_value.w_value
     return w_value
 
@@ -366,8 +370,11 @@
             tup = w_self._lookup_where(name)
             return tup
         name = promote_string(name)
-        w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
-        return w_class, unwrap_cell(space, w_value)
+        tup_w = w_self._pure_lookup_where_with_method_cache(name, version_tag)
+        w_class, w_value = tup_w
+        if needs_to_unwrap_cell(space, w_value):
+            return w_class, w_value.value
+        return tup_w   # don't make a new tuple, reuse the old one
 
     def _pure_lookup_where_possibly_with_method_cache(w_self, name, version_tag):
         if w_self.space.config.objspace.std.withmethodcache:

From noreply at buildbot.pypy.org  Wed Dec 10 16:47:38 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 10 Dec 2014 16:47:38 +0100 (CET)
Subject: [pypy-commit] pypy object-dtype: small progress
Message-ID: <20141210154738.D16DF1D22F1@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: object-dtype
Changeset: r74879:8471e5574836
Date: 2014-12-10 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/8471e5574836/

Log:	small progress

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -601,6 +601,10 @@
         #    arr.storage[i] = arg[i]
         return W_UnicodeBox(arr, 0, arr.dtype)
 
+class W_ObjectBox(W_GenericBox):
+    def __init__(self, w_obj):
+        self.w_obj = w_obj
+
 
 W_GenericBox.typedef = TypeDef("numpy.generic",
     __new__ = interp2app(W_GenericBox.descr__new__.im_func),
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -850,6 +850,13 @@
             char=NPY.UINTPLTR,
             w_box_type=space.gettypefor(boxes.W_ULongBox),
         )
+        self.w_objectdtype = W_Dtype(
+            types.ObjectType(),
+            num=NPY.OBJECT,
+            kind=NPY.OBJECTLTR,
+            char=NPY.OBJECTLTR,
+            w_box_type=space.gettypefor(boxes.W_ObjectBox),
+        )
         aliases = {
             NPY.BOOL:        ['bool_', 'bool8'],
             NPY.BYTE:        ['byte'],
@@ -868,6 +875,7 @@
             NPY.CLONGDOUBLE: ['clongdouble', 'clongfloat'],
             NPY.STRING:      ['string_', 'str'],
             NPY.UNICODE:     ['unicode_'],
+            NPY.OBJECT:      ['object_'],
         }
         self.alternate_constructors = {
             NPY.BOOL:     [space.w_bool],
@@ -886,6 +894,8 @@
             NPY.UNICODE:  [space.w_unicode],
             NPY.VOID:     [space.gettypefor(boxes.W_GenericBox)],
                            #space.w_buffer,  # XXX no buffer in space
+            NPY.OBJECT:   [space.gettypefor(boxes.W_ObjectBox),
+                           space.w_object],
         }
         float_dtypes = [self.w_float16dtype, self.w_float32dtype,
                         self.w_float64dtype, self.w_floatlongdtype]
@@ -905,7 +915,7 @@
             self.w_int64dtype, self.w_uint64dtype,
             ] + float_dtypes + complex_dtypes + [
             self.w_stringdtype, self.w_unicodedtype, self.w_voiddtype,
-            self.w_intpdtype, self.w_uintpdtype,
+            self.w_intpdtype, self.w_uintpdtype, self.w_objectdtype,
         ]
         self.float_dtypes_by_num_bytes = sorted(
             (dtype.elsize, dtype)
@@ -957,6 +967,7 @@
             'USHORT': self.w_uint16dtype,
             'FLOAT': self.w_float32dtype,
             'BOOL': self.w_booldtype,
+            'OBJECT': self.w_objectdtype,
         }
 
         typeinfo_partial = {
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1614,6 +1614,20 @@
         BoxType = boxes.W_ComplexLongBox
         ComponentBoxType = boxes.W_FloatLongBox
 
+class ObjectType(BaseType):
+    T = lltype.Signed
+    
+    def get_element_size(self):
+        return rffi.sizeof(lltype.Signed)
+
+    def coerce(self, space, dtype, w_item):
+        if isinstance(w_item, boxes.W_ObjectBox):
+            return w_item
+        return boxes.W_ObjectBox(w_item)
+
+    def store(self, arr, i, offset, box):
+        self._write(arr.storage, i, offset, self.unbox(box))
+
 class FlexibleType(BaseType):
     def get_element_size(self):
         return rffi.sizeof(self.T)

From noreply at buildbot.pypy.org  Wed Dec 10 16:47:49 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 10 Dec 2014 16:47:49 +0100 (CET)
Subject: [pypy-commit] pypy object-dtype: merge default
Message-ID: <20141210154749.62D641D22F1@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: object-dtype
Changeset: r74880:207037db3449
Date: 2014-12-10 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/207037db3449/

Log:	merge default

diff too long, truncating to 2000 out of 58563 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -89,6 +89,7 @@
 ^pypy/doc/image/lattice3\.png$
 ^pypy/doc/image/stackless_informal\.png$
 ^pypy/doc/image/parsing_example.+\.png$
+^rpython/doc/_build/.*$
 ^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
 ^compiled
 ^.git/
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -23,6 +23,7 @@
 
     the pypy-dev team <pypy-dev at python.org>
 
+
 Building
 ========
 
@@ -36,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html
diff --git a/ctypes_configure/doc/configure.txt b/ctypes_configure/doc/configure.txt
--- a/ctypes_configure/doc/configure.txt
+++ b/ctypes_configure/doc/configure.txt
@@ -19,6 +19,4 @@
 usage
 =====
 
-`sample.py`_ explains in details how to use it.
-
-.. _`sample.py`: http://codespeak.net/svn/pypy/dist/ctypes_configure/doc/sample.py
+:source:`sample.py <ctypes_configure/doc/sample.py>` explains in details how to use it.
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -59,7 +59,7 @@
     def __init__(self, basename, core=False, compiler=None, usemodules='',
                  skip=None):
         self.basename = basename
-        self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket']
+        self._usemodules = usemodules.split() + ['signal', 'time', 'itertools', '_socket']
         self._compiler = compiler
         self.core = core
         self.skip = skip
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -66,11 +66,12 @@
 
 @builtinify
 def getgrnam(name):
-    if not isinstance(name, str):
+    if not isinstance(name, basestring):
         raise TypeError("expected string")
+    name = str(name)
     res = libc.getgrnam(name)
     if not res:
-        raise KeyError(name)
+        raise KeyError("'getgrnam(): name not found: %s'" % name)
     return _group_from_gstruct(res)
 
 @builtinify
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -29,7 +29,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -40,7 +40,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
     "cStringIO", "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -64,19 +64,15 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
@@ -86,9 +82,10 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")],
 }
+if sys.platform == "win32":
+    module_suggests["cpyext"].append(("translation.shared", True))
 
 module_import_dependencies = {
     # no _rawffi if importing rpython.rlib.clibffi raises ImportError
@@ -255,10 +252,6 @@
         BoolOption("optimized_list_getitem",
                    "special case the 'list[integer]' expressions",
                    default=False),
-        BoolOption("builtinshortcut",
-                   "a shortcut for operations between built-in types. XXX: "
-                   "deprecated, not really a shortcut any more.",
-                   default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
                    default=False,
@@ -270,9 +263,6 @@
                    # weakrefs needed, because of get_subclasses()
                    requires=[("translation.rweakref", True)]),
 
-        ChoiceOption("multimethods", "the multimethod implementation to use",
-                     ["doubledispatch", "mrd"],
-                     default="mrd"),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
                    default=False,
diff --git a/pypy/config/test/test_pypyoption.py b/pypy/config/test/test_pypyoption.py
--- a/pypy/config/test/test_pypyoption.py
+++ b/pypy/config/test/test_pypyoption.py
@@ -64,7 +64,7 @@
     def check_file_exists(fn):
         assert configdocdir.join(fn).check()
 
-    from pypy.doc.config.confrest import all_optiondescrs
+    from pypy.doc.config.generate import all_optiondescrs
     configdocdir = thisdir.dirpath().dirpath().join("doc", "config")
     for descr in all_optiondescrs:
         prefix = descr._name
diff --git a/pypy/doc/TODO b/pypy/doc/TODO
new file mode 100644
--- /dev/null
+++ b/pypy/doc/TODO
@@ -0,0 +1,40 @@
+Documentation TODO
+==================
+
+General
+-------
+
+* architecture documents don't really show the separation between PyPy and
+  RPython
+  * architecture.rst is duplicate (both pypy and rpython)
+* Consider moving information from pypy/doc/{build,windows}.rst to rpython/doc
+
+
+Cleanup
+~~~~~~~
+
+* remove documentation on removed features
+  * various object spaces
+* update / remove dead links
+
+
+Meta
+~~~~
+
+* work on configuration/options documentation generation
+
+
+PyPy
+----
+
+* Update coding guide
+* Move links from project-documentation.rst to index.rst and consider killing
+  it.
+
+
+RPython
+-------
+
+* make translation.rst a high-level overview and move details in their own
+  documents
+* redo various outdated pictures in translation.rst
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -1,20 +1,17 @@
-
 .. comment: this document is very incomplete, should we generate
             it automatically?
 
-=======================
 The ``__pypy__`` module
 =======================
 
 The ``__pypy__`` module is the main entry point to special features provided
-by PyPy's standard interpreter. Its content depends on `configuration options`_ 
-which may add new functionality and functions whose existence or non-existence 
-indicates the presence of such features. 
+by PyPy's standard interpreter. Its content depends on :doc:`configuration options <config/index>`
+which may add new functionality and functions whose existence or non-existence
+indicates the presence of such features.
 
-.. _`configuration options`: config/index.html
 
 Generally available functionality
-=================================
+---------------------------------
 
  - ``internal_repr(obj)``: return the interpreter-level representation of an
    object.
@@ -22,28 +19,22 @@
    It works like a simplified array of characters (actually, depending on the
    configuration the ``array`` module internally uses this).
 
+
 Transparent Proxy Functionality
-===============================
+-------------------------------
 
-If `transparent proxies`_ are enabled (with :config:`objspace.std.withtproxy`)
+If :ref:`transparent proxies <tproxy>` are enabled (with :config:`objspace.std.withtproxy`)
 the following functions are put into ``__pypy__``:
 
  - ``tproxy(typ, controller)``: Return something that looks like it is of type
    typ. Its behaviour is completely controlled by the controller. See the docs
-   about `transparent proxies`_ for detail.
-
+   about :ref:`transparent proxies <tproxy>` for detail.
  - ``get_tproxy_controller(obj)``: If obj is really a transparent proxy, return
    its controller. Otherwise return None.
 
-.. _`transparent proxies`: objspace-proxies.html#tproxy
-
 
 Functionality available on py.py (not after translation)
-========================================================
+--------------------------------------------------------
 
  - ``isfake(obj)``: returns True if ``obj`` is faked.
-
  - ``interp_pdb()``: start a pdb at interpreter-level.
-
-
-
diff --git a/pypy/doc/_ref.txt b/pypy/doc/_ref.txt
deleted file mode 100644
--- a/pypy/doc/_ref.txt
+++ /dev/null
@@ -1,114 +0,0 @@
-.. This file is generated automatically by makeref.py script,
-   which in turn is run manually.
-
-.. _`ctypes_configure/doc/sample.py`: https://bitbucket.org/pypy/pypy/src/default/ctypes_configure/doc/sample.py
-.. _`dotviewer/`: https://bitbucket.org/pypy/pypy/src/default/dotviewer/
-.. _`lib-python/`: https://bitbucket.org/pypy/pypy/src/default/lib-python/
-.. _`lib-python/2.7/dis.py`: https://bitbucket.org/pypy/pypy/src/default/lib-python/2.7/dis.py
-.. _`lib_pypy/`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/
-.. _`lib_pypy/greenlet.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/greenlet.py
-.. _`lib_pypy/tputil.py`: https://bitbucket.org/pypy/pypy/src/default/lib_pypy/tputil.py
-.. _`pypy/bin/`: https://bitbucket.org/pypy/pypy/src/default/pypy/bin/
-.. _`pypy/bin/pyinteractive.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/bin/pyinteractive.py
-.. _`pypy/config/`: https://bitbucket.org/pypy/pypy/src/default/pypy/config/
-.. _`pypy/config/pypyoption.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/config/pypyoption.py
-.. _`pypy/doc/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/
-.. _`pypy/doc/config/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/config/
-.. _`pypy/doc/discussion/`: https://bitbucket.org/pypy/pypy/src/default/pypy/doc/discussion/
-.. _`pypy/goal/`: https://bitbucket.org/pypy/pypy/src/default/pypy/goal/
-.. _`pypy/interpreter`:
-.. _`pypy/interpreter/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/
-.. _`pypy/interpreter/argument.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/argument.py
-.. _`pypy/interpreter/astcompiler`:
-.. _`pypy/interpreter/astcompiler/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/
-.. _`pypy/interpreter/astcompiler/assemble.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/assemble.py
-.. _`pypy/interpreter/astcompiler/ast.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/ast.py
-.. _`pypy/interpreter/astcompiler/astbuilder.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/astbuilder.py
-.. _`pypy/interpreter/astcompiler/asthelpers.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/asthelpers.py
-.. _`pypy/interpreter/astcompiler/codegen.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/codegen.py
-.. _`pypy/interpreter/astcompiler/optimize.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/optimize.py
-.. _`pypy/interpreter/astcompiler/symtable.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/symtable.py
-.. _`pypy/interpreter/astcompiler/tools/Python.asdl`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/tools/Python.asdl
-.. _`pypy/interpreter/astcompiler/tools/asdl_py.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/astcompiler/tools/asdl_py.py
-.. _`pypy/interpreter/baseobjspace.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/baseobjspace.py
-.. _`pypy/interpreter/eval.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/eval.py
-.. _`pypy/interpreter/executioncontext.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/executioncontext.py
-.. _`pypy/interpreter/function.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/function.py
-.. _`pypy/interpreter/gateway.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/gateway.py
-.. _`pypy/interpreter/mixedmodule.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/mixedmodule.py
-.. _`pypy/interpreter/module.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/module.py
-.. _`pypy/interpreter/pyframe.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyframe.py
-.. _`pypy/interpreter/pyopcode.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyopcode.py
-.. _`pypy/interpreter/pyparser`:
-.. _`pypy/interpreter/pyparser/`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/
-.. _`pypy/interpreter/pyparser/future.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/future.py
-.. _`pypy/interpreter/pyparser/metaparser.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/metaparser.py
-.. _`pypy/interpreter/pyparser/parser.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/parser.py
-.. _`pypy/interpreter/pyparser/pyparse.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/pyparse.py
-.. _`pypy/interpreter/pyparser/pytokenizer.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/pyparser/pytokenizer.py
-.. _`pypy/interpreter/typedef.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/interpreter/typedef.py
-.. _`pypy/module`:
-.. _`pypy/module/`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/
-.. _`pypy/module/__builtin__/__init__.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/__builtin__/__init__.py
-.. _`pypy/module/cppyy/capi/__init__.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/capi/__init__.py
-.. _`pypy/module/cppyy/capi/builtin_capi.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/capi/builtin_capi.py
-.. _`pypy/module/cppyy/include/capi.h`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/cppyy/include/capi.h
-.. _`pypy/module/test_lib_pypy/`: https://bitbucket.org/pypy/pypy/src/default/pypy/module/test_lib_pypy/
-.. _`pypy/objspace/`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/
-.. _`pypy/objspace/std`:
-.. _`pypy/objspace/std/`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/
-.. _`pypy/objspace/std/bytesobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/bytesobject.py
-.. _`pypy/objspace/std/multimethod.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/multimethod.py
-.. _`pypy/objspace/std/objspace.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/objspace.py
-.. _`pypy/objspace/std/proxy_helpers.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/proxy_helpers.py
-.. _`pypy/objspace/std/proxyobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/proxyobject.py
-.. _`pypy/objspace/std/strbufobject.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/strbufobject.py
-.. _`pypy/objspace/std/transparent.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/transparent.py
-.. _`pypy/tool/`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/
-.. _`pypy/tool/pytest/`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/pytest/
-.. _`rpython/annotator`:
-.. _`rpython/annotator/`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/
-.. _`rpython/annotator/annrpython.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/annrpython.py
-.. _`rpython/annotator/binaryop.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/binaryop.py
-.. _`rpython/annotator/builtin.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/annotator/builtin.py
-.. _`rpython/bin/translatorshell.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/bin/translatorshell.py
-.. _`rpython/config/`: https://bitbucket.org/pypy/pypy/src/default/rpython/config/
-.. _`rpython/config/translationoption.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/config/translationoption.py
-.. _`rpython/flowspace/`: https://bitbucket.org/pypy/pypy/src/default/rpython/flowspace/
-.. _`rpython/flowspace/model.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/flowspace/model.py
-.. _`rpython/memory/`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/
-.. _`rpython/memory/gc/generation.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/generation.py
-.. _`rpython/memory/gc/hybrid.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/hybrid.py
-.. _`rpython/memory/gc/minimarkpage.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/minimarkpage.py
-.. _`rpython/memory/gc/semispace.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/memory/gc/semispace.py
-.. _`rpython/rlib`:
-.. _`rpython/rlib/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/
-.. _`rpython/rlib/listsort.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/listsort.py
-.. _`rpython/rlib/nonconst.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/nonconst.py
-.. _`rpython/rlib/objectmodel.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/objectmodel.py
-.. _`rpython/rlib/parsing/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/parsing/
-.. _`rpython/rlib/parsing/tree.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/parsing/tree.py
-.. _`rpython/rlib/rarithmetic.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rarithmetic.py
-.. _`rpython/rlib/rbigint.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rbigint.py
-.. _`rpython/rlib/rrandom.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rrandom.py
-.. _`rpython/rlib/rsocket.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/rsocket.py
-.. _`rpython/rlib/streamio.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/streamio.py
-.. _`rpython/rlib/test`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/test/
-.. _`rpython/rlib/unroll.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rlib/unroll.py
-.. _`rpython/rtyper`:
-.. _`rpython/rtyper/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/
-.. _`rpython/rtyper/lltypesystem/`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/lltypesystem/
-.. _`rpython/rtyper/lltypesystem/lltype.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/lltypesystem/lltype.py
-.. _`rpython/rtyper/rint.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rint.py
-.. _`rpython/rtyper/rlist.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rlist.py
-.. _`rpython/rtyper/rmodel.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rmodel.py
-.. _`rpython/rtyper/rtyper.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/rtyper.py
-.. _`rpython/rtyper/test/test_llinterp.py`: https://bitbucket.org/pypy/pypy/src/default/rpython/rtyper/test/test_llinterp.py
-.. _`rpython/tool/algo/`: https://bitbucket.org/pypy/pypy/src/default/rpython/tool/algo/
-.. _`rpython/translator`:
-.. _`rpython/translator/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/
-.. _`rpython/translator/backendopt/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/backendopt/
-.. _`rpython/translator/c/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/
-.. _`rpython/translator/c/src/stacklet/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/src/stacklet/
-.. _`rpython/translator/c/src/stacklet/stacklet.h`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/c/src/stacklet/stacklet.h
-.. _`rpython/translator/tool/`: https://bitbucket.org/pypy/pypy/src/default/rpython/translator/tool/
diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -1,93 +1,29 @@
-==================================================
-Goals and Architecture Overview 
-==================================================
+Goals and Architecture Overview
+===============================
 
 .. contents::
 
+This document gives an overview of the goals and architecture of PyPy. If you're
+interested in :ref:`using PyPy <using-pypy>` or :ref:`hacking on it <developing-pypy>`,
+have a look at our :ref:`getting started <getting-started-index>` section.
 
-This document gives an overview of the goals and architecture of PyPy.
-See `getting started`_ for a practical introduction and starting points. 
 
-Mission statement 
-====================
+Mission statement
+-----------------
 
-We aim to provide:
+We aim to provide a compliant, flexible and fast implementation of the Python_
+Language which uses the RPython toolchain to enable new advanced high-level
+features without having to encode the low-level details.  We call this PyPy.
 
- * a common translation and support framework for producing
-   implementations of dynamic languages, emphasizing a clean
-   separation between language specification and implementation
-   aspects.  We call this the `RPython toolchain`_.
+.. _Python: http://docs.python.org/reference/
 
- * a compliant, flexible and fast implementation of the Python_ Language 
-   which uses the above toolchain to enable new advanced high-level features 
-   without having to encode the low-level details.  We call this PyPy.
-
-By separating concerns in this way, our implementation
-of Python - and other dynamic languages - is able to automatically
-generate a Just-in-Time compiler for any dynamic language.  It also
-allows a mix-and-match approach to implementation decisions, including
-many that have historically been outside of a user's control, such as
-target platform, memory and 
-threading models, garbage collection strategies, and optimizations applied, 
-including whether or not to have a JIT in the first place.
 
 High Level Goals
-=============================
-
-RPython - the Translation Toolchain
------------------------------------------------
-
-Traditionally, language interpreters are written in a target platform language
-such as C/Posix, Java or C#.  Each implementation provides 
-a fundamental mapping between application source code and the target 
-environment.  One of 
-the goals of the "all-encompassing" environments, such as the .NET framework
-and to some extent the Java virtual machine, is to provide standardized
-and higher level functionalities in order to support language implementers
-for writing language implementations. 
-
-PyPy is experimenting with a more ambitious approach.  We are using a
-subset of the high-level language Python, called RPython_, in which we
-write languages as simple interpreters with few references to and
-dependencies on lower level details.  The `RPython toolchain`_
-produces a concrete virtual machine for the platform of our choice by
-inserting appropriate lower level aspects.  The result can be customized
-by selecting other feature and platform configurations.
-
-Our goal is to provide a possible solution to the problem of language
-implementers: having to write ``l * o * p`` interpreters for ``l``
-dynamic languages and ``p`` platforms with ``o`` crucial design
-decisions.  PyPy aims at making it possible to change each of these
-variables independently such that:
-
-* ``l``: the language that we analyze can be evolved or entirely replaced;
-
-* ``o``: we can tweak and optimize the translation process to produce 
-  platform specific code based on different models and trade-offs;
-
-* ``p``: we can write new translator back-ends to target different
-  physical and virtual platforms.
-
-By contrast, a standardized target environment - say .NET -
-enforces ``p=1`` as far as it's concerned.  This helps making ``o`` a
-bit smaller by providing a higher-level base to build upon.  Still,
-we believe that enforcing the use of one common environment 
-is not necessary.  PyPy's goal is to give weight to this claim - at least 
-as far as language implementation is concerned - showing an approach
-to the ``l * o * p`` problem that does not rely on standardization.
-
-The most ambitious part of this goal is to `generate Just-In-Time
-Compilers`_ in a language-independent way, instead of only translating
-the source interpreter into an interpreter for the target platform.
-This is an area of language implementation that is commonly considered
-very challenging because of the involved complexity.
-
-
-PyPy - the Python Interpreter
---------------------------------------------
+----------------
 
 Our main motivation for developing the translation framework is to
-provide a full featured, customizable, fast_ and `very compliant`_ Python
+provide a full featured, customizable, :ref:`fast <how-fast-is-pypy>` and
+:doc:`very compliant <cpython_differences>` Python
 implementation, working on and interacting with a large variety of
 platforms and allowing the quick introduction of new advanced language
 features.
@@ -106,88 +42,22 @@
 and fully orthogonal to the interpreter source code.
 
 
-PyPy Architecture 
-===========================
-
-As you would expect from a project implemented using ideas from the world
-of `Extreme Programming`_, the architecture of PyPy has evolved over time
-and continues to evolve.  Nevertheless, the high level architecture is 
-stable. As described above, there are two rather independent basic
-subsystems: the `PyPy Python Interpreter`_ and the `RPython Translation Toolchain`_.
-
-.. _`translation framework`:
-
-RPython Translation Toolchain
--------------------------
-
-The job of the RPython toolchain is to translate RPython_ programs
-into an efficient version of that program for one of the various target
-platforms, generally one that is considerably lower-level than Python.
-
-The approach we have taken is to reduce the level of abstraction of the
-source RPython program in several steps, from the high level down to the
-level of the target platform, whatever that may be.  Currently we
-support two broad flavours of target platforms: the ones that assume a
-C-like memory model with structures and pointers, and the ones that
-assume an object-oriented model with classes, instances and methods (as,
-for example, the Java and .NET virtual machines do).
-
-The RPython toolchain never sees the RPython source code or syntax
-trees, but rather starts with the *code objects* that define the
-behaviour of the function objects one gives it as input.  It can be
-considered as "freezing" a pre-imported RPython program into an
-executable form suitable for the target platform.
-
-The steps of the translation process can be summarized as follows:
-
-* The code object of each source functions is converted to a `control
-  flow graph` by the `Flow Object Space`_.
-
-* The control flow graphs are processed by the Annotator_, which
-  performs whole-program type inference to annotate each variable of
-  the control flow graph with the types it may take at run-time.
-
-* The information provided by the annotator is used by the RTyper_ to
-  convert the high level operations of the control flow graphs into
-  operations closer to the abstraction level of the target platform.
-
-* Optionally, `various transformations`_ can then be applied which, for
-  example, perform optimizations such as inlining, add capabilities
-  such as stackless-style concurrency, or insert code for the
-  `garbage collector`_.
-
-* Then, the graphs are converted to source code for the target platform
-  and compiled into an executable.
-
-This process is described in much more detail in the `document about
-the RPython toolchain`_ and in the paper `Compiling dynamic language
-implementations`_.
-
-.. _`control flow graph`: translation.html#the-flow-model
-.. _`Flow Object Space`: objspace.html#the-flow-object-space
-.. _Annotator: translation.html#the-annotation-pass
-.. _RTyper: rtyper.html#overview
-.. _`various transformations`: translation.html#the-optional-transformations
-.. _`document about the RPython toolchain`: translation.html
-.. _`garbage collector`: garbage_collection.html
-.. _`RPython toolchain`: translation.html
-.. _`standard interpreter`: 
-.. _`python interpreter`: 
+.. _python-interpreter:
 
 PyPy Python Interpreter
--------------------------------------
+-----------------------
 
 PyPy's *Python Interpreter* is written in RPython and implements the
 full Python language.  This interpreter very closely emulates the
 behavior of CPython.  It contains the following key components:
 
-- a bytecode compiler responsible for producing Python code objects 
+- a bytecode compiler responsible for producing Python code objects
   from the source code of a user application;
 
-- a `bytecode evaluator`_ responsible for interpreting 
+- a :doc:`bytecode evaluator <interpreter>` responsible for interpreting
   Python code objects;
 
-- a `standard object space`_, responsible for creating and manipulating
+- a :ref:`standard object space <standard-object-space>`, responsible for creating and manipulating
   the Python objects seen by the application.
 
 The *bytecode compiler* is the preprocessing phase that produces a
@@ -200,64 +70,6 @@
 lists, as well as the operations between them, like addition or
 truth-value-testing.
 
-This division between bytecode evaluator and object space is very
-important, as it gives a lot of flexibility.  One can plug in 
-different `object spaces`_ to get different or enriched behaviours 
-of the Python objects.  Additionally, a special more abstract object
-space, the `flow object space`_, allows us to reuse the bytecode
-evaluator for our translation framework.
-
-.. _`bytecode evaluator`: interpreter.html
-.. _`standard object space`: objspace.html#the-standard-object-space
-.. _`object spaces`: objspace.html
-.. _`flow object space`: objspace.html#the-flow-object-space
-
-.. _`the translation framework`:
-
-
-Further reading
-===============
-
-All of PyPy's documentation can be reached from the `documentation
-index`_.  Of particular interest after reading this document might be:
-
- * `getting-started`_: a hands-on guide to getting involved with the
-   PyPy source code.
-
- * `PyPy's approach to virtual machine construction`_: a paper
-   presented to the Dynamic Languages Symposium attached to OOPSLA
-   2006.
-
- * `The translation document`_: a detailed description of our
-   translation process.
-
- * `JIT Generation in PyPy`_, describing how we produce a Just-in-time
-   Compiler from an interpreter.
-
- * A tutorial of how to use the `RPython toolchain`_ to `implement your own
-   interpreter`_.
-
-.. _`documentation index`: index.html#project-documentation
-.. _`getting-started`: getting-started-dev.html
-.. _`PyPy's approach to virtual machine construction`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dls2006/pypy-vm-construction.pdf
-.. _`the translation document`: translation.html
-.. _`RPython toolchain`: translation.html
-.. _`Compiling dynamic language implementations`: https://bitbucket.org/pypy/extradoc/raw/tip/eu-report/D05.1_Publish_on_translating_a_very-high-level_description.pdf
-.. _`Technical reports`: index-report.html
-
-.. _`getting started`: getting-started.html
-.. _`Extreme Programming`: http://www.extremeprogramming.org/
-
-.. _fast: faq.html#how-fast-is-pypy
-.. _`very compliant`: cpython_differences.html
-
-.. _`RPython`: coding-guide.html#rpython
-
-.. _Python: http://docs.python.org/reference/
-.. _Psyco: http://psyco.sourceforge.net
-.. _`generate Just-In-Time Compilers`: jit/index.html
-.. _`JIT Generation in PyPy`: jit/index.html
-.. _`implement your own interpreter`: http://morepypy.blogspot.com/2011/04/tutorial-writing-interpreter-with-pypy.html
-
-.. include:: _ref.txt
-
+This division between bytecode evaluator and object space gives a lot of
+flexibility.  One can plug in different :doc:`object spaces <objspace>` to get
+different or enriched behaviours of the Python objects.
diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst
deleted file mode 100644
--- a/pypy/doc/arm.rst
+++ /dev/null
@@ -1,166 +0,0 @@
-=========================
-Cross-translating for ARM
-=========================
-
-
-Here we describe the setup required and the steps needed to follow to translate
-an interpreter using the RPython translator to target ARM using a cross
-compilation toolchain.
-
-To translate an RPython program for ARM we can either
-translate directly on an ARM device following the normal translation steps. Unfortunately this is not really feasible on most ARM machines. The alternative is to cross-translate using a cross-compilation toolchain.
-
-To cross-translate we run the translation on a more powerful (usually
-x86) machine and generate a binary for ARM using a cross-compiler to compile
-the generated C code. There are several constraints when doing this. In
-particular we currently only support Linux as translation host and target
-platforms (tested on Ubuntu). Also we need a 32-bit environment to run the
-translation. This can be done either on a 32bit host or in 32bit chroot.
-
-
-Requirements
-------------
-
-The tools required to cross translate from a Linux based host to an ARM based Linux target are:
-
-- A checkout of PyPy (default branch).
-- The GCC ARM cross compiler (on Ubuntu it is the ``gcc-arm-linux-gnueabi package``) but other toolchains should also work.
-- Scratchbox 2, a cross-compilation engine (``scratchbox2`` Ubuntu package).
-- A 32-bit PyPy or Python.
-- And the following (or corresponding) packages need to be installed to create an ARM based chroot:
-
-  * ``debootstrap`` 
-  * ``schroot``
-  * ``binfmt-support``
-  * ``qemu-system``
-  * ``qemu-user-static``
-
-- The dependencies above are in addition to the ones needed for a regular
-  translation, `listed here`_.
-
-.. _`listed here`: getting-started-python.html#translating-the-pypy-python-interpreter
-
-
-Creating a Qemu based ARM chroot
---------------------------------
-
-First we will need to create a rootfs containing the packages and dependencies
-required in order to translate PyPy or other interpreters. We are going to
-assume, that the files will be placed in ``/srv/chroot/precise_arm``.
-
-Create the rootfs by calling:
-
-::
-
-   mkdir -p /srv/chroot/precise_arm
-   qemu-debootstrap --variant=buildd --arch=armel precise /srv/chroot/precise_arm/  http://ports.ubuntu.com/ubuntu-ports/
-
-Next, copy the qemu-arm-static binary to the rootfs.
-
-:: 
-
-  cp /usr/bin/qemu-arm-static /srv/chroot/precise_arm/usr/bin/qemu-arm-static
-
-For easier configuration and management we will create a schroot pointing to
-the rootfs. We need to add a configuration block (like the one below) to the
-schroot configuration file in /etc/schroot/schroot.conf.
-
-
-::
-
-  [precise_arm]
-  directory=/srv/chroot/precise_arm
-  users=USERNAME
-  root-users=USERNAME
-  groups=users
-  aliases=default
-  type=directory
-
-
-To verify that everything is working in the chroot, running ``schroot -c
-precise_arm`` should start a shell running in the schroot environment using
-qemu-arm to execute the ARM binaries. Running ``uname -m`` in the chroot should 
-yeild a result like ``armv7l``. Showing that we are emulating an ARM system.
-
-Start the schroot as the user root in order to configure the apt sources and
-to install the following packages:
-
-
-::
-
-  schroot -c precise_arm -u root
-  echo "deb http://ports.ubuntu.com/ubuntu-ports/ precise main universe restricted" > /etc/apt/sources.list
-  apt-get update
-  apt-get install libffi-dev libgc-dev python-dev build-essential libncurses5-dev libbz2-dev
-
-
-Now all dependencies should be in place and we can exit the schroot environment.
-
-
-Configuring scratchbox2
------------------------
-
-To configure the scratchbox we need to cd into the root directory of the rootfs
-we created before. From there we can call the sb2 configuration tools which
-will take the current directory as the base directory for the scratchbox2
-environment.
-
-::
-
-  cd /srv/chroot/precise_arm
-  sb2-init -c `which qemu-arm` ARM `which arm-linux-gnueabi-gcc`
-
-This will create a scratchbox2 based environment called ARM that maps calls to
-gcc done within the scratchbox to the arm-linux-gnueabi-gcc outside the
-scratchbox. Now we should have a working cross compilation toolchain in place
-and can start cross-translating programs for ARM.
-
-Translation
------------
-
-Having performed all the preliminary steps we should now be able to cross
-translate a program for ARM.  You can use this_ minimal
-target to test your setup before applying it to a larger project.
-
-Before starting the translator we need to set two environment variables, so the
-translator knows how to use the scratchbox environment. We need to set the
-**SB2** environment variable to point to the rootfs and the **SB2OPT** should
-contain the command line options for the sb2 command. If our rootfs is in the
-folder /srv/chroot/precise_arm and the scratchbox environment is called "ARM",
-the variables would be defined as follows.
-
-
-::
-
-  export SB2=/srv/chroot/precise_arm
-  export SB2OPT='-t ARM'
-
-Once this is set, you can call the translator. For example save this file 
-
-::
-
-  def main(args):
-      print "Hello World"
-      return 0
-
-  def target(*args):
-      return main, None
-
-and call the translator
-
-::
-
-  pypy ~/path_to_pypy_checkout/rpython/bin/rpython -O1 --platform=arm target.py
-
-If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``.
-
-To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call      
-
-::
-
-  pypy <path to rpython>/rpython/bin/rpython -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py
-
-The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_
-
-.. _`issue 1377`: https://bugs.pypy.org/issue1377
-.. _`issue 1376`: https://bugs.pypy.org/issue1376
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/build.rst
@@ -0,0 +1,170 @@
+Building PyPy from Source
+=========================
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Even when using PyPy to build PyPy, translation is time-consuming -- 30
+minutes on a fast machine -- and RAM-hungry.  You will need **at least** 2 GB
+of memory on a 32-bit machine and 4GB on a 64-bit machine.
+
+
+Clone the repository
+--------------------
+
+If you prefer to compile your own PyPy, or if you want to modify it, you
+will need to obtain a copy of the sources.  This can be done either by
+`downloading them from the download page`_ or by checking them out from the
+repository using mercurial.  We suggest using mercurial if you want to access
+the current development.
+
+.. _downloading them from the download page: http://pypy.org/download.html
+
+You must issue the following command on your
+command line, DOS box, or terminal::
+
+    hg clone http://bitbucket.org/pypy/pypy pypy
+
+This will clone the repository and place it into a directory
+named ``pypy``, and will get you the PyPy source in ``pypy/pypy`` and
+documentation files in ``pypy/pypy/doc``.
+We try to ensure that the tip is always stable, but it might
+occasionally be broken.  You may want to check out `our nightly tests`_:
+find a revision (12-chars alphanumeric string, e.g. "963e808156b3")
+that passed at least the
+``{linux32}`` tests (corresponding to a ``+`` sign on the
+line ``success``) and then, in your cloned repository, switch to this revision
+using::
+
+    hg up -r XXXXX
+
+where XXXXX is the revision id.
+
+.. _our nightly tests: http://buildbot.pypy.org/summary?branch=<trunk>
+
+
+Install build-time dependencies
+-------------------------------
+
+To build PyPy on Unix using the C translation backend, you need at least a C
+compiler and ``make`` installed. Further, some optional modules have additional
+dependencies:
+
+cffi, ctypes
+    libffi, pkg-config
+
+zlib
+    libz
+
+bz2
+    libbz2
+
+lzma (PyPy3 only)
+    liblzma
+
+sqlite3
+    libsqlite3
+
+curses
+    libncurses + cffi dependencies from above
+
+pyexpat
+    libexpat1
+
+_ssl
+    libssl
+
+Make sure to have these libraries (with development headers) installed before
+building PyPy, otherwise the resulting binary will not contain these modules.
+
+On Debian, this is the command to install all build-time dependencies::
+
+    apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
+    libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev
+
+For the optional lzma module on PyPy3 you will also need ``liblzma-dev``.
+
+On Fedora::
+
+    yum install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
+    lib-sqlite3-devel ncurses-devel expat-devel openssl-devel
+
+For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+
+On SLES11::
+
+    zypper install gcc make python-devel pkg-config \
+    zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
+    libexpat-devel libffi-devel python-curses
+
+For the optional lzma module on PyPy3 you will also need ``xz-devel``.
+
+On Mac OS X, most of these build-time dependencies are installed alongside
+the Developer Tools. However, note that in order for the installation to
+find them you may need to run::
+
+    xcode-select --install
+
+
+Run the translation
+-------------------
+
+Translate with JIT::
+
+    pypy rpython/bin/rpython --opt=jit pypy/goal/targetpypystandalone.py
+
+Translate without JIT::
+
+    pypy rpython/bin/rpython --opt=2 pypy/goal/targetpypystandalone.py
+
+(You can use ``python`` instead of ``pypy`` here, which will take longer
+but works too.)
+
+If everything works correctly this will create an executable ``pypy-c`` in the
+current directory. The executable behaves mostly like a normal Python
+interpreter (see :doc:`cpython_differences`).
+
+
+.. _translate-pypy:
+
+Translating with non-standard options
+-------------------------------------
+
+It is possible to have non-standard features enabled for translation,
+but they are not really tested any more.  Look, for example, at the
+:doc:`objspace proxies <objspace-proxies>` document.
+
+
+
+Installation
+------------
+
+PyPy dynamically finds the location of its libraries depending on the location
+of the executable. The directory hierarchy of a typical PyPy installation
+looks like this::
+
+    ./bin/pypy
+    ./include/
+    ./lib_pypy/
+    ./lib-python/2.7
+    ./site-packages/
+
+The hierarchy shown above is relative to a PREFIX directory. PREFIX is
+computed by starting from the directory where the executable resides, and
+"walking up" the filesystem until we find a directory containing ``lib_pypy``
+and ``lib-python/2.7``.
+
+To install PyPy system wide on unix-like systems, it is recommended to put the
+whole hierarchy alone (e.g. in ``/opt/pypy``) and put a symlink to the
+``pypy`` executable into ``/usr/bin`` or ``/usr/local/bin``.
+
+If the executable fails to find suitable libraries, it will report ``debug:
+WARNING: library path not found, using compiled-in sys.path`` and then attempt
+to continue normally. If the default path is usable, most code will be fine.
+However, the ``sys.prefix`` will be unset and some existing libraries assume
+that this is never the case.
+
+
+.. TODO windows
diff --git a/pypy/doc/cleanup.rst b/pypy/doc/cleanup.rst
deleted file mode 100644
--- a/pypy/doc/cleanup.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-Old documentation that needs review
------------------------------------
-
-.. The following stuff is old (and crufty?), and needs further investigation:
-
-.. doc-index: This needs merging somehow
-
-.. toctree::
-
-   distribution.rst
-
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -1,21 +1,17 @@
-====================================
 Coding Guide
-====================================
+============
 
 .. contents::
 
 This document describes coding requirements and conventions for
 working with the PyPy code base.  Please read it carefully and
 ask back any questions you might have. The document does not talk
-very much about coding style issues. We mostly follow `PEP 8`_ though.
+very much about coding style issues. We mostly follow :pep:`8` though.
 If in doubt, follow the style that is already present in the code base.
 
-.. _`PEP 8`: http://www.python.org/dev/peps/pep-0008/
-
-.. _`RPython`:
 
 Overview and motivation
-========================
+------------------------
 
 We are writing a Python interpreter in Python, using Python's well known
 ability to step behind the algorithmic problems as a language. At first glance,
@@ -25,7 +21,7 @@
 
 
 CPython vs. PyPy
--------------------
+~~~~~~~~~~~~~~~~
 
 Compared to the CPython implementation, Python takes the role of the C
 Code. We rewrite the CPython interpreter in Python itself.  We could
@@ -44,20 +40,20 @@
 but let's stick with this somewhat canonical approach.
 
 
-.. _`application-level`:
-.. _`interpreter-level`:
+.. _application-level:
+.. _interpreter-level:
 
 Application-level and interpreter-level execution and objects
--------------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Since Python is used for implementing all of our code base, there is a
-crucial distinction to be aware of: that between *interpreter-level* objects and 
+crucial distinction to be aware of: that between *interpreter-level* objects and
 *application-level* objects.  The latter are the ones that you deal with
 when you write normal python programs.  Interpreter-level code, however,
 cannot invoke operations nor access attributes from application-level
 objects.  You will immediately recognize any interpreter level code in
 PyPy, because half the variable and object names start with a ``w_``, which
-indicates that they are `wrapped`_ application-level values. 
+indicates that they are `wrapped`_ application-level values.
 
 Let's show the difference with a simple example.  To sum the contents of
 two variables ``a`` and ``b``, one would write the simple application-level
@@ -80,10 +76,10 @@
 interpreting.
 
 
-.. _`app-preferable`: 
+.. _app-preferable:
 
-Application level is often preferable 
--------------------------------------
+Application level is often preferable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Application-level code is substantially higher-level, and therefore
 correspondingly easier to write and debug.  For example, suppose we want
@@ -113,11 +109,11 @@
             space.setitem(w_self, w_key, w_value)
 
 This interpreter-level implementation looks much more similar to the C
-source code.  It is still more readable than its C counterpart because 
-it doesn't contain memory management details and can use Python's native 
-exception mechanism. 
+source code.  It is still more readable than its C counterpart because
+it doesn't contain memory management details and can use Python's native
+exception mechanism.
 
-In any case, it should be obvious that the application-level implementation 
+In any case, it should be obvious that the application-level implementation
 is definitely more readable, more elegant and more maintainable than the
 interpreter-level one (and indeed, dict.update is really implemented at
 applevel in PyPy).
@@ -129,10 +125,11 @@
 level code is usually preferable.  We have an abstraction (called the
 'Gateway') which allows the caller of a function to remain ignorant of
 whether a particular function is implemented at application or
-interpreter level. 
+interpreter level.
+
 
 Our runtime interpreter is "RPython"
-----------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In order to make a C code generator feasible all code on interpreter level has
 to restrict itself to a subset of the Python language, and we adhere to some
@@ -159,7 +156,7 @@
 elegant: For the definition of all the opcodes of the Python
 interpreter, the module ``dis`` is imported and used to initialize our
 bytecode interpreter.  (See ``__initclass__`` in
-`pypy/interpreter/pyopcode.py`_).  This
+:source:`pypy/interpreter/pyopcode.py`).  This
 saves us from adding extra modules to PyPy. The import code is run at
 startup time, and we are allowed to use the CPython builtin import
 function.
@@ -172,308 +169,13 @@
 enables the code generator to emit efficient machine level replacements
 for pure integer objects, for instance.
 
-RPython
-=================
-
-RPython Definition
-------------------
-
-RPython is a restricted subset of Python that is amenable to static analysis.
-Although there are additions to the language and some things might surprisingly
-work, this is a rough list of restrictions that should be considered. Note
-that there are tons of special cased restrictions that you'll encounter
-as you go. The exact definition is "RPython is everything that our translation
-toolchain can accept" :)
-
-.. _`wrapped object`: coding-guide.html#wrapping-rules
-
-Flow restrictions
--------------------------
-
-**variables**
-
-  variables should contain values of at most one type as described in
-  `Object restrictions`_ at each control flow point, that means for
-  example that joining control paths using the same variable to
-  contain both a string and a int must be avoided.  It is allowed to
-  mix None (basically with the role of a null pointer) with many other
-  types: `wrapped objects`, class instances, lists, dicts, strings, etc.
-  but *not* with int, floats or tuples.
-
-**constants**
-
-  all module globals are considered constants.  Their binding must not
-  be changed at run-time.  Moreover, global (i.e. prebuilt) lists and
-  dictionaries are supposed to be immutable: modifying e.g. a global
-  list will give inconsistent results.  However, global instances don't
-  have this restriction, so if you need mutable global state, store it
-  in the attributes of some prebuilt singleton instance.
-
-
-
-**control structures**
-
-  all allowed, ``for`` loops restricted to builtin types, generators
-  very restricted.
-
-**range**
-
-  ``range`` and ``xrange`` are identical. ``range`` does not necessarily create an array,
-  only if the result is modified. It is allowed everywhere and completely
-  implemented. The only visible difference to CPython is the inaccessibility
-  of the ``xrange`` fields start, stop and step.
-
-**definitions**
-
-  run-time definition of classes or functions is not allowed.
-
-**generators**
-
-  generators are supported, but their exact scope is very limited. you can't
-  merge two different generator in one control point.
-
-**exceptions**
-
-+ fully supported
-+ see below `Exception rules`_ for restrictions on exceptions raised by built-in operations
-
-
-Object restrictions
--------------------------
-
-We are using
-
-**integer, float, boolean**
-
-  works.
-
-**strings**
-
-  a lot of, but not all string methods are supported and those that are
-  supported, not necesarilly accept all arguments.  Indexes can be
-  negative.  In case they are not, then you get slightly more efficient
-  code if the translator can prove that they are non-negative.  When
-  slicing a string it is necessary to prove that the slice start and
-  stop indexes are non-negative. There is no implicit str-to-unicode cast
-  anywhere. Simple string formatting using the ``%`` operator works, as long
-  as the format string is known at translation time; the only supported
-  formatting specifiers are ``%s``, ``%d``, ``%x``, ``%o``, ``%f``, plus
-  ``%r`` but only for user-defined instances. Modifiers such as conversion
-  flags, precision, length etc. are not supported. Moreover, it is forbidden
-  to mix unicode and strings when formatting.
-
-**tuples**
-
-  no variable-length tuples; use them to store or return pairs or n-tuples of
-  values. Each combination of types for elements and length constitute
-  a separate and not mixable type.
-
-**lists**
-
-  lists are used as an allocated array.  Lists are over-allocated, so list.append()
-  is reasonably fast. However, if you use a fixed-size list, the code
-  is more efficient. Annotator can figure out most of the time that your
-  list is fixed-size, even when you use list comprehension.
-  Negative or out-of-bound indexes are only allowed for the
-  most common operations, as follows:
-
-  - *indexing*:
-    positive and negative indexes are allowed. Indexes are checked when requested
-    by an IndexError exception clause.
-  
-  - *slicing*:
-    the slice start must be within bounds. The stop doesn't need to, but it must
-    not be smaller than the start.  All negative indexes are disallowed, except for
-    the [:-1] special case.  No step.  Slice deletion follows the same rules.
-    
-  - *slice assignment*:
-    only supports ``lst[x:y] = sublist``, if ``len(sublist) == y - x``.
-    In other words, slice assignment cannot change the total length of the list,
-    but just replace items.
-
-  - *other operators*:
-    ``+``, ``+=``, ``in``, ``*``, ``*=``, ``==``, ``!=`` work as expected.
-
-  - *methods*:
-    append, index, insert, extend, reverse, pop.  The index used in pop() follows
-    the same rules as for *indexing* above.  The index used in insert() must be within
-    bounds and not negative.
-
-**dicts**
-
-  dicts with a unique key type only, provided it is hashable. Custom
-  hash functions and custom equality will not be honored.
-  Use ``rpython.rlib.objectmodel.r_dict`` for custom hash functions.
-
-
-**list comprehensions**
-
-  May be used to create allocated, initialized arrays.
-
-**functions**
-
-+ statically called functions may use defaults and a variable number of
-  arguments (which may be passed as a list instead of a tuple, so write code
-  that does not depend on it being a tuple).
-
-+ dynamic dispatch enforces the use of signatures that are equal for all
-  possible called function, or at least "compatible enough".  This
-  concerns mainly method calls, when the method is overridden or in any
-  way given different definitions in different classes.  It also concerns
-  the less common case of explicitly manipulated function objects.
-  Describing the exact compatibility rules is rather involved (but if you
-  break them, you should get explicit errors from the rtyper and not
-  obscure crashes.)
-
-**builtin functions**
-
-  A number of builtin functions can be used.  The precise set can be
-  found in `rpython/annotator/builtin.py`_ (see ``def builtin_xxx()``).
-  Some builtin functions may be limited in what they support, though.
-
-  ``int, float, str, ord, chr``... are available as simple conversion
-  functions.  Note that ``int, float, str``... have a special meaning as
-  a type inside of isinstance only.
-
-**classes**
-
-+ methods and other class attributes do not change after startup
-+ single inheritance is fully supported
-+ use `rpython.rlib.objectmodel.import_from_mixin(M)` in a class
-  body to copy the whole content of a class `M`.  This can be used
-  to implement mixins: functions and staticmethods are duplicated
-  (the other class attributes are just copied unmodified).
-
-+ classes are first-class objects too
-
-**objects**
-
-  Normal rules apply. The only special methods that are honoured are
-  ``__init__``, ``__del__``, ``__len__``, ``__getitem__``, ``__setitem__``,
-  ``__getslice__``, ``__setslice__``, and ``__iter__``. To handle slicing,
-  ``__getslice__`` and ``__setslice__`` must be used; using ``__getitem__`` and
-   ``__setitem__`` for slicing isn't supported. Additionally, using negative
-   indices for slicing is still not support, even when using ``__getslice__``.
-
-This layout makes the number of types to take care about quite limited.
-
-
-Integer Types
--------------------------
-
-While implementing the integer type, we stumbled over the problem that
-integers are quite in flux in CPython right now. Starting with Python 2.4,
-integers mutate into longs on overflow.  In contrast, we need
-a way to perform wrap-around machine-sized arithmetic by default, while still
-being able to check for overflow when we need it explicitly.  Moreover, we need
-a consistent behavior before and after translation.
-
-We use normal integers for signed arithmetic.  It means that before
-translation we get longs in case of overflow, and after translation we get a
-silent wrap-around.  Whenever we need more control, we use the following
-helpers (which live in `rpython/rlib/rarithmetic.py`_):
-
-**ovfcheck()**
-
-  This special function should only be used with a single arithmetic operation
-  as its argument, e.g. ``z = ovfcheck(x+y)``.  Its intended meaning is to
-  perform the given operation in overflow-checking mode.
-
-  At run-time, in Python, the ovfcheck() function itself checks the result
-  and raises OverflowError if it is a ``long``.  But the code generators use
-  ovfcheck() as a hint: they replace the whole ``ovfcheck(x+y)`` expression
-  with a single overflow-checking addition in C.
-
-**intmask()**
-
-  This function is used for wrap-around arithmetic.  It returns the lower bits
-  of its argument, masking away anything that doesn't fit in a C "signed long int".
-  Its purpose is, in Python, to convert from a Python ``long`` that resulted from a
-  previous operation back to a Python ``int``.  The code generators ignore
-  intmask() entirely, as they are doing wrap-around signed arithmetic all the time
-  by default anyway.  (We have no equivalent of the "int" versus "long int"
-  distinction of C at the moment and assume "long ints" everywhere.)
-
-**r_uint**
-
-  In a few cases (e.g. hash table manipulation), we need machine-sized unsigned
-  arithmetic.  For these cases there is the r_uint class, which is a pure
-  Python implementation of word-sized unsigned integers that silently wrap
-  around.  ("word-sized" and "machine-sized" are used equivalently and mean
-  the native size, which you get using "unsigned long" in C.)
-  The purpose of this class (as opposed to helper functions as above)
-  is consistent typing: both Python and the annotator will propagate r_uint
-  instances in the program and interpret all the operations between them as
-  unsigned.  Instances of r_uint are special-cased by the code generators to
-  use the appropriate low-level type and operations.
-  Mixing of (signed) integers and r_uint in operations produces r_uint that
-  means unsigned results.  To convert back from r_uint to signed integers, use
-  intmask().
-
-
-Exception rules
----------------------
-
-Exceptions are by default not generated for simple cases.::
-
-    #!/usr/bin/python
-
-        lst = [1,2,3,4,5]
-        item = lst[i]    # this code is not checked for out-of-bound access
-
-        try:
-            item = lst[i]
-        except IndexError:
-            # complain
-
-Code with no exception handlers does not raise exceptions (after it has been
-translated, that is.  When you run it on top of CPython, it may raise
-exceptions, of course). By supplying an exception handler, you ask for error
-checking. Without, you assure the system that the operation cannot fail.
-This rule does not apply to *function calls*: any called function is
-assumed to be allowed to raise any exception.
-
-For example::
-
-    x = 5.1
-    x = x + 1.2       # not checked for float overflow
-    try:
-        x = x + 1.2
-    except OverflowError:
-        # float result too big
-
-But::
-
-    z = some_function(x, y)    # can raise any exception
-    try:
-        z = some_other_function(x, y)
-    except IndexError:
-        # only catches explicitly-raised IndexErrors in some_other_function()
-        # other exceptions can be raised, too, and will not be caught here.
-
-The ovfcheck() function described above follows the same rule: in case of
-overflow, it explicitly raise OverflowError, which can be caught anywhere.
-
-Exceptions explicitly raised or re-raised will always be generated.
-
-PyPy is debuggable on top of CPython
-------------------------------------
-
-PyPy has the advantage that it is runnable on standard
-CPython.  That means, we can run all of PyPy with all exception
-handling enabled, so we might catch cases where we failed to
-adhere to our implicit assertions.
-
-.. _`wrapping rules`:
-.. _`wrapped`:
-
-
+.. _wrapped:
 
 Wrapping rules
-==============
+--------------
 
 Wrapping
---------- 
+~~~~~~~~
 
 PyPy is made of Python source code at two levels: there is on the one hand
 *application-level code* that looks like normal Python code, and that
@@ -500,7 +202,7 @@
 structure.
 
 For example, an application-level Python ``list``
-is implemented by the `standard object space`_ as an
+is implemented by the :ref:`standard object space <standard-object-space>` as an
 instance of ``W_ListObject``, which has an instance attribute
 ``wrappeditems`` (an interpreter-level list which contains the
 application-level list's items as wrapped objects).
@@ -509,7 +211,7 @@
 
 
 Naming conventions
-------------------
+~~~~~~~~~~~~~~~~~~
 
 * ``space``: the object space is only visible at
   interpreter-level code, where it is by convention passed around by the name
@@ -529,14 +231,14 @@
 
 
 Operations on ``w_xxx``
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 The core bytecode interpreter considers wrapped objects as black boxes.
 It is not allowed to inspect them directly.  The allowed
 operations are all implemented on the object space: they are
 called ``space.xxx()``, where ``xxx`` is a standard operation
 name (``add``, ``getattr``, ``call``, ``eq``...). They are documented in the
-`object space document`_.
+:ref:`object space document <objspace-interface>`.
 
 A short warning: **don't do** ``w_x == w_y`` or ``w_x is w_y``!
 rationale for this rule is that there is no reason that two
@@ -548,12 +250,11 @@
 use ``space.is_true(space.is_(w_x, w_y))`` or better
 ``space.is_w(w_x, w_y)``.
 
-.. _`object space document`: objspace.html#interface
 
-.. _`applevel-exceptions`: 
+.. _applevel-exceptions:
 
 Application-level exceptions
-----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Interpreter-level code can use exceptions freely.  However,
 all application-level exceptions are represented as an
@@ -585,10 +286,10 @@
 instances of subclasses.
 
 
-.. _`modules`:
+.. _modules:
 
 Modules in PyPy
-===============
+---------------
 
 Modules visible from application programs are imported from
 interpreter or application level files.  PyPy reuses almost all python
@@ -597,18 +298,19 @@
 because they rely on implementation details of CPython.
 
 If we don't just modify an original CPython module but need to rewrite
-it from scratch we put it into `lib_pypy/`_ as a pure application level
+it from scratch we put it into :source:`lib_pypy/` as a pure application level
 module.
 
 When we need access to interpreter-level objects we put the module into
-`pypy/module`_.  Such modules use a `mixed module mechanism`_
+:source:`pypy/module`.  Such modules use a `mixed module mechanism`_
 which makes it convenient to use both interpreter- and application-level parts
 for the implementation.  Note that there is no extra facility for
 pure-interpreter level modules, you just write a mixed module and leave the
 application-level part empty.
 
+
 Determining the location of a module implementation
----------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can interactively find out where a module comes from, when running py.py.
 here are examples for the possible locations::
@@ -626,8 +328,9 @@
     '/home/hpk/pypy-dist/lib-python/2.7/os.py'
     >>>>
 
+
 Module directories / Import order
----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Here is the order in which PyPy looks up Python modules:
 
@@ -650,10 +353,11 @@
 
     The modified CPython library.
 
-.. _`modify modules`:
+
+.. _modify modules:
 
 Modifying a CPython library module or regression test
--------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Although PyPy is very compatible with CPython we sometimes need
 to change modules contained in our copy of the standard library,
@@ -665,34 +369,31 @@
 to see what is changed we have a branch called `vendor/stdlib`
 wich contains the unmodified cpython stdlib
 
-.. _`mixed module mechanism`:
-.. _`mixed modules`:
+
+.. _mixed module mechanism:
+.. _mixed-modules:
 
 Implementing a mixed interpreter/application level Module
----------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If a module needs to access PyPy's interpreter level
 then it is implemented as a mixed module.
 
-Mixed modules are directories in `pypy/module`_ with an  `__init__.py`
+Mixed modules are directories in :source:`pypy/module` with an  `__init__.py`
 file containing specifications where each name in a module comes from.
 Only specified names will be exported to a Mixed Module's applevel
 namespace.
 
-Sometimes it is necessary to really write some functions in C (or
-whatever target language). See `rffi`_ and `external functions
-documentation`_ for details. The latter approach is cumbersome and
-being phased out and former has currently quite a few rough edges.
+Sometimes it is necessary to really write some functions in C (or whatever
+target language). See :ref:`rffi <rpython:rffi>` details.
 
-.. _`rffi`: rffi.html
-.. _`external functions documentation`: translation.html#extfunccalls
 
 application level definitions
-.............................
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Application level specifications are found in the `appleveldefs`
 dictionary found in ``__init__.py`` files of directories in ``pypy/module``.
-For example, in `pypy/module/__builtin__/__init__.py`_ you find the following
+For example, in :source:`pypy/module/__builtin__/__init__.py` you find the following
 entry specifying where ``__builtin__.locals`` comes from::
 
      ...
@@ -703,12 +404,13 @@
 interpreted at application level and the wrapped function value for ``locals``
 will be extracted accordingly.
 
+
 interpreter level definitions
-.............................
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Interpreter level specifications are found in the ``interpleveldefs``
 dictionary found in ``__init__.py`` files of directories in ``pypy/module``.
-For example, in `pypy/module/__builtin__/__init__.py`_ the following
+For example, in :source:`pypy/module/__builtin__/__init__.py` the following
 entry specifies where ``__builtin__.len`` comes from::
 
      ...
@@ -724,7 +426,7 @@
         return space.len(w_obj)
 
 Exposed interpreter level functions usually take a ``space`` argument
-and some wrapped values (see `wrapping rules`_) .
+and some wrapped values (see `Wrapping rules`_) .
 
 You can also use a convenient shortcut in ``interpleveldefs`` dictionaries:
 namely an expression in parentheses to specify an interpreter level
@@ -743,37 +445,40 @@
 --withoutmod-mymodule (the latter being the default)) for py.py and
 translate.py.
 
+
 Testing modules in ``lib_pypy/``
---------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can go to the `pypy/module/test_lib_pypy/`_ directory and invoke the testing tool
+You can go to the :source:`pypy/module/test_lib_pypy/` directory and invoke the testing tool
 ("py.test" or "python ../../pypy/test_all.py") to run tests against the
-lib_pypy hierarchy.  Note, that tests in `pypy/module/test_lib_pypy/`_ are allowed
+lib_pypy hierarchy.  Note, that tests in :source:`pypy/module/test_lib_pypy/` are allowed
 and encouraged to let their tests run at interpreter level although
-`lib_pypy/`_ modules eventually live at PyPy's application level.
+:source:`lib_pypy/` modules eventually live at PyPy's application level.
 This allows us to quickly test our python-coded reimplementations
 against CPython.
 
+
 Testing modules in ``pypy/module``
-----------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Simply change to ``pypy/module`` or to a subdirectory and `run the
 tests as usual`_.
 
 
 Testing modules in ``lib-python``
------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In order to let CPython's regression tests run against PyPy
-you can switch to the `lib-python/`_ directory and run
+you can switch to the :source:`lib-python/` directory and run
 the testing tool in order to start compliance tests.
 (XXX check windows compatibility for producing test reports).
 
+
 Naming conventions and directory layout
-===========================================
+---------------------------------------
 
 Directory and File Naming
--------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - directories/modules/namespaces are always **lowercase**
 
@@ -786,8 +491,9 @@
 
 - keep filenames concise and completion-friendly.
 
+
 Naming of python objects
-------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~
 
 - class names are **CamelCase**
 
@@ -803,8 +509,9 @@
   includes w_self.  Don't use ``w_`` in application level
   python only code.
 
+
 Committing & Branching to the repository
------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - write good log messages because several people
   are reading the diffs.
@@ -815,22 +522,23 @@
   ``try1`` doesn't already exists) you should do::
 
     hg branch try1
-    
+
   The branch will be recorded in the repository only after a commit. To switch
   back to the default branch::
-  
+
     hg update default
-    
+
   For further details use the help or refer to the `official wiki`_::
-  
+
     hg help branch
 
-.. _`official wiki`: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: http://mercurial.selenic.com/wiki/Branch
 
-.. _`using development tracker`:
+
+.. _using-development-tracker:
 
 Using the development bug/feature tracker
-=========================================
+-----------------------------------------
 
 We have a `development tracker`_, based on Richard Jones'
 `roundup`_ application.  You can file bugs,
@@ -838,15 +546,14 @@
 for the next milestone, both from an E-Mail and from a
 web interface.
 
-.. _`development tracker`: https://bugs.pypy.org/
-.. _`roundup`: http://roundup.sourceforge.net/
+.. _development tracker: https://bugs.pypy.org/
+.. _roundup: http://roundup.sourceforge.net/
 
 
-.. _`testing in PyPy`:
-.. _`test-design`: 
+.. _testing:
 
 Testing in PyPy
-===============
+---------------
 
 Our tests are based on the `py.test`_ tool which lets you write
 unittests without boilerplate.  All tests of modules
@@ -859,12 +566,11 @@
 - **Application Level tests**. They run at application level which means
   that they look like straight python code but they are interpreted by PyPy.
 
-.. _`standard object space`: objspace.html#standard-object-space
-.. _`objectspace`: objspace.html
-.. _`py.test`: http://pytest.org/
+.. _py.test: http://pytest.org/
+
 
 Interpreter level tests
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 You can write test functions and methods like this::
 
@@ -880,8 +586,9 @@
 module global level and use plain 'assert' statements thanks to the
 usage of the `py.test`_ tool.
 
+
 Application Level tests
------------------------
+~~~~~~~~~~~~~~~~~~~~~~~
 
 For testing the conformance and well-behavedness of PyPy it
 is often sufficient to write "normal" application-level
@@ -916,7 +623,8 @@
             assert self.d["a"] == 1
             assert self.d["b"] == 2
 
-.. _`run the tests as usual`:
+
+.. _run the tests as usual:
 
 Another possibility is to use cls.space.appexec, for example::
 
@@ -959,7 +667,7 @@
 
 
 Command line tool test_all
---------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can run almost all of PyPy's tests by invoking::
 
@@ -969,8 +677,9 @@
 located in the ``py/bin/`` directory.  For switches to
 modify test execution pass the ``-h`` option.
 
+
 Coverage reports
-----------------
+~~~~~~~~~~~~~~~~
 
 In order to get coverage reports the `pytest-cov`_ plugin is included.
 it adds some extra requirements ( coverage_ and `cov-core`_ )
@@ -978,12 +687,13 @@
 
   python test_all.py --cov file_or_direcory_to_cover file_or_directory
 
-.. _`pytest-cov`: http://pypi.python.org/pypi/pytest-cov
-.. _`coverage`: http://pypi.python.org/pypi/coverage
-.. _`cov-core`: http://pypi.python.org/pypi/cov-core
+.. _pytest-cov: http://pypi.python.org/pypi/pytest-cov
+.. _coverage: http://pypi.python.org/pypi/coverage
+.. _cov-core: http://pypi.python.org/pypi/cov-core
+
 
 Test conventions
-----------------
+~~~~~~~~~~~~~~~~
 
 - adding features requires adding appropriate tests.  (It often even
   makes sense to first write the tests so that you are sure that they
@@ -993,29 +703,28 @@
   which contain unit tests.  Such scripts can usually be executed
   directly or are collectively run by pypy/test_all.py
 
-.. _`change documentation and website`:
+
+.. _change documentation and website:
 
 Changing documentation and website
-==================================
+----------------------------------
 
 documentation/website files in your local checkout
----------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Most of the PyPy's documentation is kept in `pypy/doc`.
 You can simply edit or add '.rst' files which contain ReST-markuped
 files.  Here is a `ReST quickstart`_ but you can also just look
 at the existing documentation and see how things work.
 
-.. _`ReST quickstart`: http://docutils.sourceforge.net/docs/user/rst/quickref.html
-
 Note that the web site of http://pypy.org/ is maintained separately.
 For now it is in the repository https://bitbucket.org/pypy/pypy.org
 
+.. _ReST quickstart: http://docutils.sourceforge.net/docs/user/rst/quickref.html
+
+
 Automatically test documentation/website changes
-------------------------------------------------
-
-.. _`sphinx home page`:
-.. _`sphinx`: http://sphinx.pocoo.org/
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 We automatically check referential integrity and ReST-conformance.  In order to
 run the tests you need sphinx_ installed.  Then go to the local checkout
@@ -1035,5 +744,4 @@
 
 which will check that remote URLs are reachable.
 
-
-.. include:: _ref.txt
+.. _sphinx: http://sphinx.pocoo.org/
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -40,9 +40,9 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx',
-              'sphinx.ext.todo', 'sphinx.ext.ifconfig', 'sphinx.ext.graphviz',
-              'pypyconfig']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.extlinks',
+              'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.ifconfig',
+              'sphinx.ext.graphviz', 'pypyconfig']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -215,7 +215,8 @@
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None),
+                       'rpython': ('http://rpython.readthedocs.org/en/latest/', None)}
 
 # -- Options for manpage output-------------------------------------------------
 
@@ -225,3 +226,4 @@
    u'The PyPy Project', 1)
 ]
 
+extlinks = {'source': ('https://bitbucket.org/pypy/pypy/src/default/%s', '')}
diff --git a/pypy/doc/config/confrest.py b/pypy/doc/config/confrest.py
deleted file mode 100644
--- a/pypy/doc/config/confrest.py
+++ /dev/null
@@ -1,62 +0,0 @@
-from pypy.doc.confrest import *
-from pypy.config.makerestdoc import make_cmdline_overview
-from pypy.config import pypyoption
-from rpython.config.config import Config
-from rpython.config import translationoption
-
-
-all_optiondescrs = [pypyoption.pypy_optiondescription,
-                    translationoption.translation_optiondescription,
-                   ]
-start_to_descr = dict([(descr._name, descr) for descr in all_optiondescrs])
-
-class PyPyPage(PyPyPage): 
-    def fill(self):
-        super(PyPyPage, self).fill()
-        self.menubar[:] = html.div(
-            html.a("general documentation", href="../index.html",
-                   class_="menu"), " ",
-            html.a("config index", href="index.html",
-                   class_="menu"), " ",
-            html.a("command-line overview", href="commandline.html",
-                   class_="menu"), " ",
-            " ", id="menubar")
-
-class Project(Project): 
-    stylesheet = "../style.css"
-    title = "PyPy Configuration"
-    prefix_title = "PyPy Configuration"
-    Page = PyPyPage 
-
-    def get_content(self, txtpath, encoding):
-        if txtpath.basename == "commandline.rst":
-            result = []
-            for line in txtpath.read().splitlines():
-                if line.startswith('.. GENERATE:'):
-                    start = line[len('.. GENERATE:'):].strip()
-                    descr = start_to_descr[start]
-                    line = make_cmdline_overview(descr, title=False).text()
-                result.append(line)
-            return "\n".join(result)
-        fullpath = txtpath.purebasename
-        start = fullpath.split(".")[0]

From noreply at buildbot.pypy.org  Wed Dec 10 16:47:50 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 10 Dec 2014 16:47:50 +0100 (CET)
Subject: [pypy-commit] pypy default: formatting
Message-ID: <20141210154750.9F31C1D22F1@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74881:6cc9bad3f7f5
Date: 2014-12-10 17:46 +0200
http://bitbucket.org/pypy/pypy/changeset/6cc9bad3f7f5/

Log:	formatting

diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -42,8 +42,9 @@
             oldop = self.pure_operations.get(args, None)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 assert oldop.getopnum() == op.getopnum()
-                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
+                self.optimizer.make_equal_to(op.result,
+                                             self.getvalue(oldop.result),
+                                             True)
                 return
             else:
                 self.pure_operations[args] = op

From noreply at buildbot.pypy.org  Wed Dec 10 16:47:52 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 10 Dec 2014 16:47:52 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20141210154752.2965A1D22F1@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74882:db19352f7582
Date: 2014-12-10 17:46 +0200
http://bitbucket.org/pypy/pypy/changeset/db19352f7582/

Log:	merge

diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -14,6 +14,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.module._rawffi.alt.type_converter import FromAppLevelConverter, ToAppLevelConverter
 from pypy.module._rawffi.interp_rawffi import got_libffi_error, wrap_dlopenerror
+from pypy.module._rawffi import lasterror
 
 import os
 if os.name == 'nt':
@@ -201,11 +202,23 @@
         self.func = func
         self.argchain = argchain
 
+    def before(self):
+        lasterror.restore_last_error(self.space)
+
+    def after(self):
+        lasterror.save_last_error(self.space)
+
     def get_longlong(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.LONGLONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.LONGLONG)
+        self.after()
+        return x
 
     def get_ulonglong(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.ULONGLONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.ULONGLONG)
+        self.after()
+        return x
 
     def get_signed(self, w_ffitype):
         # if the declared return type of the function is smaller than LONG,
@@ -216,64 +229,94 @@
         # to space.wrap in order to get a nice applevel <int>.
         #
         restype = w_ffitype.get_ffitype()
+        self.before()
         call = self.func.call
         if restype is libffi.types.slong:
-            return call(self.argchain, rffi.LONG)
+            x = call(self.argchain, rffi.LONG)
         elif restype is libffi.types.sint:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.INT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.INT))
         elif restype is libffi.types.sshort:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT))
         elif restype is libffi.types.schar:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR))
         else:
-            self.error(w_ffitype)
+            raise self.error(w_ffitype)
+        self.after()
+        return x
 
     def get_unsigned(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.ULONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.ULONG)
+        self.after()
+        return x
 
     def get_unsigned_which_fits_into_a_signed(self, w_ffitype):
         # the same comment as get_signed apply
         restype = w_ffitype.get_ffitype()
+        self.before()
         call = self.func.call
         if restype is libffi.types.uint:
             assert not libffi.IS_32_BIT
             # on 32bit machines, we should never get here, because it's a case
             # which has already been handled by get_unsigned above.
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT))
         elif restype is libffi.types.ushort:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT))
         elif restype is libffi.types.uchar:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR))
         else:
-            self.error(w_ffitype)
+            raise self.error(w_ffitype)
+        self.after()
+        return x
 
 
     def get_pointer(self, w_ffitype):
+        self.before()
         ptrres = self.func.call(self.argchain, rffi.VOIDP)
+        self.after()
         return rffi.cast(rffi.ULONG, ptrres)
 
     def get_char(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.UCHAR)
+        self.before()
+        x = self.func.call(self.argchain, rffi.UCHAR)
+        self.after()
+        return x
 
     def get_unichar(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.WCHAR_T)
+        self.before()
+        x = self.func.call(self.argchain, rffi.WCHAR_T)
+        self.after()
+        return x
 
     def get_float(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.DOUBLE)
+        self.before()
+        x = self.func.call(self.argchain, rffi.DOUBLE)
+        self.after()
+        return x
 
     def get_singlefloat(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.FLOAT)
+        self.before()
+        x = self.func.call(self.argchain, rffi.FLOAT)
+        self.after()
+        return x
 
     def get_struct(self, w_ffitype, w_structdescr):
+        self.before()
         addr = self.func.call(self.argchain, rffi.LONG, is_struct=True)
+        self.after()
         return w_structdescr.fromaddress(self.space, addr)
 
     def get_struct_rawffi(self, w_ffitype, w_structdescr):
+        self.before()
         uintval = self.func.call(self.argchain, rffi.ULONG, is_struct=True)
+        self.after()
         return w_structdescr.fromaddress(self.space, uintval)
 
     def get_void(self, w_ffitype):
-        return self.func.call(self.argchain, lltype.Void)
+        self.before()
+        x = self.func.call(self.argchain, lltype.Void)
+        self.after()
+        return x
 
 
 def unpack_argtypes(space, w_argtypes, w_restype):
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -18,6 +18,7 @@
 from rpython.rlib.rarithmetic import intmask, r_uint
 from pypy.module._rawffi.buffer import RawFFIBuffer
 from pypy.module._rawffi.tracker import tracker
+from pypy.module._rawffi import lasterror
 
 TYPEMAP = {
     # XXX A mess with unsigned/signed/normal chars :-/
@@ -495,10 +496,14 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                lasterror.restore_last_error(space)
                 self.ptr.call(args_ll, result.ll_buffer)
+                lasterror.save_last_error(space)
                 return space.wrap(result)
             else:
+                lasterror.restore_last_error(space)
                 self.ptr.call(args_ll, lltype.nullptr(rffi.VOIDP.TO))
+                lasterror.save_last_error(space)
                 return space.w_None
         except StackCheckError, e:
             raise OperationError(space.w_ValueError, space.wrap(e.message))
@@ -615,12 +620,10 @@
 
 if sys.platform == 'win32':
     def get_last_error(space):
-        from rpython.rlib.rwin32 import GetLastError
-        return space.wrap(GetLastError())
+        return space.wrap(lasterror.fetch_last_error(space))
     @unwrap_spec(error=int)
     def set_last_error(space, error):
-        from rpython.rlib.rwin32 import SetLastError
-        SetLastError(error)
+        lasterror.store_last_error(space, error)
 else:
     # always have at least a dummy version of these functions
     # (https://bugs.pypy.org/issue1242)
diff --git a/pypy/module/_rawffi/lasterror.py b/pypy/module/_rawffi/lasterror.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_rawffi/lasterror.py
@@ -0,0 +1,40 @@
+# For Windows only.
+# https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
+
+import os
+
+_MS_WINDOWS = os.name == "nt"
+
+
+if _MS_WINDOWS:
+    from rpython.rlib import rwin32
+    from pypy.interpreter.executioncontext import ExecutionContext
+
+
+    ExecutionContext._rawffi_last_error = 0
+
+    def fetch_last_error(space):
+        ec = space.getexecutioncontext()
+        return ec._rawffi_last_error
+
+    def store_last_error(space, last_error):
+        ec = space.getexecutioncontext()
+        ec._rawffi_last_error = last_error
+
+    def restore_last_error(space):
+        ec = space.getexecutioncontext()
+        lasterror = ec._rawffi_last_error
+        rwin32.SetLastError(lasterror)
+
+    def save_last_error(space):
+        lasterror = rwin32.GetLastError()
+        ec = space.getexecutioncontext()
+        ec._rawffi_last_error = lasterror
+
+else:
+
+    def restore_last_error(space):
+        pass
+
+    def save_last_error(space):
+        pass
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -16,6 +16,7 @@
         #include "src/precommondefs.h"
         #include <stdlib.h>
         #include <stdio.h>
+        #include <errno.h>
 
         struct x
         {
@@ -204,6 +205,24 @@
             return inp;
         }
 
+        RPY_EXPORTED
+        int check_errno(int incoming)
+        {
+            int old_errno = errno;
+            errno = incoming;
+            return old_errno;
+        }
+
+        #ifdef _WIN32
+        #include <Windows.h>
+        RPY_EXPORTED
+        int check_last_error(int incoming)
+        {
+            int old_errno = GetLastError();
+            SetLastError(incoming);
+            return old_errno;
+        }
+        #endif
         '''))
         eci = ExternalCompilationInfo(include_dirs=[cdir])
         return str(platform.compile([c_file], eci, 'x', standalone=False))
@@ -1150,6 +1169,37 @@
         raises(OverflowError, "arg1[0] = 10**900")
         arg1.free()
 
+    def test_errno(self):
+        import _rawffi
+        lib = _rawffi.CDLL(self.lib_name)
+        A = _rawffi.Array('i')
+        f = lib.ptr('check_errno', ['i'], 'i')
+        _rawffi.set_errno(42)
+        arg = A(1)
+        arg[0] = 43
+        res = f(arg)
+        assert res[0] == 42
+        z = _rawffi.get_errno()
+        assert z == 43
+        arg.free()
+
+    def test_last_error(self):
+        import sys
+        if sys.platform != 'win32':
+            skip("Windows test")
+        import _rawffi
+        lib = _rawffi.CDLL(self.lib_name)
+        A = _rawffi.Array('i')
+        f = lib.ptr('check_last_error', ['i'], 'i')
+        _rawffi.set_last_error(42)
+        arg = A(1)
+        arg[0] = 43
+        res = f(arg)
+        assert res[0] == 42
+        z = _rawffi.get_last_error()
+        assert z == 43
+        arg.free()
+
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -17,9 +17,13 @@
         self.w_value = w_value
 
 
+def needs_to_unwrap_cell(space, w_value):
+    return (space.config.objspace.std.withtypeversion and
+            isinstance(w_value, TypeCell))
+
+
 def unwrap_cell(space, w_value):
-    if (space.config.objspace.std.withtypeversion and
-            isinstance(w_value, TypeCell)):
+    if needs_to_unwrap_cell(space, w_value):
         return w_value.w_value
     return w_value
 
@@ -366,8 +370,11 @@
             tup = w_self._lookup_where(name)
             return tup
         name = promote_string(name)
-        w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
-        return w_class, unwrap_cell(space, w_value)
+        tup_w = w_self._pure_lookup_where_with_method_cache(name, version_tag)
+        w_class, w_value = tup_w
+        if needs_to_unwrap_cell(space, w_value):
+            return w_class, w_value.value
+        return tup_w   # don't make a new tuple, reuse the old one
 
     def _pure_lookup_where_possibly_with_method_cache(w_self, name, version_tag):
         if w_self.space.config.objspace.std.withmethodcache:
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -129,7 +129,8 @@
                default=False, cmdline=None),
 
     # misc
-    BoolOption("verbose", "Print extra information", default=False),
+    BoolOption("verbose", "Print extra information", default=False,
+               cmdline="--verbose"),
     StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"),
     StrOption("profopt", "Specify profile based optimization script",
               cmdline="--profopt"),
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -528,6 +528,10 @@
     pyfunc = staticmethod(getattr)
 
     def constfold(self):
+        from rpython.flowspace.flowcontext import FlowingError
+        if len(self.args) == 3:
+            raise FlowingError(
+                "getattr() with three arguments not supported: %s" % (self,))
         w_obj, w_name = self.args
         # handling special things like sys
         if (w_obj in NOT_REALLY_CONST and
@@ -538,7 +542,6 @@
             try:
                 result = getattr(obj, name)
             except Exception as e:
-                from rpython.flowspace.flowcontext import FlowingError
                 etype = e.__class__
                 msg = "getattr(%s, %s) always raises %s: %s" % (
                     obj, name, etype, e)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -964,9 +964,40 @@
         assembler_call = False
         if warmrunnerstate.inlining:
             if warmrunnerstate.can_inline_callable(greenboxes):
+                # We've found a potentially inlinable function; now we need to
+                # see if it's already on the stack. In other words: are we about
+                # to enter recursion? If so, we don't want to inline the
+                # recursion, which would be equivalent to unrolling a while
+                # loop.
                 portal_code = targetjitdriver_sd.mainjitcode
-                return self.metainterp.perform_call(portal_code, allboxes,
-                                                    greenkey=greenboxes)
+                count = 0
+                for f in self.metainterp.framestack:
+                    if f.jitcode is not portal_code:
+                        continue
+                    gk = f.greenkey
+                    if gk is None:
+                        continue
+                    assert len(gk) == len(greenboxes)
+                    i = 0
+                    for i in range(len(gk)):
+                        if not gk[i].same_constant(greenboxes[i]):
+                            break
+                    else:
+                        count += 1
+                memmgr = self.metainterp.staticdata.warmrunnerdesc.memory_manager
+                if count >= memmgr.max_unroll_recursion:
+                    # This function is recursive and has exceeded the
+                    # maximum number of unrollings we allow. We want to stop
+                    # inlining it further and to make sure that, if it
+                    # hasn't happened already, the function is traced
+                    # separately as soon as possible.
+                    if have_debug_prints():
+                        loc = targetjitdriver_sd.warmstate.get_location_str(greenboxes)
+                        debug_print("recursive function (not inlined):", loc)
+                    warmrunnerstate.dont_trace_here(greenboxes)
+                else:
+                    return self.metainterp.perform_call(portal_code, allboxes,
+                                greenkey=greenboxes)
             assembler_call = True
             # verify that we have all green args, needed to make sure
             # that assembler that we call is still correct
diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py
--- a/rpython/jit/metainterp/test/test_list.py
+++ b/rpython/jit/metainterp/test/test_list.py
@@ -97,28 +97,6 @@
         assert res == f(10)
         self.check_resops(setarrayitem_gc=0, call=0, getarrayitem_gc=0)
 
-    def test_vlist_alloc_and_set(self):
-        # the check_loops fails, because [non-null] * n is only supported
-        # if n < 15 (otherwise it is implemented as a residual call)
-        jitdriver = JitDriver(greens = [], reds = ['n'])
-        def f(n):
-            l = [1] * 20
-            while n > 0:
-                jitdriver.can_enter_jit(n=n)
-                jitdriver.jit_merge_point(n=n)
-                l = [1] * 20
-                l[3] = 5
-                x = l[-17] + l[5] - 1
-                if n < 3:
-                    return x
-                n -= 1
-            return l[0]
-
-        res = self.meta_interp(f, [10], listops=True)
-        assert res == f(10)
-        py.test.skip("'[non-null] * n' for n >= 15 gives a residual call so far")
-        self.check_loops(setarrayitem_gc=0, getarrayitem_gc=0, call=0)
-
     def test_arraycopy_simpleoptimize(self):
         def f():
             l = [1, 2, 3, 4]
diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -1112,6 +1112,37 @@
         assert res == 2095
         self.check_resops(call_assembler=12)
 
+    def test_inline_recursion_limit(self):
+        driver = JitDriver(greens = ["threshold", "loop"], reds=["i"])
+        @dont_look_inside
+        def f():
+            set_param(driver, "max_unroll_recursion", 10)
+        def portal(threshold, loop, i):
+            f()
+            if i > threshold:
+                return i
+            while True:
+                driver.jit_merge_point(threshold=threshold, loop=loop, i=i)
+                if loop:
+                    portal(threshold, False, 0)
+                else:
+                    portal(threshold, False, i + 1)
+                    return i
+                if i > 10:
+                    return 1
+                i += 1
+                driver.can_enter_jit(threshold=threshold, loop=loop, i=i)
+
+        res1 = portal(10, True, 0)
+        res2 = self.meta_interp(portal, [10, True, 0], inline=True)
+        assert res1 == res2
+        self.check_resops(call_assembler=2)
+
+        res1 = portal(9, True, 0)
+        res2 = self.meta_interp(portal, [9, True, 0], inline=True)
+        assert res1 == res2
+        self.check_resops(call_assembler=0)
+
     def test_handle_jitexception_in_portal(self):
         # a test for _handle_jitexception_in_portal in blackhole.py
         driver = JitDriver(greens = ['codeno'], reds = ['i', 'str'],
diff --git a/rpython/jit/metainterp/test/test_send.py b/rpython/jit/metainterp/test/test_send.py
--- a/rpython/jit/metainterp/test/test_send.py
+++ b/rpython/jit/metainterp/test/test_send.py
@@ -385,8 +385,7 @@
         self.check_target_token_count(4)
 
     def test_two_behaviors(self):
-        py.test.skip("XXX fix me!!!!!!! problem in optimize.py")
-        myjitdriver = JitDriver(greens = [], reds = ['x', 'y'])
+        myjitdriver = JitDriver(greens = [], reds = ['y', 'x'])
         class Int:
             def __init__(self, value):
                 self.value = value
@@ -402,11 +401,6 @@
             return x.value
         res = self.meta_interp(f, [len(cases)])
         assert res == 110
-        # The generated loops don't contain a new_with_vtable at all.  This
-        # is true if we replace "if cases[y]" above with "if not cases[y]"
-        # -- so there is no good reason that it fails.
-        self.check_loops(new_with_vtable=0)
-        self.check_trace_count(2)
 
     def test_behavior_change_after_a_while(self):
         myjitdriver = JitDriver(greens = [], reds = ['y', 'x'])
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -69,7 +69,8 @@
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
                     function_threshold=4,
-                    enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, **kwds):
+                    enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, 
+                    max_unroll_recursion=7, **kwds):
     from rpython.config.config import ConfigError
     translator = interp.typer.annotator.translator
     try:
@@ -91,6 +92,7 @@
         jd.warmstate.set_param_retrace_limit(retrace_limit)
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
+        jd.warmstate.set_param_max_unroll_recursion(max_unroll_recursion)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
         return interp, graph
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -291,6 +291,11 @@
             if self.warmrunnerdesc.memory_manager:
                 self.warmrunnerdesc.memory_manager.max_unroll_loops = value
 
+    def set_param_max_unroll_recursion(self, value):
+        if self.warmrunnerdesc:
+            if self.warmrunnerdesc.memory_manager:
+                self.warmrunnerdesc.memory_manager.max_unroll_recursion = value
+
     def disable_noninlinable_function(self, greenkey):
         cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
         cell.flags |= JC_DONT_TRACE_HERE
@@ -567,19 +572,26 @@
         jd = self.jitdriver_sd
         cpu = self.cpu
 
-        def can_inline_greenargs(*greenargs):
+        def can_inline_callable(greenkey):
+            greenargs = unwrap_greenkey(greenkey)
             if can_never_inline(*greenargs):
                 return False
             cell = JitCell.get_jitcell(*greenargs)
             if cell is not None and (cell.flags & JC_DONT_TRACE_HERE) != 0:
                 return False
             return True
-        def can_inline_callable(greenkey):
-            greenargs = unwrap_greenkey(greenkey)
-            return can_inline_greenargs(*greenargs)
-        self.can_inline_greenargs = can_inline_greenargs
         self.can_inline_callable = can_inline_callable
 
+        def dont_trace_here(greenkey):
+            # Set greenkey as somewhere that tracing should not occur into;
+            # notice that, as per the description of JC_DONT_TRACE_HERE earlier,
+            # if greenkey hasn't been traced separately, setting
+            # JC_DONT_TRACE_HERE will force tracing the next time the function
+            # is encountered.
+            cell = JitCell.ensure_jit_cell_at_key(greenkey)
+            cell.flags |= JC_DONT_TRACE_HERE
+        self.dont_trace_here = dont_trace_here
+
         if jd._should_unroll_one_iteration_ptr is None:
             def should_unroll_one_iteration(greenkey):
                 return False
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -463,6 +463,7 @@
     'max_unroll_loops': 'number of extra unrollings a loop can cause',
     'enable_opts': 'INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): '
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
+    'max_unroll_recursion': 'how many levels deep to unroll a recursive function'
     }
 
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
@@ -476,6 +477,7 @@
               'max_retrace_guards': 15,
               'max_unroll_loops': 0,
               'enable_opts': 'all',
+              'max_unroll_recursion': 7,
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
 
diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py
--- a/rpython/rtyper/lltypesystem/lltype.py
+++ b/rpython/rtyper/lltypesystem/lltype.py
@@ -182,8 +182,8 @@
     def _freeze_(self):
         return True
 
-    def _inline_is_varsize(self, last):
-        return False
+    def _note_inlined_into(self, parent, first, last):
+        """Called when this type is being used inline in a container."""
 
     def _is_atomic(self):
         return False
@@ -201,8 +201,9 @@
 class ContainerType(LowLevelType):
     _adtmeths = {}
 
-    def _inline_is_varsize(self, last):
-        raise TypeError("%r cannot be inlined in structure" % self)
+    def _note_inlined_into(self, parent, first, last):
+        raise TypeError("%r cannot be inlined in %r" % (
+            self.__class__.__name__, parent.__class__.__name__))
 
     def _install_extras(self, adtmeths={}, hints={}):
         self._adtmeths = frozendict(adtmeths)
@@ -279,11 +280,13 @@
 
         # look if we have an inlined variable-sized array as the last field
         if fields:
+            first = True
             for name, typ in fields[:-1]:
-                typ._inline_is_varsize(False)
+                typ._note_inlined_into(self, first=first, last=False)
                 first = False
             name, typ = fields[-1]
-            if typ._inline_is_varsize(True):
+            typ._note_inlined_into(self, first=first, last=True)
+            if typ._is_varsize():
                 self._arrayfld = name
         self._flds = frozendict(flds)
         self._names = tuple(names)
@@ -299,11 +302,14 @@
                 return first, FIRSTTYPE
         return None, None
 
-    def _inline_is_varsize(self, last):
-        if self._arrayfld:
+    def _note_inlined_into(self, parent, first, last):
+        if self._arrayfld is not None:
             raise TypeError("cannot inline a var-sized struct "
                             "inside another container")
-        return False
+        if self._gckind == 'gc':
+            if not first or not isinstance(parent, GcStruct):
+                raise TypeError("a GcStruct can only be inlined as the first "
+                                "field of another GcStruct")
 
     def _is_atomic(self):
         for typ in self._flds.values():
@@ -428,15 +434,18 @@
         if isinstance(self.OF, ContainerType) and self.OF._gckind != 'raw':
             raise TypeError("cannot have a %s container as array item type"
                             % (self.OF._gckind,))
-        self.OF._inline_is_varsize(False)
+        self.OF._note_inlined_into(self, first=False, last=False)
 
         self._install_extras(**kwds)
 
-    def _inline_is_varsize(self, last):
-        if not last:
+    def _note_inlined_into(self, parent, first, last):
+        if not last or not isinstance(parent, Struct):
             raise TypeError("cannot inline an array in another container"
                             " unless as the last field of a structure")
-        return True
+        if self._gckind == 'gc':
+            raise TypeError("cannot inline a GC array inside a structure")
+        if parent._gckind == 'gc' and self._hints.get('nolength', False):
+            raise TypeError("cannot inline a no-length array inside a GcStruct")
 
     def _is_atomic(self):
         return self.OF._is_atomic()
@@ -474,9 +483,6 @@
 
 class GcArray(Array):
     _gckind = 'gc'
-    def _inline_is_varsize(self, last):
-        raise TypeError("cannot inline a GC array inside a structure")
-
 
 class FixedSizeArray(Struct):
     # behaves more or less like a Struct with fields item0, item1, ...
@@ -508,7 +514,7 @@
         if isinstance(self.OF, ContainerType) and self.OF._gckind != 'raw':
             raise TypeError("cannot have a %s container as array item type"
                             % (self.OF._gckind,))
-        self.OF._inline_is_varsize(False)
+        self.OF._note_inlined_into(self, first=False, last=False)
 
     def _str_fields(self):
         return str(self.OF)
@@ -579,8 +585,11 @@
     def __str__(self):
         return "%s (opaque)" % self.tag
 
-    def _inline_is_varsize(self, last):
-        return False    # OpaqueType can be inlined
+    def _note_inlined_into(self, parent, first, last):
+        # OpaqueType can be inlined, but not GcOpaqueType
+        if self._gckind == 'gc':
+            raise TypeError("%r cannot be inlined in %r" % (
+                self.__class__.__name__, parent.__class__.__name__))
 
     def _container_example(self):
         return _opaque(self)
@@ -599,10 +608,6 @@
     def __str__(self):
         return "%s (gcopaque)" % self.tag
 
-    def _inline_is_varsize(self, last):
-        raise TypeError("%r cannot be inlined in structure" % self)
-
-
 class ForwardReference(ContainerType):
     _gckind = 'raw'
     def become(self, realcontainertype):
diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py
--- a/rpython/rtyper/lltypesystem/test/test_lltype.py
+++ b/rpython/rtyper/lltypesystem/test/test_lltype.py
@@ -808,6 +808,20 @@
     assert F.RESULT == Signed
     assert F.ARGS == (Signed,)
 
+def test_cannot_inline_random_stuff_in_gcstruct():
+    S = GcStruct('S')
+    GcStruct('X', ('a', S))    # works
+    py.test.raises(TypeError, GcStruct, 'X', ('a', Signed), ('b', S))
+    GcStruct('X', ('a', Array(Signed)))   # works
+    py.test.raises(TypeError, GcStruct, 'X', ('a', Array(Signed)),
+                                             ('b', Signed))
+    Struct('X', ('a', Array(Signed, hints={'nolength': True})))   # works
+    py.test.raises(TypeError, GcStruct, 'X',
+                   ('a', Array(Signed, hints={'nolength': True})))
+    GcStruct('X', ('a', OpaqueType('foo')))   # works
+    py.test.raises(TypeError, GcStruct, 'X', ('a', GcOpaqueType('foo')))
+
+
 class TestTrackAllocation:
     def test_automatic_tracking(self):
         # calls to start_tracking_allocations/stop_tracking_allocations
diff --git a/rpython/translator/backendopt/inline.py b/rpython/translator/backendopt/inline.py
--- a/rpython/translator/backendopt/inline.py
+++ b/rpython/translator/backendopt/inline.py
@@ -478,6 +478,7 @@
               'malloc': 2,
               'instrument_count': 0,
               'debug_assert': -1,
+              'jit_force_virtualizable': 0,
               }
 
 def block_weight(block, weights=OP_WEIGHTS):
diff --git a/rpython/translator/backendopt/storesink.py b/rpython/translator/backendopt/storesink.py
--- a/rpython/translator/backendopt/storesink.py
+++ b/rpython/translator/backendopt/storesink.py
@@ -5,7 +5,7 @@
 from rpython.translator import simplify
 
 def has_side_effects(op):
-    if op.opname == 'debug_assert':
+    if op.opname == 'debug_assert' or op.opname == 'jit_force_virtualizable':
         return False
     try:
         return getattr(llop, op.opname).sideeffects

From noreply at buildbot.pypy.org  Wed Dec 10 17:37:37 2014
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 10 Dec 2014 17:37:37 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: pypy_execute_frame_trampoline can
 (obviosuly) execute random python code,
 so we need to teach llexternal about it. Without,
 you get e.g. the wrong asmgcc labels, etc.
Message-ID: <20141210163737.10FEB1D2390@cobra.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: vmprof
Changeset: r74883:59ba61e4f315
Date: 2014-12-10 16:37 +0000
http://bitbucket.org/pypy/pypy/changeset/59ba61e4f315/

Log:	pypy_execute_frame_trampoline can (obviosuly) execute random python
	code, so we need to teach llexternal about it. Without, you get e.g.
	the wrong asmgcc labels, etc.

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -56,7 +56,8 @@
     [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF],
     llmemory.GCREF,
     compilation_info=eci,
-    _nowrapper=True, sandboxsafe=True)
+    _nowrapper=True, sandboxsafe=True,
+    random_effects_on_gcobjs=True)
 
 pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], lltype.Void, compilation_info=eci)
 vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.CCHARP, rffi.LONG], lltype.Void, compilation_info=eci)

From noreply at buildbot.pypy.org  Wed Dec 10 17:43:39 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Dec 2014 17:43:39 +0100 (CET)
Subject: [pypy-commit] pypy default: (cfbolz, tanzim, arigo)
Message-ID: <20141210164339.9A37E1C07BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74884:a3b346cf3412
Date: 2014-12-10 16:43 +0000
http://bitbucket.org/pypy/pypy/changeset/a3b346cf3412/

Log:	(cfbolz, tanzim, arigo)

	meh.

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -125,6 +125,8 @@
         else:
             return self.space.builtin
 
+    _NO_CELLS = []
+
     @jit.unroll_safe
     def initialize_frame_scopes(self, outer_func, code):
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
@@ -143,7 +145,7 @@
         nfreevars = len(code.co_freevars)
         if not nfreevars:
             if not ncellvars:
-                self.cells = []
+                self.cells = self._NO_CELLS
                 return            # no self.cells needed - fast path
         elif outer_func is None:
             space = self.space

From noreply at buildbot.pypy.org  Wed Dec 10 18:59:12 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Wed, 10 Dec 2014 18:59:12 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Krakatau takes a long time to
 warmup, so discard warmup iterations.
Message-ID: <20141210175912.772641D22EA@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r294:63916bb5a798
Date: 2014-12-10 17:48 +0000
http://bitbucket.org/pypy/benchmarks/changeset/63916bb5a798/

Log:	Krakatau takes a long time to warmup, so discard warmup iterations.

	When it is warmed up, it runs much faster than cold, so we then need
	to run things several times to make it long enough to be reliable.

diff --git a/own/bm_krakatau.py b/own/bm_krakatau.py
--- a/own/bm_krakatau.py
+++ b/own/bm_krakatau.py
@@ -47,19 +47,22 @@
                 source = javaclass.generateAST(c, makeGraph).print_()
 
 
+WARMUP_ITERATIONS = 30 # Krakatau needs a number of iterations to warmup...
+
 def main(n):
     l = []
     old_stdout = sys.stdout
     sys.stdout = cStringIO.StringIO()
     try:
-        for i in range(n):
+        for i in range(WARMUP_ITERATIONS + n):
             t0 = time.time()
-            decompileClass()
+            for j in range(4):
+                decompileClass()
             time_elapsed = time.time() - t0
             l.append(time_elapsed)
     finally:
         sys.stdout = old_stdout
-    return l
+    return l[WARMUP_ITERATIONS:]
 
 if __name__ == "__main__":
     parser = optparse.OptionParser(

From noreply at buildbot.pypy.org  Wed Dec 10 18:59:17 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Dec 2014 18:59:17 +0100 (CET)
Subject: [pypy-commit] pypy default: (cfbolz, arigo)
Message-ID: <20141210175917.643F81D22EA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74885:479bfad309e3
Date: 2014-12-10 17:59 +0000
http://bitbucket.org/pypy/pypy/changeset/479bfad309e3/

Log:	(cfbolz, arigo)

	no cookie

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -373,7 +373,7 @@
         tup_w = w_self._pure_lookup_where_with_method_cache(name, version_tag)
         w_class, w_value = tup_w
         if needs_to_unwrap_cell(space, w_value):
-            return w_class, w_value.value
+            return w_class, w_value.w_value
         return tup_w   # don't make a new tuple, reuse the old one
 
     def _pure_lookup_where_possibly_with_method_cache(w_self, name, version_tag):

From noreply at buildbot.pypy.org  Wed Dec 10 19:18:22 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 10 Dec 2014 19:18:22 +0100 (CET)
Subject: [pypy-commit] pypy default: Makes no sense to have f_lineno end up
 by complete chance as a 32-bit int...
Message-ID: <20141210181822.1D3791C3A66@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74886:bc40f87eddfc
Date: 2014-12-10 18:17 +0000
http://bitbucket.org/pypy/pypy/changeset/bc40f87eddfc/

Log:	Makes no sense to have f_lineno end up by complete chance as a
	32-bit int...

diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -58,7 +58,7 @@
     w_globals = from_ref(space, py_frame.c_f_globals)
 
     frame = space.FrameClass(space, code, w_globals, outer_func=None)
-    frame.f_lineno = py_frame.c_f_lineno
+    frame.f_lineno = rffi.getintfield(py_frame, 'c_f_lineno')
     w_obj = space.wrap(frame)
     track_reference(space, py_obj, w_obj)
     return w_obj

From noreply at buildbot.pypy.org  Thu Dec 11 12:22:24 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Dec 2014 12:22:24 +0100 (CET)
Subject: [pypy-commit] pypy default: Meh again.
Message-ID: <20141211112224.DC9CB1C02EE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74887:2cb2f52b915b
Date: 2014-12-11 11:22 +0000
http://bitbucket.org/pypy/pypy/changeset/2cb2f52b915b/

Log:	Meh again.

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -17,13 +17,9 @@
         self.w_value = w_value
 
 
-def needs_to_unwrap_cell(space, w_value):
-    return (space.config.objspace.std.withtypeversion and
-            isinstance(w_value, TypeCell))
-
-
 def unwrap_cell(space, w_value):
-    if needs_to_unwrap_cell(space, w_value):
+    if (space.config.objspace.std.withtypeversion and
+            isinstance(w_value, TypeCell)):
         return w_value.w_value
     return w_value
 
@@ -372,7 +368,8 @@
         name = promote_string(name)
         tup_w = w_self._pure_lookup_where_with_method_cache(name, version_tag)
         w_class, w_value = tup_w
-        if needs_to_unwrap_cell(space, w_value):
+        if (space.config.objspace.std.withtypeversion and
+                isinstance(w_value, TypeCell)):
             return w_class, w_value.w_value
         return tup_w   # don't make a new tuple, reuse the old one
 

From noreply at buildbot.pypy.org  Thu Dec 11 13:37:29 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Thu, 11 Dec 2014 13:37:29 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Stop Krakatau from using cache
	files.
Message-ID: <20141211123729.EA0FE1C3358@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r295:d1adfc92d840
Date: 2014-12-11 12:35 +0000
http://bitbucket.org/pypy/benchmarks/changeset/d1adfc92d840/

Log:	Stop Krakatau from using cache files.

diff --git a/lib/Krakatau/Krakatau/stdcache.py b/lib/Krakatau/Krakatau/stdcache.py
--- a/lib/Krakatau/Krakatau/stdcache.py
+++ b/lib/Krakatau/Krakatau/stdcache.py
@@ -6,11 +6,9 @@
         self.env = env
         self.filename = filename
 
-        try:
-            with open(self.filename, 'rb') as f:
-                fdata = f.read()
-        except IOError:
-            fdata = ''
+        # XXX for a benchmark, we don't ever want to use a cache, so we simply
+        # don't load data from a cache file (even if it exists)
+        fdata = ''
 
         #Note, we assume \n will never appear in a class name. This should be true for classes in the Java package,
         #but isn't necessarily true for user defined classes (Which we don't cache anyway)
@@ -46,4 +44,4 @@
         class_ = self.env.getClass(name, partial=True)
         if shouldCache(name):
             self._cache_info(class_)
-        return class_.flags
\ No newline at end of file
+        return class_.flags

From noreply at buildbot.pypy.org  Thu Dec 11 13:41:18 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Dec 2014 13:41:18 +0100 (CET)
Subject: [pypy-commit] creflect default: in-progress
Message-ID: <20141211124118.AFEBD1C30BE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r194:ee34a7bd891b
Date: 2014-12-11 11:37 +0000
http://bitbucket.org/cffi/creflect/changeset/ee34a7bd891b/

Log:	in-progress

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -411,18 +411,23 @@
 }
 
 static _crx_type_t *_zef_struct_or_union(_crx_builder_t *cb, const char *name,
-                                         int flag)
+                                         int flag, const char *prefix_name)
 {
     if (PyErr_Occurred())
         return NULL;
 
     PyObject *name_obj;
     CTypeDescrObject *ct;
+    size_t name_len = strlen(name);
+    size_t prefix_len = strlen(prefix_name);
 
-    name_obj = PyString_FromString(name);
+    name_obj = PyString_FromStringAndSize(NULL, name_len + prefix_len);
     if (name_obj == NULL)
         return NULL;
 
+    memcpy(PyString_AS_STRING(name_obj), prefix_name, prefix_len);
+    memcpy(PyString_AS_STRING(name_obj) + prefix_len, name, name_len);
+
     ct = get_cached_type(get_types_dict(cb), name_obj);
     if (ct && (ct->ct_flags == flag))
         goto done;
@@ -433,6 +438,9 @@
 
     ct->ct_flags = flag;
 
+    if (put_cached_type(get_types_dict(cb), name_obj, ct) < 0)
+        ct = NULL;
+
  done:
     Py_DECREF(name_obj);
     return ct;
@@ -440,12 +448,12 @@
 
 static _crx_type_t *zef_get_struct_type(_crx_builder_t *cb, const char *name)
 {
-    return _zef_struct_or_union(cb, name, CT_STRUCT);
+    return _zef_struct_or_union(cb, name, CT_STRUCT, "struct ");
 }
 
 static _crx_type_t *zef_get_union_type(_crx_builder_t *cb, const char *name)
 {
-    return _zef_struct_or_union(cb, name, CT_UNION);
+    return _zef_struct_or_union(cb, name, CT_UNION, "union ");
 }
 
 static _crx_type_t *zef_get_enum_type(_crx_builder_t *cb, const char *name)
diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -112,33 +112,24 @@
 static PyObject *ctypeget_kind(CTypeDescrObject *ct, void *context)
 {
     char *result;
-    if (ct->ct_flags & CT_PRIMITIVE_ANY) {
-        if (ct->ct_flags & CT_IS_ENUM)
-            result = "enum";
-        else
-            result = "primitive";
-    }
-    else if (ct->ct_flags & CT_POINTER) {
+    if (ct->ct_flags & CT_IS_ENUM)
+        result = "enum";
+    else if (ct->ct_flags & CT_PRIMITIVE_ANY)
+        result = "primitive";
+    else if (ct->ct_flags & CT_POINTER)
         result = "pointer";
-    }
-    else if (ct->ct_flags & CT_ARRAY) {
+    else if (ct->ct_flags & CT_ARRAY)
         result = "array";
-    }
-    else if (ct->ct_flags & CT_VOID) {
+    else if (ct->ct_flags & CT_VOID)
         result = "void";
-    }
-    else if (ct->ct_flags & CT_STRUCT) {
+    else if (ct->ct_flags & CT_STRUCT)
         result = "struct";
-    }
-    else if (ct->ct_flags & CT_UNION) {
+    else if (ct->ct_flags & CT_UNION)
         result = "union";
-    }
-    else if (ct->ct_flags & CT_FUNCTION) {
+    else if (ct->ct_flags & CT_FUNCTION)
         result = "function";
-    }
-    else if (ct->ct_flags & CT_UNKNOWN) {
+    else if (ct->ct_flags & CT_UNKNOWN)
         result = "opaque";
-    }
     else
         result = "?";
 
@@ -208,9 +199,15 @@
 static PyObject *ctypeget_elements(CTypeDescrObject *ct, void *context)
 {
     if (ct->ct_flags & CT_IS_ENUM) {
-        PyObject *res = PyTuple_GetItem(ct->ct_stuff, 1);
-        if (res) res = PyDict_Copy(res);
-        return res;
+        if (ct->ct_size >= 0) {
+            PyObject *res = PyTuple_GetItem(ct->ct_stuff, 1);
+            if (res) res = PyDict_Copy(res);
+            return res;
+        }
+        else {
+            Py_INCREF(Py_None);
+            return Py_None;
+        }
     }
     return nosuchattr("elements");
 }
@@ -218,9 +215,15 @@
 static PyObject *ctypeget_relements(CTypeDescrObject *ct, void *context)
 {
     if (ct->ct_flags & CT_IS_ENUM) {
-        PyObject *res = PyTuple_GetItem(ct->ct_stuff, 0);
-        if (res) res = PyDict_Copy(res);
-        return res;
+        if (ct->ct_size >= 0) {
+            PyObject *res = PyTuple_GetItem(ct->ct_stuff, 0);
+            if (res) res = PyDict_Copy(res);
+            return res;
+        }
+        else {
+            Py_INCREF(Py_None);
+            return Py_None;
+        }
     }
     return nosuchattr("relements");
 }
diff --git a/zeffir/test/struct.crx b/zeffir/test/struct.crx
--- a/zeffir/test/struct.crx
+++ b/zeffir/test/struct.crx
@@ -3,7 +3,7 @@
     long b;
 } mystruct_t;
 
-typedef struct {
+typedef struct myarrstr_s {
     int foo[5];
     int bar[7];
     int baz[];
@@ -16,7 +16,7 @@
     int a, b;
 } mystruct_t;
 
-typedef struct {
+typedef struct myarrstr_s {
     int foo[5];
     int bar[];
     int baz[];
diff --git a/zeffir/test/test_struct.py b/zeffir/test/test_struct.py
--- a/zeffir/test/test_struct.py
+++ b/zeffir/test/test_struct.py
@@ -11,6 +11,13 @@
     p.b += 1
     assert p.b == 0x600112234
 
+def test_types():
+    ffi, lib = support.compile_and_open('struct')
+    assert 'mystruct_t' in ffi.types
+    assert 'struct myarrstr_s' in ffi.types
+    assert 'myarrstr_t' in ffi.types
+    ffi.new("struct myarrstr_s")
+
 def test_offsetof():
     ffi, lib = support.compile_and_open('struct')
     assert ffi.offsetof("mystruct_t", "a") == 0

From noreply at buildbot.pypy.org  Thu Dec 11 13:41:19 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Dec 2014 13:41:19 +0100 (CET)
Subject: [pypy-commit] creflect default: Record which constants were
 declared as part of an enum
Message-ID: <20141211124119.D79A31C30BE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r195:9d9333df0699
Date: 2014-12-11 12:09 +0000
http://bitbucket.org/cffi/creflect/changeset/9d9333df0699/

Log:	Record which constants were declared as part of an enum

diff --git a/creflect/cparser.py b/creflect/cparser.py
--- a/creflect/cparser.py
+++ b/creflect/cparser.py
@@ -317,11 +317,13 @@
                                           % (tp,))
 
     def add_enum_values_declaration(self, typedecl, values):
+        enumerators = []
         for enumerator in values.enumerators:
+            enumerators.append(enumerator.name)
             self.declarations.append(
                 model.ConstDecl(enumerator.name,
                                 model.QualType(model.int_type)))
-        typedecl.complete = True
+        typedecl.enumerators = enumerators
 
     def parse_constant(self, constant):
         return int(constant.value)   # xxx
diff --git a/creflect/creflect.h b/creflect/creflect.h
--- a/creflect/creflect.h
+++ b/creflect/creflect.h
@@ -64,7 +64,8 @@
     _crx_type_t *(*get_unknown_type)(_CRX_SELF, const char *);
     void (*complete)(_CRX_SELF, _crx_type_t *,
                      size_t, size_t, _crx_field_t[], int);
-    void (*complete_enum)(_CRX_SELF, _crx_type_t *, _crx_type_t *);
+    void (*complete_enum)(_CRX_SELF, _crx_type_t *, _crx_type_t *,
+                          const char *[], int);
     void (*define_type)(_CRX_SELF, const char *, _crx_type_t *, int);
     void (*define_var)(_CRX_SELF, const char *, _crx_type_t *, int,
                        const void *);
diff --git a/creflect/creflect_debug_print.c b/creflect/creflect_debug_print.c
--- a/creflect/creflect_debug_print.c
+++ b/creflect/creflect_debug_print.c
@@ -200,10 +200,15 @@
 }
 
 static void tst_complete_enum(_crx_builder_t *cb, _crx_type_t *t,
-                              _crx_type_t *inttype)
+                              _crx_type_t *inttype,
+                              const char *enumvalues[], int nvalues)
 {
+    int i;
     assert(memcmp(t->text, "ENUM ", 5) == 0);
     printf("%s = %s\n", t->text, inttype->text);
+    for (i = 0; i < nvalues; i++) {
+        printf("| %s\n", enumvalues[i]);
+    }
 }
 
 static void tst_define_type(_crx_builder_t *cb, const char *name,
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -741,10 +741,10 @@
 
     def __init__(self, type):
         self.type = type
-        self.complete = False
+        self.enumerators = None
 
     def write_declaration(self, funcblock):
-        if not self.complete or self.type.name is None:
+        if self.enumerators is None or self.type.name is None:
             return     # opaque
         tp = "%s %s" % (self.type.kind, self.type.name)
         block = CodeBlock(funcblock)
@@ -767,7 +767,14 @@
         t1 = self.type.get_type_var(block)
         t2 = block.write_crx_type_var('_CRX_INT_TYPE(cb, %s, _crx_sc_int)' %
                                           (expr,))
-        block.writeline('cb->complete_enum(cb, %s, %s);' % (t1, t2))
+        if len(self.enumerators) == 0:
+            e_var = "0"
+        else:
+            block.writedecl("const char *e[] = { %s };" % (
+                ", ".join(['"%s"' % e for e in self.enumerators]),))
+            e_var = "e"
+        block.writeline('cb->complete_enum(cb, %s, %s, %s, %d);' % (
+            t1, t2, e_var, len(self.enumerators)))
         funcblock.write_subblock(block)
 
 
diff --git a/creflect/test/codegen/enum-003.c b/creflect/test/codegen/enum-003.c
--- a/creflect/test/codegen/enum-003.c
+++ b/creflect/test/codegen/enum-003.c
@@ -15,9 +15,11 @@
 #expect NUMCONST AA = int 0
     }
     {
+        const char *e[] = { "AA" };
         t2 = cb->get_enum_type(cb, "myname_e");
         t3 = _CRX_INT_TYPE(cb, (enum myname_e)-1, _crx_sc_int);
-        cb->complete_enum(cb, t2, t3);
+        cb->complete_enum(cb, t2, t3, e, 1);
 #expect ENUM myname_e = unsigned int
+#expect | AA
     }
 }
diff --git a/creflect/test/codegen/enum-003b.c b/creflect/test/codegen/enum-003b.c
--- a/creflect/test/codegen/enum-003b.c
+++ b/creflect/test/codegen/enum-003b.c
@@ -20,9 +20,12 @@
 #expect NUMCONST BB = int 3
     }
     {
+        const char *e[] = { "AA", "BB" };
         t3 = cb->get_enum_type(cb, "myname_e");
         t4 = _CRX_INT_TYPE(cb, (enum myname_e)-1, _crx_sc_int);
-        cb->complete_enum(cb, t3, t4);
+        cb->complete_enum(cb, t3, t4, e, 2);
 #expect ENUM myname_e = unsigned int
+#expect | AA
+#expect | BB
     }
 }
diff --git a/creflect/test/codegen/enum-003c.c b/creflect/test/codegen/enum-003c.c
--- a/creflect/test/codegen/enum-003c.c
+++ b/creflect/test/codegen/enum-003c.c
@@ -13,9 +13,11 @@
 #expect NUMCONST AA = int -3
     }
     {
+        const char *e[] = { "AA" };
         t2 = cb->get_enum_type(cb, "myname_e");
         t3 = _CRX_INT_TYPE(cb, (enum myname_e)-1, _crx_sc_int);
-        cb->complete_enum(cb, t2, t3);
+        cb->complete_enum(cb, t2, t3, e, 1);
 #expect ENUM myname_e = int
+#expect | AA
     }
 }
diff --git a/creflect/test/codegen/enum-004.c b/creflect/test/codegen/enum-004.c
--- a/creflect/test/codegen/enum-004.c
+++ b/creflect/test/codegen/enum-004.c
@@ -15,12 +15,14 @@
     {
         foo_t p1;
         char b[sizeof(*p1)];
+        const char *e[] = { "AA" };
         memset(b, -1, sizeof(b));
         p1 = (foo_t)b;
         t2 = cb->get_enum_type(cb, "$$foo_t");
         t3 = _CRX_INT_TYPE(cb, *p1, _crx_sc_int);
-        cb->complete_enum(cb, t2, t3);
+        cb->complete_enum(cb, t2, t3, e, 1);
 #expect ENUM $$foo_t = unsigned int
+#expect | AA
     }
     {
         foo_t *p1;
diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -410,8 +410,8 @@
     return _zef_array_type(cb, ctitem, (size_t)-1, NULL);
 }
 
-static _crx_type_t *_zef_struct_or_union(_crx_builder_t *cb, const char *name,
-                                         int flag, const char *prefix_name)
+static _crx_type_t *_zef_struct_union_enum(_crx_builder_t *cb, const char *name,
+                                           int flag, const char *prefix_name)
 {
     if (PyErr_Occurred())
         return NULL;
@@ -448,17 +448,17 @@
 
 static _crx_type_t *zef_get_struct_type(_crx_builder_t *cb, const char *name)
 {
-    return _zef_struct_or_union(cb, name, CT_STRUCT, "struct ");
+    return _zef_struct_union_enum(cb, name, CT_STRUCT, "struct ");
 }
 
 static _crx_type_t *zef_get_union_type(_crx_builder_t *cb, const char *name)
 {
-    return _zef_struct_or_union(cb, name, CT_UNION, "union ");
+    return _zef_struct_union_enum(cb, name, CT_UNION, "union ");
 }
 
 static _crx_type_t *zef_get_enum_type(_crx_builder_t *cb, const char *name)
 {
-    abort();
+    return _zef_struct_union_enum(cb, name, CT_UNKNOWN | CT_IS_ENUM, "enum ");
 }
 
 static _crx_qual_type zef_get_user_type(_crx_builder_t *cb, const char *name)
@@ -738,10 +738,22 @@
     Py_DECREF(interned_fields);
 }
 
-static void zef_complete_enum(_crx_builder_t *cb, _crx_type_t *t,
-                              _crx_type_t *inttype)
+static void zef_complete_enum(_crx_builder_t *cb, _crx_type_t *ct,
+                              _crx_type_t *inttype,
+                              const char *enumvalues[], int nvalues)
 {
-    abort();
+    assert(inttype->ct_flags & (CT_PRIMITIVE_SIGNED | CT_PRIMITIVE_UNSIGNED));
+    assert(ct->ct_flags & CT_IS_ENUM);
+    if (ct->ct_size >= 0) {
+        PyErr_Format(ZefError, "duplicate declaration of the enum '%s'",
+                     ct->ct_name);
+        return;
+    }
+
+    //...
+
+    ct->ct_flags = inttype->ct_flags | CT_IS_ENUM;
+    ct->ct_size = inttype->ct_size;
 }
 
 static void zef_define_type(_crx_builder_t *cb, const char *name,
diff --git a/zeffir/test/enum.crx b/zeffir/test/enum.crx
new file mode 100644
--- /dev/null
+++ b/zeffir/test/enum.crx
@@ -0,0 +1,8 @@
+enum foo_e { AA, BB, CC };
+
+
+// CREFLECT: start
+
+enum foo_e { AA, CC };
+
+// CREFLECT: end
diff --git a/zeffir/test/test_enum.py b/zeffir/test/test_enum.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_enum.py
@@ -0,0 +1,10 @@
+import py
+import support
+
+
+def test_simple_enum():
+    ffi, lib = support.compile_and_open('enum')
+    assert lib.AA == 0
+    assert lib.CC == 2
+    n = int(ffi.cast('enum foo_e', -5))
+    assert n == 2 ** 32 - 5     # unsigned

From noreply at buildbot.pypy.org  Thu Dec 11 13:41:20 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Dec 2014 13:41:20 +0100 (CET)
Subject: [pypy-commit] creflect default: Enums
Message-ID: <20141211124120.E0A151C30BE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r196:6009daa6e243
Date: 2014-12-11 12:41 +0000
http://bitbucket.org/cffi/creflect/changeset/6009daa6e243/

Log:	Enums

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -742,6 +742,10 @@
                               _crx_type_t *inttype,
                               const char *enumvalues[], int nvalues)
 {
+    PyObject *l_dict;
+    PyObject *d1, *d2;
+    int i;
+
     assert(inttype->ct_flags & (CT_PRIMITIVE_SIGNED | CT_PRIMITIVE_UNSIGNED));
     assert(ct->ct_flags & CT_IS_ENUM);
     if (ct->ct_size >= 0) {
@@ -750,10 +754,39 @@
         return;
     }
 
-    //...
+    d1 = PyDict_New();
+    if (d1 == NULL)
+        goto error;
+
+    d2 = PyDict_New();
+    if (d2 == NULL)
+        goto error;
+
+    l_dict = ((zeffir_builder_t *)cb)->l_dict;
+
+    for (i = 0; i < nvalues; i++) {
+        PyObject *x, *n = PyString_FromString(enumvalues[i]);
+        if (n == NULL)
+            goto error;
+
+        x = PyDict_GetItem(l_dict, n);
+        if (x == NULL || !(PyInt_Check(x) || PyLong_Check(x)))
+            continue;
+
+        if (PyDict_SetItem(d2, n, x) < 0 || PyDict_SetItem(d1, x, n) < 0)
+            goto error;
+    }
+
+    ct->ct_stuff = PyTuple_Pack(2, d1, d2);
+    if (ct->ct_stuff == NULL)
+        goto error;
 
     ct->ct_flags = inttype->ct_flags | CT_IS_ENUM;
     ct->ct_size = inttype->ct_size;
+
+ error:
+    Py_XDECREF(d1);
+    Py_XDECREF(d2);
 }
 
 static void zef_define_type(_crx_builder_t *cb, const char *name,
@@ -812,7 +845,7 @@
 
     assert(ct->ct_flags & CT_PRIMITIVE_ANY);
 
-    PyObject *x = PyInt_FromLong(value->as_int);
+    PyObject *x = convert_to_object((char *)value, ct);
     if (x == NULL)
         return;
 
diff --git a/zeffir/test/const.crx b/zeffir/test/const.crx
new file mode 100644
--- /dev/null
+++ b/zeffir/test/const.crx
@@ -0,0 +1,10 @@
+#define MY_LL_CONST   -0x01234567890abcdefLL
+#define MY_DBL_CONST  -1.23
+
+
+// CREFLECT: start
+
+#define MY_LL_CONST ...
+static const double MY_DBL_CONST;
+
+// CREFLECT: end
diff --git a/zeffir/test/test_const.py b/zeffir/test/test_const.py
new file mode 100644
--- /dev/null
+++ b/zeffir/test/test_const.py
@@ -0,0 +1,8 @@
+import py
+import support
+
+
+def test_constants():
+    ffi, lib = support.compile_and_open('const')
+    assert lib.MY_DBL_CONST == -1.23
+    assert lib.MY_LL_CONST == -0x01234567890abcdef
diff --git a/zeffir/test/test_enum.py b/zeffir/test/test_enum.py
--- a/zeffir/test/test_enum.py
+++ b/zeffir/test/test_enum.py
@@ -8,3 +8,9 @@
     assert lib.CC == 2
     n = int(ffi.cast('enum foo_e', -5))
     assert n == 2 ** 32 - 5     # unsigned
+
+def test_elements():
+    ffi, lib = support.compile_and_open('enum')
+    foo_e = ffi.typeof("enum foo_e")
+    assert foo_e.elements == {'AA': 0, 'CC': 2}
+    assert foo_e.relements == {0: 'AA', 2: 'CC'}

From noreply at buildbot.pypy.org  Thu Dec 11 15:15:17 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Dec 2014 15:15:17 +0100 (CET)
Subject: [pypy-commit] creflect default: Can't call a "..." function right
	now
Message-ID: <20141211141517.835A01C30BE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r197:d0a4621d2df2
Date: 2014-12-11 14:15 +0000
http://bitbucket.org/cffi/creflect/changeset/d0a4621d2df2/

Log:	Can't call a "..." function right now

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -805,6 +805,10 @@
     if (PyErr_Occurred())
         return;
 
+    if (type->ct_flags & CT_FUNCTION) {
+        type = zef_get_pointer_type(cb, type, quals);
+    }
+
     PyObject *l_dict = ((zeffir_builder_t *)cb)->l_dict;
 
     PyObject *x = make_global_var(type, addr);
diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -205,8 +205,11 @@
 
     zfs = (ZefFuncSupportObject *)ct->ct_stuff;
     if (zfs->zfs_trampl == NULL) {
-        PyErr_SetString(ZefError, "cdata '%s' cannot be called, because no "
-                        "wrapper was generated for functions of this type ");
+        const char *extra = (zfs->zfs_dotdotdot ?
+                             " (because signature ends in '...')" : "");
+        PyErr_Format(ZefError, "cdata '%s' cannot be called, because no "
+                     "wrapper was generated for functions of this type%s",
+                     ct->ct_name, extra);
         return NULL;
     }
 
diff --git a/zeffir/test/function.crx b/zeffir/test/function.crx
--- a/zeffir/test/function.crx
+++ b/zeffir/test/function.crx
@@ -18,11 +18,16 @@
     *p++ = 0;
 }
 
+void cant_call_dotdotdot(int x, ...)
+{
+}
+
 
 // CREFLECT: start
 
 int simple_function(int);
 int add_from_array(int array[], int count);
 void returning_nothing(char *);
+void cant_call_dotdotdot(int x, ...);
 
 // CREFLECT: end
diff --git a/zeffir/test/test_function.py b/zeffir/test/test_function.py
--- a/zeffir/test/test_function.py
+++ b/zeffir/test/test_function.py
@@ -40,3 +40,12 @@
     assert str(e.value) == ("typeof(lib.func) not supported: the "
                             "exact type of functions is unknown (declare that "
                             "function as a function pointer instead)")
+
+def test_cant_call_dotdotdot():
+    ffi, lib = support.compile_and_open('function')
+    p = lib.cant_call_dotdotdot
+    assert ffi.typeof(p) == ffi.typeof("void(*)(int, ...)")
+    e = py.test.raises(ffi.error, p, 5)
+    assert str(e.value) == ("cdata 'void(int, ...)' cannot be called, because "
+                            "no wrapper was generated for functions of this "
+                            "type (because signature ends in '...')")

From noreply at buildbot.pypy.org  Thu Dec 11 15:35:37 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Dec 2014 15:35:37 +0100 (CET)
Subject: [pypy-commit] pypy default: Yet another attempt to fix this test
Message-ID: <20141211143537.234521D27FB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74888:3a8d1013feda
Date: 2014-12-11 14:26 +0000
http://bitbucket.org/pypy/pypy/changeset/3a8d1013feda/

Log:	Yet another attempt to fix this test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -342,9 +342,9 @@
         guard_false(i114, descr=...)
         --TICK--
         i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=4 OS=110>)
-        raw_store(i119, 0, i112, descr=<ArrayS 2>)
-        raw_store(i119, 2, i112, descr=<ArrayS 2>)
-        raw_store(i119, 4, i112, descr=<ArrayS 2>)
+        raw_store(i119, 0, i160, descr=<ArrayS 2>)
+        raw_store(i119, 2, i160, descr=<ArrayS 2>)
+        raw_store(i119, 4, i160, descr=<ArrayS 2>)
         setfield_gc(p167, i119, descr=<FieldU pypy.module._cffi_backend.cdataobj.W_CData.inst__cdata .+>)
         i123 = arraylen_gc(p67, descr=<ArrayP .>)
         jump(..., descr=...)

From noreply at buildbot.pypy.org  Thu Dec 11 15:35:38 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Dec 2014 15:35:38 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix model.py for test__cffi_call
Message-ID: <20141211143538.585081D27FB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74889:cd2dc9e91d09
Date: 2014-12-11 14:35 +0000
http://bitbucket.org/pypy/pypy/changeset/cd2dc9e91d09/

Log:	Fix model.py for test__cffi_call

diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -415,9 +415,12 @@
         self.match_descr(op.descr, exp_descr)
 
 
-    def _next_op(self, iter_ops, assert_raises=False):
+    def _next_op(self, iter_ops, assert_raises=False, ignore_ops=set()):
         try:
-            op = iter_ops.next()
+            while True:
+                op = iter_ops.next()
+                if op.name not in ignore_ops:
+                    break
         except StopIteration:
             self._assert(assert_raises, "not enough operations")
             return
@@ -454,9 +457,7 @@
         else:
             assert 0, "'{{{' not followed by '}}}'"
         while exp_ops:
-            op = self._next_op(iter_ops)
-            if op.name in ignore_ops:
-                continue
+            op = self._next_op(iter_ops, ignore_ops=ignore_ops)
             # match 'op' against any of the exp_ops; the first successful
             # match is kept, and the exp_op gets removed from the list
             for i, exp_op in enumerate(exp_ops):
@@ -491,10 +492,7 @@
                     self.match_any_order(iter_exp_ops, iter_ops, ignore_ops)
                     continue
                 else:
-                    while True:
-                        op = self._next_op(iter_ops)
-                        if op.name not in ignore_ops:
-                            break
+                    op = self._next_op(iter_ops, ignore_ops=ignore_ops)
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
                 if type(exp_op) is not str and exp_op[4] is False:    # optional operation
@@ -504,7 +502,7 @@
                 raise
         #
         # make sure we exhausted iter_ops
-        self._next_op(iter_ops, assert_raises=True)
+        self._next_op(iter_ops, assert_raises=True, ignore_ops=ignore_ops)
 
     def match(self, expected_src, ignore_ops=[]):
         def format(src, opindex=None):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -318,6 +318,16 @@
             jump(i4, descr=...)
         """
         assert self.match(loop, expected, ignore_ops=['force_token'])
+        #
+        loop = """
+            [i0]
+            i1 = int_add(i0, 1)
+            i4 = force_token()
+        """
+        expected = """
+            i1 = int_add(i0, 1)
+        """
+        assert self.match(loop, expected, ignore_ops=['force_token'])
 
     def test_match_dots_in_arguments(self):
         loop = """

From noreply at buildbot.pypy.org  Thu Dec 11 15:40:42 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Dec 2014 15:40:42 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Kill that old version of Krakatau
 to de-confuse ourselves
Message-ID: <20141211144042.BDC081C02EE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r296:0e62cdf642ae
Date: 2014-12-11 14:40 +0000
http://bitbucket.org/pypy/benchmarks/changeset/0e62cdf642ae/

Log:	Kill that old version of Krakatau to de-confuse ourselves

diff too long, truncating to 2000 out of 14355 lines

diff --git a/lib/Krakatau/Documentation/assembler.txt b/lib/Krakatau/Documentation/assembler.txt
deleted file mode 100644
--- a/lib/Krakatau/Documentation/assembler.txt
+++ /dev/null
@@ -1,88 +0,0 @@
-Krakatau Assembler Syntax
-
-This guide is intended to help write bytecode assembly files for use with the Krakatau assembler. It assumes that you are already familiar with the JVM classfile format and how to write bytecode. If not, you can find a simple tutorial to writing bytecode at https://greyhat.gatech.edu/wiki/index.php?title=Java_Bytecode_Tutorial. You can also find some examples of assembler files in the examples directory.
-
-Krakatau syntax is largely backwards compatible with the classic Jasmin assembler syntax. In a couple of places, backwards compatibility is broken either by the introduction of new keywords or to fix ambiguities in the Jasmin syntax. However, Krakatau is not necessarily compatible with the extensions introduced by JasminXT.
-
-The basic format for an assembler file consists of a list of classfile entries. Each entry will result in the generation of a seperate classfile, so a single assembly file can contain multiple classes where convienent. These entries are completely independent - mutiple classes never share constant pool entries, fields, methods, or directives, even the version directive. Each one has the format
-
-.bytecode major minor  (optional)
-class directives
-.class classref
-.super classref
-interface declarations
-class directives
-topitems
-.end class
-
-The .end class on the final entry may be ommitted. So the simplest possible assembler file to declare a class named Foo would be
-
-.class Foo
-.super java/lang/Object
-
-To declare three classes A, B, and C in the same file with B and C inheriting from A and different versions, you could do
-
-.class A
-.super java/lang/Object
-.end class
-.class B
-.super A
-.end class
-.class C
-.super A
-
-The classfile version is specified by the .bytecode directive. It is specified by major, minor, a pair of decimal integers. If ommitted, the default is version 49.0. So the following is equivalent to the earlier example
-
-.bytecode 49 0
-.class Foo
-.super java/lang/Object
-
-Other class directives include .runtimevisible, .runtimeinvisible, .signature, .attribute, .source, .inner, .innerlength, and .enclosing. These are used to control the attributes of the class and will be covered later.
-
-Topitems are the actual meat of the class. There are three types: fields, methods, and constant definitions. The last is unique to Krakatau and is closely related to the rest of the syntax. In Krakatau, there are multiple ways to specify a constant pool entry. The most common are via WORD tokens, symbolic references and numerical references. The later constist of square brackets with lowercase alphanumerics and underscores inside.
-
-When you specify .class Foo, the string Foo isn't directly included in the output. The classfile format says that the class field is actually a two byte index into the constant pool of the classfile. This points to a Class_info which points to a Utf8_info which holds the actual name of the class. Therefore, Krakatau implicitly creates constant pool entries and inserts the appropriate references. But this process can be controlled more directly.
-
-Instead of writing
-.class Foo
-.super java/lang/Object
-
-You could explicitly write out all the classfile references as follows
-
-.class [foocls]
-.super [objcls]
-
-.const [foocls] = Class [fooutf]
-.const [fooutf] = Utf8 Foo
-.const [objcls] = Class [objutf]
-.const [objutf] = Utf8 java/lang/Object
-
-There are two types of references. If the contents are a decimal int, then it is a direct numerical reference to a particular slot in the constant pool. You are responsible for making sure that everything is consistent and that the contents of that slot are valid. This option is most useful for specifiying the null entry [0]. For example, to express the Object class itself, one would do
-
-.class java/lang/Object
-.super [0]
-
-If the contents are any other nonempty lowercase alphanumeric + underscores string, it is interperted as a symbolic reference. This is a reference to some slot but you don't care which one. Krakatau will pick an available slot and fill it in automatically. Symbolic references may be ommitted from the generated classfile if unused or merged with identical entries, including automatically generated entries.
-
-With that out of the way, the basic form of a constant definition is
-
-.const ref = entrytype arguments
-
-You can also just define one reference in terms of another. You are responsible for making sure there are no circular references or duplicated definitions. If not, unpredictable results may occur.
-
-Examples include
-
-.const [1] = Class Foo
-.const [1] = Class [fooutf]
-.const [89] = Long 1234567L
-.const [mynat] = NameAndType main ([Ljava/lang/String;)V
-.const [myref] = [myotherref]
-.const [really42] = [42]
-
-
-
-
-
-
-
-
diff --git a/lib/Krakatau/Krakatau/__init__.py b/lib/Krakatau/Krakatau/__init__.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-'''Monkeypatch a fix for bugs.python.org/issue9825 in case users are running an old version.'''
-import collections
-try:
-    del collections.OrderedDict.__del__
-except AttributeError:
-    pass
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/__init__.py b/lib/Krakatau/Krakatau/assembler/__init__.py
deleted file mode 100644
diff --git a/lib/Krakatau/Krakatau/assembler/assembler.py b/lib/Krakatau/Krakatau/assembler/assembler.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/assembler.py
+++ /dev/null
@@ -1,580 +0,0 @@
-import collections
-import struct, operator
-
-from . import instructions, codes
-from .. import constant_pool
-from ..classfile import ClassFile
-from ..method import Method
-from ..field import Field
-
-class AssemblerError(Exception):
-    def __init__(self, message, data=None):
-        super(AssemblerError, self).__init__(message)
-        self.data = data
-
-def error(msg):
-    raise AssemblerError(msg)
-
-class PoolRef(object):
-    def __init__(self, *args, **kwargs):
-        self.index = kwargs.get('index')
-        self.lbl = kwargs.get('lbl')
-        self.args = args
-
-    def toIndex(self, pool, forbidden=(), **kwargs):
-        if self.index is not None:
-            return self.index
-        if self.lbl:
-            self.index = pool.getLabel(self.lbl, forbidden, **kwargs)
-        else:
-            self.args = [(x.toIndex(pool) if isinstance(x, PoolRef) else x) for x in self.args]
-            self.index = pool.getItem(*self.args, **kwargs)
-        return self.index
-
-class PoolInfo(object):
-    def __init__(self):
-        self.pool = constant_pool.ConstPool()
-        self.lbls = {}
-        self.fixed = {} # constant pool entries in a specific slot
-        self.bootstrap = [] #entries for the BootstrapMethods attribute if any
-
-    def getLabel(self, lbl, forbidden=(), **kwargs):
-        if lbl in forbidden:
-            error('Circular constant pool reference: ' + ', '.join(forbidden))
-        forbidden = forbidden + (lbl,)
-        return self.lbls[lbl].toIndex(self, forbidden, **kwargs)
-
-    def getItem(self, type_, *args, **kwargs):
-        if type_ == 'InvokeDynamic':
-            self.bootstrap.append(args[:-1])
-            args = len(self.bootstrap)-1, args[-1]    
-        return self.pool.addItem((type_, tuple(args)), **kwargs)
-
-    def Utf8(self, s):
-        return self.getItem('Utf8', s)
-
-    def assignFixedSlots(self):
-        self.pool.reserved.update(self.fixed)
-        for i,v in self.fixed.items():
-            if v.args and v.args[0] in ('Double','Long'):
-                self.pool.reserved.add(i+1)
-                
-        #TODO - order these in terms of dependencies?
-        for index, value in self.fixed.items():
-            used = value.toIndex(self, index=index)
-            if used != index: #we need to copy an existing item
-                self.pool.copyItem(used, index)
-
-_format_ops = collections.defaultdict(tuple)
-_format_ops[''] = instructions.instrs_noarg
-_format_ops['>B'] = 'iload', 'lload', 'fload', 'dload', 'aload', 'istore', 'lstore', 'fstore', 'dstore', 'astore', 'ret'
-_format_ops['>h'] = 'ifeq', 'ifne', 'iflt', 'ifge', 'ifgt', 'ifle', 'if_icmpeq', 'if_icmpne', 'if_icmplt', 'if_icmpge', 'if_icmpgt', 'if_icmple', 'if_acmpeq', 'if_acmpne', 'goto', 'jsr', 'ifnull', 'ifnonnull'
-_format_ops['>H'] = 'ldc_w', 'ldc2_w', 'getstatic', 'putstatic', 'getfield', 'putfield', 'invokevirtual', 'invokespecial', 'invokestatic', 'new', 'anewarray', 'checkcast', 'instanceof'
-
-_format_ops['>b'] += 'bipush', 
-_format_ops['>Bb'] += 'iinc', 
-_format_ops['>h'] += 'sipush', 
-_format_ops['>HB'] += 'multianewarray',
-_format_ops['>HBB'] += 'invokeinterface',
-_format_ops['>HH'] += 'invokedynamic',
-_format_ops['>B'] += 'ldc', 'newarray'
-_format_ops['>i'] += 'goto_w', 'jsr_w'
-
-op_structs = {}
-for fmt, ops in _format_ops.items():
-    _s = struct.Struct(fmt)
-    for _op in ops:
-        op_structs[_op] = _s
-
-def getPadding(pos):
-    return (3-pos) % 4
-
-def getInstrLen(instr, pos):
-    op = instr[0]
-    if op in op_structs:
-        return 1 + op_structs[op].size
-    elif op == 'wide':
-        return 2 + 2 * len(instr[1][1])
-    else:
-        padding = getPadding(pos)
-        count = len(instr[1][1])
-        if op == 'tableswitch':
-            return 13 + padding + 4*count
-        else:
-            return 9 + padding + 8*count 
-
-def assembleInstruction(instr, labels, pos, pool):
-    def lbl2Off(lbl):
-        if lbl not in labels:
-            del labels[None]
-            error('Undefined label: {}\nDefined labels for current method are: {}'.format(lbl, ', '.join(sorted(labels))))
-        return labels[lbl] - pos
-
-    op = instr[0]
-    first = chr(instructions.allinstructions.index(op))
-
-    instr = [(x.toIndex(pool) if isinstance(x, PoolRef) else x) for x in instr[1:]]
-    if op in instructions.instrs_lbl:
-        instr[0] = lbl2Off(instr[0])
-    if op in op_structs:
-        rest = op_structs[op].pack(*instr)
-        return first+rest
-    elif op == 'wide':
-        subop, args = instr[0]
-        prefix = chr(instructions.allinstructions.index(subop))
-        fmt = '>Hh' if len(args) > 1 else '>H'
-        rest = struct.pack(fmt, *args)
-        return first + prefix + rest
-    else:
-        padding = getPadding(pos)
-        param, jumps, default = instr[0]
-        default = lbl2Off(default)
-
-        if op == 'tableswitch':
-            jumps = map(lbl2Off, jumps)
-            low, high = param, param + len(jumps)-1
-            temp = struct.Struct('>i')
-            part1 = first + '\0'*padding + struct.pack('>iii', default, low, high)
-            return part1 + ''.join(map(temp.pack, jumps))
-        elif op == 'lookupswitch':
-            jumps = {k:lbl2Off(lbl) for k,lbl in jumps}
-            jumps = sorted(jumps.items())
-            temp = struct.Struct('>ii')
-            part1 = first + '\0'*padding + struct.pack('>ii', default, len(jumps))
-            part2 = ''.join(map(temp.pack, *zip(*jumps))) if jumps else ''
-            return part1 + part2
-
-def groupList(pairs):
-    d = collections.defaultdict(list)
-    for k,v in pairs:
-        d[k].append(v)
-    return d
-
-def splitList(pairs):
-    d = groupList(pairs)
-    return d[False], d[True]
-       
-def assembleCodeAttr(statements, pool, version, addLineNumbers, jasmode):
-    directives, lines = splitList(statements)
-    dir_offsets = collections.defaultdict(list)
-
-    offsets = []
-    labels = {}
-    pos = 0
-    #first run through to calculate bytecode offsets
-    #this is greatly complicated due to the need to
-    #handle Jasmine line number directives
-    for t, statement in statements:
-        if t:
-            lbl, instr = statement
-            labels[lbl] = pos
-            if instr is not None:
-                offsets.append(pos)
-                pos += getInstrLen(instr, pos)
-        #some directives require us to keep track of the corresponding bytecode offset
-        elif statement[0] in ('.line','.stackmap'):
-            dir_offsets[statement[0]].append(pos)
-    code_len = pos
-
-    code_bytes = ''
-    for lbl, instr in lines:
-        if instr is not None:
-            code_bytes += assembleInstruction(instr, labels, len(code_bytes), pool)
-    assert(len(code_bytes) == code_len)
-
-    directive_dict = groupList(directives)
-    limits = groupList(directive_dict['.limit'])
-
-    stack = min(limits['stack'] + [65535]) 
-    locals_ = min(limits['locals'] + [65535]) 
-
-    excepts = []
-    for name, start, end, target in directive_dict['.catch']:
-        #Hack for compatibility with Jasmin
-        if jasmode and name.args and (name.args[1].args == ('Utf8','all')):
-            name.index = 0
-        vals = labels[start], labels[end], labels[target], name.toIndex(pool)
-        excepts.append(struct.pack('>HHHH',*vals))
-    
-    attributes = []
-
-    #StackMapTable
-    def pack_vt(vt):
-        s = chr(codes.vt_codes[vt[0]])
-        if vt[0] == 'Object':
-            s += struct.pack('>H', vt[1].toIndex(pool))        
-        elif vt[0] == 'Uninitialized':
-            s += struct.pack('>H', labels[vt[1]])
-        return s
-
-    if directive_dict['.stackmap']:
-        frames = []
-        last_pos = -1
-
-        for pos, info in zip(dir_offsets['.stackmap'], directive_dict['.stackmap']):
-            offset = pos - last_pos - 1
-            last_pos = pos
-            assert(offset >= 0)
-
-            tag = info[0]
-            if tag == 'same':
-                if offset >= 64:
-                    error('Max offset on a same frame is 63.')
-                frames.append(chr(offset))            
-            elif tag == 'same_locals_1_stack_item':
-                if offset >= 64:
-                    error('Max offset on a same_locals_1_stack_item frame is 63.')
-                frames.append(chr(64 + offset) + pack_vt(info[2][0]))            
-            elif tag == 'same_locals_1_stack_item_extended':
-                frames.append(struct.pack('>BH', 247, offset) + pack_vt(info[2][0]))            
-            elif tag == 'chop':
-                if not (1 <= info[1] <= 3):
-                    error('Chop frame can only remove 1-3 locals')
-                frames.append(struct.pack('>BH', 251-info[1], offset))
-            elif tag == 'same_extended':
-                frames.append(struct.pack('>BH', 251, offset))
-            elif tag == 'append':
-                local_vts = map(pack_vt, info[2])
-                if not (1 <= len(local_vts) <= 3):
-                    error('Append frame can only add 1-3 locals')
-                frames.append(struct.pack('>BH', 251+len(local_vts), offset) + ''.join(local_vts))
-            elif tag == 'full':
-                local_vts = map(pack_vt, info[2])
-                stack_vts = map(pack_vt, info[3])
-                frame = struct.pack('>BH', 255, offset)
-                frame += struct.pack('>H', len(local_vts)) + ''.join(local_vts)
-                frame += struct.pack('>H', len(stack_vts)) + ''.join(stack_vts)
-                frames.append(frame)
-
-        sm_body = ''.join(frames)
-        sm_attr = struct.pack('>HIH', pool.Utf8("StackMapTable"), len(sm_body)+2, len(frames)) + sm_body
-        attributes.append(sm_attr)
-
-    #line number attribute
-    if addLineNumbers and not directive_dict['line']:
-        dir_offsets['line'] = directive_dict['line'] = offsets
-    if directive_dict['line']:
-        lntable = [struct.pack('>HH',x,y) for x,y in zip(dir_offsets['line'], directive_dict['line'])]
-        ln_attr = struct.pack('>HIH', pool.Utf8("LineNumberTable"), 2+4*len(lntable), len(lntable)) + ''.join(lntable)        
-        attributes.append(ln_attr)
-
-    if directive_dict['.var']:
-        sfunc = struct.Struct('>HHHHH').pack
-        vartable = []
-        for index, name, desc, start, end in directive_dict['.var']:
-            start, end = labels[start], labels[end]
-            name, desc = name.toIndex(pool), desc.toIndex(pool)
-            vartable.append(sfunc(start, end-start, name, desc, index))
-        var_attr = struct.pack('>HIH', pool.Utf8("LocalVariableTable"), 2+10*len(vartable), len(vartable)) + ''.join(vartable)        
-        attributes.append(var_attr)
-
-    if not code_len:
-        return None
-
-    for attrname, data in directive_dict['.codeattribute']:
-        attr = struct.pack('>HI', attrname.toIndex(pool), len(data)) + data
-        attributes.append(attr)        
-
-
-    #Old versions use shorter fields for stack, locals, and code length
-    header_fmt = '>HHI' if version > (45,2) else '>BBH'
-
-    name_ind = pool.Utf8("Code")
-    attr_len = struct.calcsize(header_fmt) + 4 + len(code_bytes) + 8*len(excepts) + sum(map(len, attributes))
-    
-    assembled_bytes = struct.pack('>HI', name_ind, attr_len)
-    assembled_bytes += struct.pack(header_fmt, stack, locals_, len(code_bytes))
-    assembled_bytes += code_bytes
-    assembled_bytes += struct.pack('>H', len(excepts)) + ''.join(excepts)
-    assembled_bytes += struct.pack('>H', len(attributes)) + ''.join(attributes)
-    return assembled_bytes
-
-def _assembleEVorAnnotationSub(pool, init_args, isAnnot):
-    #call types
-    C_ANNOT, C_ANNOT2, C_EV = range(3)
-    init_callt = C_ANNOT if isAnnot else C_EV
-
-    stack = [(init_callt, init_args)]
-    parts = []
-    add = parts.append
-
-    while stack:
-        callt, args = stack.pop()
-
-        if callt == C_ANNOT:
-            typeref, keylines = args
-            add(struct.pack('>HH', typeref.toIndex(pool), len(keylines)))
-            for pair in reversed(keylines):
-                stack.append((C_ANNOT2, pair))
-
-        elif callt == C_ANNOT2:
-            name, val = args
-            add(struct.pack('>H', name.toIndex(pool)))
-            stack.append((C_EV, val))
-
-        elif callt == C_EV:
-            tag, data = args
-            assert(tag in codes.et_rtags)
-            add(tag)
-
-            if tag in 'BCDFIJSZsc':
-                add(struct.pack('>H', data[0].toIndex(pool)))
-            elif tag == 'e':
-                add(struct.pack('>HH', data[0].toIndex(pool), data[1].toIndex(pool)))
-            elif tag == '@':
-                stack.append((C_ANNOT, data[0]))
-            elif tag == '[':
-                add(struct.pack('>H', len(data[1])))
-                for arrval in reversed(data[1]):
-                    stack.append((C_EV, arrval))
-    return ''.join(parts)
-
-def assembleElementValue(val, pool):
-    return  _assembleEVorAnnotationSub(pool, val, False)
-
-def assembleAnnotation(annotation, pool):
-    return  _assembleEVorAnnotationSub(pool, annotation, True)
-
-def assembleMethod(header, statements, pool, version, addLineNumbers, jasmode):
-    mflags, (name, desc) = header
-    name = name.toIndex(pool)
-    desc = desc.toIndex(pool)
-
-    flagbits = map(Method.flagVals.get, mflags)
-    flagbits = reduce(operator.__or__, flagbits, 0)
-
-    meth_statements, code_statements = splitList(statements)
-
-    method_attributes = []
-    code_attr = assembleCodeAttr(code_statements, pool, version, addLineNumbers, jasmode)
-    if code_attr is not None:
-        method_attributes.append(code_attr)
-
-    directive_dict = groupList(meth_statements)
-    if directive_dict['.throws']:
-        t_inds = [struct.pack('>H', x.toIndex(pool)) for x in directive_dict['.throws']]
-        throw_attr = struct.pack('>HIH', pool.Utf8("Exceptions"), 2+2*len(t_inds), len(t_inds)) + ''.join(t_inds)        
-        method_attributes.append(throw_attr)
-
-    #Runtime annotations
-    for vis in ('Invisible','Visible'):
-        paramd = groupList(directive_dict['.runtime'+vis.lower()])
-
-        if None in paramd:
-            del paramd[None]
-
-        if paramd:
-            parts = []
-            for i in range(max(paramd)):
-                annotations = [assembleAnnotation(a, pool) for a in paramd[i]]
-                part = struct.pack('>H', len(annotations)) + ''.join(annotations)
-                parts.append(part)
-            attrlen = 1+sum(map(len, parts))
-            attr = struct.pack('>HIB', pool.Utf8("Runtime{}ParameterAnnotations".format(vis)), attrlen, len(parts)) + ''.join(parts)
-            method_attributes.append(attr)
-
-    if '.annotationdefault' in directive_dict:
-        val = directive_dict['.annotationdefault'][0]
-        data = assembleElementValue(val, pool)
-        attr = struct.pack('>HI', pool.Utf8("AnnotationDefault"), len(data)) + data        
-        method_attributes.append(attr)
-
-    assembleClassFieldMethodAttributes(method_attributes.append, directive_dict, pool)
-    return struct.pack('>HHHH', flagbits, name, desc, len(method_attributes)) + ''.join(method_attributes)
-
-def getLdcRefs(statements):
-    lines = [x[1][1] for x in statements if x[0] and x[1][0]]
-    instructions = [x[1] for x in lines if x[1] is not None]
-
-    for instr in instructions:
-        op = instr[0]
-        if op == 'ldc':
-            yield instr[1]
- 
-def addLdcRefs(methods, pool):
-    def getRealRef(ref, forbidden=()):
-        '''Get the root PoolRef associated with a given PoolRef, following labels'''
-        if ref.index is None and ref.lbl:
-            if ref.lbl in forbidden:
-                error('Circular constant pool reference: ' + ', '.join(forbidden))
-            forbidden = forbidden + (ref.lbl,)
-            return getRealRef(pool.lbls[ref.lbl], forbidden) #recursive call
-        return ref
-
-    #We attempt to estimate how many slots are needed after merging identical entries
-    #So we can reserve the correct number of slots without leaving unused gaps
-    #However, in complex cases, such as string/class/mt referring to an explicit
-    #reference, we may overestimate
-    ldc_refs = collections.defaultdict(set)
-
-    for header, statements in methods:
-        for ref in getLdcRefs(statements):
-            ref = getRealRef(ref)
-            if ref.index is not None:
-                continue
-
-            type_ = ref.args[0]
-            if type_ in ('Int','Float'):
-                key = ref.args[1]
-            elif type_ in ('String','Class','MethodType'): 
-                uref = getRealRef(ref.args[1])
-                key = uref.index, uref.args[1:]
-            else: #for MethodHandles, don't even bother trying to estimate merging
-                key = ref.args[1:] 
-            ldc_refs[type_].add(key)    
-
-    #TODO - make this a little cleaner so we don't have to mess with the ConstantPool internals
-    num = sum(map(len, ldc_refs.values()))
-    slots = [pool.pool.getAvailableIndex() for _ in range(num)]
-    pool.pool.reserved.update(slots)
-
-    for type_ in ('Int','Float'):
-        for arg in ldc_refs[type_]:
-            pool.getItem(type_, arg, index=slots.pop())
-    for type_ in ('String','Class','MethodType'):
-        for ind,args in ldc_refs[type_]:
-            arg = ind if ind is not None else pool.Utf8(*args)
-            pool.getItem(type_, arg, index=slots.pop())
-    for type_ in ('MethodHandle',):
-        for code, ref in ldc_refs[type_]:
-            pool.getItem(type_, code, ref.toIndex(pool), index=slots.pop())
-    assert(not slots)
-    assert(not pool.pool.reserved)
-
-def assembleClassFieldMethodAttributes(addcb, directive_dict, pool):
-    for vis in ('Invisible','Visible'):
-        paramd = groupList(directive_dict['.runtime'+vis.lower()])
-        if None in paramd:
-            annotations = [assembleAnnotation(a, pool) for a in paramd[None]]
-            attrlen = 2+sum(map(len, annotations))
-            attr = struct.pack('>HIH', pool.Utf8("Runtime{}Annotations".format(vis)), attrlen, len(annotations)) + ''.join(annotations)
-            addcb(attr)
-
-    for name in directive_dict['.signature']:
-        attr = struct.pack('>HIH', pool.Utf8("Signature"), 2, name.toIndex(pool))
-        addcb(attr)
-
-    #.innerlength directive overrides the normal attribute length calculation
-    hasoverride = len(directive_dict['.innerlength']) > 0
-
-    for name, data in directive_dict['.attribute']:    
-        name_ind = name.toIndex(pool)
-
-        if hasoverride and pool.pool.getArgsCheck('Utf8', name_ind) == 'InnerClasses':
-            attrlen = directive_dict['.innerlength'][0]
-        else:
-            attrlen = len(data)
-
-        attr = struct.pack('>HI', name_ind, attrlen) + data
-        addcb(attr)
-
-def assembleClassAttributes(addcb, directive_dict, pool, addLineNumbers, jasmode, filename):
-
-    sourcefile = directive_dict.get('.source',[None])[0] #PoolRef or None
-    if jasmode and not sourcefile:
-        sourcefile = pool.Utf8(filename)
-    elif addLineNumbers and not sourcefile:
-        sourcefile = pool.Utf8("SourceFile")
-    if sourcefile:
-        attr = struct.pack('>HIH', pool.Utf8("SourceFile"), 2, sourcefile.toIndex(pool))
-        addcb(attr)
-
-    if '.inner' in directive_dict:
-        parts = []
-        for inner, outer, name, flags in directive_dict['.inner']:
-            flagbits = map(ClassFile.flagVals.get, flags)
-            flagbits = reduce(operator.__or__, flagbits, 0)
-            part = struct.pack('>HHHH', inner.toIndex(pool), outer.toIndex(pool), name.toIndex(pool), flagbits)
-            parts.append(part)
-
-        #.innerlength directive overrides the normal attribute length calculation
-        innerlen = 2+8*len(parts) if '.innerlength' not in directive_dict else directive_dict['.innerlength'][0]
-        attr = struct.pack('>HIH', pool.Utf8("InnerClasses"), innerlen, len(parts)) + ''.join(parts)
-        addcb(attr)
-
-    if '.enclosing' in directive_dict:
-        class_, nat = directive_dict['.enclosing'][0]
-        attr = struct.pack('>HIHH', pool.Utf8("EnclosingMethod"), 4, class_.toIndex(pool), nat.toIndex(pool))
-        addcb(attr)
-
-    assembleClassFieldMethodAttributes(addcb, directive_dict, pool)
-
-
-def assemble(tree, addLineNumbers, jasmode, filename):
-    pool = PoolInfo()
-    version, cattrs1, classdec, superdec, interface_decs, cattrs2, topitems = tree
-    if not version: #default to version 49.0 except in Jasmin compatibility mode
-        version = (45,3) if jasmode else (49,0)
-
-    #scan topitems, plus statements in each method to get cpool directives
-    interfaces = []
-    fields = []
-    methods = []
-    attributes = []
-
-    directive_dict = groupList(cattrs1 + cattrs2)
-    top_d = groupList(topitems)
-
-    for slot, value in top_d['const']:
-        if slot.index is not None:
-            pool.fixed[slot.index] = value
-        else:
-            pool.lbls[slot.lbl] = value
-    pool.assignFixedSlots()
-
-    #Now find all cp references used in an ldc instruction
-    #Since they must be <=255, we give them priority in assigning slots
-    #to maximize the chance of a successful assembly
-    addLdcRefs(top_d['method'], pool)
-
-    for flags, name, desc, const, field_directives in top_d['field']:
-        flagbits = map(Field.flagVals.get, flags)
-        flagbits = reduce(operator.__or__, flagbits, 0)
-        name = name.toIndex(pool)
-        desc = desc.toIndex(pool)
-
-        fattrs = []
-        if const is not None:
-            attr = struct.pack('>HIH', pool.Utf8("ConstantValue"), 2, const.toIndex(pool))
-            fattrs.append(attr)
-
-        assembleClassFieldMethodAttributes(fattrs.append, groupList(field_directives), pool)
-
-        field_code = struct.pack('>HHHH', flagbits, name, desc, len(fattrs)) + ''.join(fattrs)
-        fields.append(field_code)
-
-    for header, statements in top_d['method']:
-        methods.append(assembleMethod(header, statements, pool, version, addLineNumbers, jasmode))
-
-    if pool.bootstrap:
-        entries = [struct.pack('>H' + 'H'*len(bsargs), bsargs[0], len(bsargs)-1, *bsargs[1:]) for bsargs in pool.bootstrap]   
-        attrbody = ''.join(entries)
-        attrhead = struct.pack('>HIH', pool.Utf8("BootstrapMethods"), 2+len(attrbody), len(entries))
-        attributes.append(attrhead + attrbody)
-
-    #Explicit class attributes
-    assembleClassAttributes(attributes.append, directive_dict, pool, addLineNumbers, jasmode, filename)
-
-    interfaces = [struct.pack('>H', x.toIndex(pool)) for x in interface_decs]
-    intf, cflags, this = classdec
-    cflags = set(cflags)
-    if intf:
-        cflags.add('INTERFACE')
-    if jasmode:
-        cflags.add('SUPER')
-
-    flagbits = map(ClassFile.flagVals.get, cflags)
-    flagbits = reduce(operator.__or__, flagbits, 0)
-    this = this.toIndex(pool)
-    super_ = superdec.toIndex(pool)
-
-    major, minor = version
-    class_code = '\xCA\xFE\xBA\xBE' + struct.pack('>HH', minor, major)
-    class_code += pool.pool.bytes()
-    class_code += struct.pack('>HHH', flagbits, this, super_)
-    for stuff in (interfaces, fields, methods, attributes):
-        bytes_ = struct.pack('>H', len(stuff)) + ''.join(stuff)
-        class_code += bytes_
-
-    name = pool.pool.getArgs(this)[0]
-    return name, class_code
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/codes.py b/lib/Krakatau/Krakatau/assembler/codes.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/codes.py
+++ /dev/null
@@ -1,11 +0,0 @@
-_handle_types = 'getField getStatic putField putStatic invokeVirtual invokeStatic invokeSpecial newInvokeSpecial invokeInterface'.split()
-handle_codes = dict(zip(_handle_types, range(1,10)))
-
-newarr_codes = dict(zip('boolean char float double byte short int long'.split(), range(4,12)))
-
-vt_keywords = ['Top','Integer','Float','Double','Long','Null','UninitializedThis','Object','Uninitialized']
-vt_codes = {k:i for i,k in enumerate(vt_keywords)}
-
-
-et_rtags = dict(zip('BCDFIJSZsce@[', 'byte char double int float long short boolean string class enum annotation array'.split()))
-et_tags = {v:k for k,v in et_rtags.items()}
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/disassembler.py b/lib/Krakatau/Krakatau/assembler/disassembler.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/disassembler.py
+++ /dev/null
@@ -1,539 +0,0 @@
-import collections
-import re
-
-from . import instructions, tokenize, parse, assembler, codes
-from ..binUnpacker import binUnpacker
-from ..classfile import ClassFile
-
-MAX_INLINE_LENGTH = 50
-
-rhandle_codes = {v:k for k,v in codes.handle_codes.items()}
-rnewarr_codes = {v:k for k,v in codes.newarr_codes.items()}
-
-not_word_regex = '(?:{}|{}|{}|;)'.format(tokenize.int_base, tokenize.float_base, tokenize.t_CPINDEX)
-not_word_regex = re.compile(not_word_regex, re.VERBOSE)
-is_word_regex = re.compile(tokenize.t_WORD.__doc__+'$')
-assert(is_word_regex.match("''") is None)
-
-def isWord(s):
-    '''Determine if s can be used as an inline word'''
-    if s in parse.badwords or (not_word_regex.match(s) is not None):
-        return False
-    #eliminate unprintable characters below 32
-    #also, don't allow characters above 127 to keep things simpler
-    return (is_word_regex.match(s) is not None) and min(s) > ' ' and max(s) <= '\x7f'
-
-def rstring(s, allowWord=True):
-    '''Returns a representation of the string. If allowWord is true, it will be unquoted if possible'''
-    if allowWord and isWord(s):
-        return s
-    try:
-        if s.encode('ascii') == s:
-            return repr(str(s))
-    except (UnicodeEncodeError, UnicodeDecodeError):
-        pass
-    return repr(s)
-
-class PoolManager(object):
-    def __init__(self, pool):
-        self.const_pool = pool #keep this around for the float conversion function
-        self.pool = pool.pool
-        self.bootstrap_methods = [] #filled in externally
-        self.used = set() #which cp entries are used non inline and so must be printed
-
-        #For each type, store the function needed to generate the rhs of a constant pool specifier
-        temp1 = lambda ind: rstring(self.cparg1(ind))
-        temp2 = lambda ind: self.utfref(self.cparg1(ind))
-
-        self.cpref_table = {
-            "Utf8": temp1,
-
-            "Class": temp2,
-            "String": temp2,
-            "MethodType": temp2,
-
-            "NameAndType": self.multiref,
-            "Field": self.multiref,
-            "Method": self.multiref,
-            "InterfaceMethod": self.multiref,
-
-            "Int": self.ldc,
-            "Long": self.ldc,
-            "Float": self.ldc,
-            "Double": self.ldc,
-
-            "MethodHandle": self.methodhandle_notref,
-            "InvokeDynamic": self.invokedynamic_notref,
-            }
-
-    def cparg1(self, ind):
-        return self.pool[ind][1][0]
-
-    def inlineutf(self, ind, allowWord=True):
-        '''Returns the word if it's short enough to inline, else None'''
-        arg = self.cparg1(ind)
-        rstr = rstring(arg, allowWord=allowWord)
-        if len(rstr) <= MAX_INLINE_LENGTH:
-            return rstr
-        return None
-
-    def ref(self, ind):
-        self.used.add(ind)
-        return '[_{}]'.format(ind)
-
-    def utfref(self, ind):
-        if ind == 0:
-            return '[0]'
-        inline = self.inlineutf(ind)
-        return inline if inline is not None else self.ref(ind)
-
-    #Also works for Strings and MethodTypes
-    def classref(self, ind):
-        if ind == 0:
-            return '[0]'
-        inline = self.inlineutf(self.cparg1(ind))
-        return inline if inline is not None else self.ref(ind)
-
-    #For Field, Method, IMethod, and NameAndType. Effectively notref
-    def multiref(self, ind):
-        if ind == 0:
-            return '[0]'
-        typen, args = self.pool[ind]
-        if typen == "Utf8":
-            return self.utfref(ind)
-        elif typen == "Class":
-            return self.classref(ind)
-        return ' '.join(map(self.multiref, args))
-
-    #Special case for instruction fieldrefs as a workaround for Jasmin's awful syntax
-    def notjasref(self, ind):
-        typen, args = self.pool[ind]
-        cind = self.cparg1(ind)
-        inline = self.inlineutf(self.cparg1(cind))
-        if inline is None:
-            return self.ref(ind)
-        return inline + ' ' + self.multiref(args[1])
-
-    def ldc(self, ind):
-        typen, args = self.pool[ind]
-        arg = args[0]
-
-        if typen == 'String':
-            inline = self.inlineutf(arg, allowWord=False)
-            return inline if inline is not None else self.ref(ind)
-        elif typen in ('Int','Long','Float','Double'):
-            if typen == "Float" or typen == "Double":
-                arg = self.const_pool.getArgs(ind)[0]
-
-            rstr = repr(arg).rstrip("Ll")
-            if typen == "Float" or typen == "Long":
-                rstr += typen[0]
-            return rstr
-        else:
-            return self.ref(ind)
-
-    def methodhandle_notref(self, ind):
-        typen, args = self.pool[ind]
-        code = rhandle_codes[args[0]]
-        return code + ' ' + self.ref(args[1])
-
-    def invokedynamic_notref(self, ind):
-        typen, args = self.pool[ind]
-        bs_args = self.bootstrap_methods[args[0]]
-
-        parts = [self.methodhandle_notref(bs_args[0])]
-        parts += map(self.ref, bs_args[1:])
-        parts += [':', self.multiref(args[1])]
-        return ' '.join(parts)
-
-    def printConstDefs(self, add):
-        defs = {}
-
-        while self.used:
-            temp, self.used = self.used, set()
-            for ind in temp:
-                if ind in defs:
-                    continue
-                typen = self.pool[ind][0]
-                defs[ind] = self.cpref_table[typen](ind)
-
-        for ind in sorted(defs):
-            add('.const [_{}] = {} {}'.format(ind, self.pool[ind][0], defs[ind]))
-
-def getAttributeTriples(obj): #name_ind, name, data
-    return [(name_ind, name, data1) for (name_ind, data1), (name, data2) in zip(obj.attributes_raw, obj.attributes)]
-
-def getAttributesDict(obj):
-    d = collections.defaultdict(list)
-    for ind, name, attr in getAttributeTriples(obj):
-        d[name].append((ind, attr))
-    return d
-
-fmt_lookup = {k:v.format for k,v in assembler.op_structs.items()}
-def getInstruction(b, getlbl, poolm):
-    pos = b.off
-    op = b.get('B')
-
-    name = instructions.allinstructions[op]
-    if name == 'wide':
-        name2 = instructions.allinstructions[b.get('B')]
-        if name2 == 'iinc':
-            args = list(b.get('>Hh'))
-        else:
-            args = [b.get('>H')]
-
-        parts = [name, name2] + map(str, args)
-        return '\t' + ' '.join(parts)
-    elif name == 'tableswitch' or name == 'lookupswitch':
-        padding = assembler.getPadding(pos)
-        b.getRaw(padding)
-
-        default = getlbl(b.get('>i')+pos)
-
-        if name == 'lookupswitch':
-            num = b.get('>I')
-            entries = ['\t'+name]
-            entries += ['\t\t{} : {}'.format(b.get('>i'), getlbl(b.get('>i')+pos)) for _ in range(num)]
-        else:
-            low, high = b.get('>ii')
-            num = high-low+1
-            entries = ['\t{} {}'.format(name, low)]
-            entries += ['\t\t{}'.format(getlbl(b.get('>i')+pos)) for _ in range(num)]
-        entries += ['\t\tdefault : {}'.format(default)]
-        return '\n'.join(entries)
-    else:
-        args = list(b.get(fmt_lookup[name], forceTuple=True))
-        #remove extra padding 0
-        if name in ('invokeinterface','invokedynamic'):
-            args = args[:-1]
-
-        funcs = {
-                'OP_CLASS': poolm.classref,
-                'OP_CLASS_INT': poolm.classref,
-                'OP_FIELD': poolm.notjasref, #this is a special case due to the jasmin thing
-                'OP_METHOD': poolm.multiref,
-                'OP_METHOD_INT': poolm.multiref,
-                'OP_DYNAMIC': poolm.ref,
-                'OP_LDC1': poolm.ldc,
-                'OP_LDC2': poolm.ldc,
-                'OP_NEWARR': rnewarr_codes.get,
-            }
-
-        token_t = tokenize.wordget[name]
-        if token_t == 'OP_LBL':
-            assert(len(args) == 1)
-            args[0] = getlbl(args[0]+pos)
-        elif token_t in funcs:
-            args[0] = funcs[token_t](args[0])
-
-        parts = [name] + map(str, args)
-        return '\t' + ' '.join(parts)
-
-def disMethodCode(code, add, poolm):
-    if code is None:
-        return
-    add('\t; method code size: {} bytes'.format(code.codelen))
-    add('\t.limit stack {}'.format(code.stack))
-    add('\t.limit locals {}'.format(code.locals))
-
-    lbls = set()
-    def getlbl(x):
-        lbls.add(x)
-        return 'L'+str(x)
-
-    for e in code.except_raw:
-        parts = poolm.classref(e.type_ind), getlbl(e.start), getlbl(e.end), getlbl(e.handler)
-        add('\t.catch {} from {} to {} using {}'.format(*parts))
-
-    code_attributes = getAttributesDict(code)
-    frames = getStackMapTable(code_attributes, poolm, getlbl)
-
-    instrs = []
-    b = binUnpacker(code.bytecode_raw)
-    while b.size():
-        instrs.append((b.off, getInstruction(b, getlbl, poolm)))
-    instrs.append((b.off, None))
-
-    for off, instr in instrs:
-        if off in lbls:
-            add('L{}:'.format(off))
-        if off in frames:
-            add(frames[off])
-        if instr:
-            add(instr)
-
-    #Generic code attributes
-    for name in code_attributes:
-        #We can't disassemble these because Jasmin's format for these attributes
-        #is overly cumbersome and not easy to disassemble into, but we can't just
-        #leave them as binary blobs either as they are verified by the JVM and the
-        #later two contain constant pool references which won't be preserved even
-        #if the bytecode isn't changed. For now, we just ommit them entirely.
-        #TODO - find a better solution
-        if name in ("LineNumberTable","LocalVariableTable","LocalVariableTypeTable"):
-            continue
-
-        for name_ind, attr in code_attributes[name]:
-            add('.codeattribute {} {!r}'.format(poolm.utfref(name_ind), attr))
-
-def getVerificationType(bytes_, poolm, getLbl):
-    s = codes.vt_keywords[bytes_.get('>B')]
-    if s == 'Object':
-        s += ' ' + poolm.classref(bytes_.get('>H'))
-    elif s == 'Uninitialized':
-        s += ' ' + getLbl(bytes_.get('>H'))
-    return s
-
-def getStackMapTable(code_attributes, poolm, getLbl):
-    smt_attrs = code_attributes['StackMapTable']
-
-    frames = {}
-    offset = 0
-
-    if smt_attrs:
-        assert(len(smt_attrs) == 1)
-        bytes_ = binUnpacker(smt_attrs.pop()[1])
-        count = bytes_.get('>H')
-        getVT = lambda: getVerificationType(bytes_, poolm, getLbl)
-
-        for _ in range(count):
-            tag = bytes_.get('>B')
-            header, contents = None, []
-
-            if 0 <= tag <= 63:
-                offset += tag
-                header = 'same'
-            elif 64 <= tag <= 127:
-                offset += tag - 64
-                header = 'same_locals_1_stack_item'
-                contents.append('\tstack ' + getVT())
-            elif tag == 247:
-                offset += bytes_.get('>H')
-                header = 'same_locals_1_stack_item_extended'
-                contents.append('\tstack ' + getVT())
-            elif 248 <= tag <= 250:
-                offset += bytes_.get('>H')
-                header = 'chop ' + str(251-tag)
-            elif tag == 251:
-                offset += bytes_.get('>H')
-                header = 'same_extended'
-            elif 252 <= tag <= 254:
-                offset += bytes_.get('>H')
-                header = 'append'
-                contents.append('\tlocals ' + ' '.join(getVT() for _ in range(tag-251)))
-            elif tag == 255:
-                offset += bytes_.get('>H')
-                header = 'full'
-                local_count = bytes_.get('>H')
-                contents.append('\tlocals ' + ' '.join(getVT() for _ in range(local_count)))
-                stack_count = bytes_.get('>H')
-                contents.append('\tstack ' + ' '.join(getVT() for _ in range(stack_count)))
-
-            if contents:
-                contents.append('.end stack')
-            contents = ['.stack ' + header] + contents
-            frame = '\n'.join(contents)
-            frames[offset] = frame
-            offset += 1 #frames after the first have an offset one larger than the listed offset
-    return frames
-
-def disCFMAttribute(name_ind, name, bytes_, add, poolm):
-    for vis in ('Visible', 'Invisible'):
-        if name == 'Runtime{}Annotations'.format(vis):
-            count = bytes_.get('>H')
-            for _ in range(count):
-                disAnnotation(bytes_, '.runtime{} '.format(vis.lower()), add, poolm, '')
-            if count: #otherwise we'll create an empty generic attribute
-                return
-
-    if name == "Signature":
-        add('.signature {}'.format(poolm.utfref(bytes_.get('>H'))))
-        return
-    #Create generic attribute if it can't be represented by a standard directive
-    add('.attribute {} {!r}'.format(poolm.utfref(name_ind), bytes_.getRaw(bytes_.size())))
-
-def disMethodAttribute(name_ind, name, bytes_, add, poolm):
-    if name == 'Code':
-        return
-    elif name == 'AnnotationDefault':
-        disElementValue(bytes_, '.annotationdefault ', add, poolm, '')
-        return
-    elif name == 'Exceptions':
-        count = bytes_.get('>H')
-        for _ in range(count):
-            add('.throws ' + poolm.classref(bytes_.get('>H')))
-        if count: #otherwise we'll create an empty generic attribute
-            return
-
-    for vis in ('Visible', 'Invisible'):
-        if name == 'Runtime{}ParameterAnnotations'.format(vis):
-            for i in range(bytes_.get('>B')):
-                for _ in range(bytes_.get('>H')):
-                    disAnnotation(bytes_, '.runtime{} parameter {} '.format(vis.lower(), i), add, poolm, '')
-            return #generic fallback on empty list not yet supported
-
-    disCFMAttribute(name_ind, name, bytes_, add, poolm)
-
-def disMethod(method, add, poolm):
-    mflags = ' '.join(map(str.lower, method.flags))
-    add('.method {} {} : {}'.format(mflags, poolm.utfref(method.name_id), poolm.utfref(method.desc_id)))
-
-    for name_ind, name, attr in getAttributeTriples(method):
-        disMethodAttribute(name_ind, name, binUnpacker(attr), add, poolm)
-
-    disMethodCode(method.code, add, poolm)
-    add('.end method')
-
-def _disEVorAnnotationSub(bytes_, add, poolm, isAnnot, init_prefix, init_indent):
-    C_ANNOT, C_ANNOT2, C_ANNOT3, C_EV, C_EV2 = range(5)
-    init_callt = C_ANNOT if isAnnot else C_EV
-
-    stack = [(init_callt, init_prefix, init_indent)]
-    while stack:
-        callt, prefix, indent = stack.pop()
-
-        if callt == C_ANNOT:
-            add(indent + prefix + 'annotation ' + poolm.utfref(bytes_.get('>H')))
-            #ones we want to happen last should be first on the stack. Annot3 is the final call which ends the annotation
-            stack.append((C_ANNOT3, None, indent))
-            stack.extend([(C_ANNOT2, None, indent)] * bytes_.get('>H'))
-
-        elif callt == C_ANNOT2:
-            key = poolm.utfref(bytes_.get('>H'))
-            stack.append((C_EV, key + ' = ', indent+'\t'))
-
-        elif callt == C_ANNOT3:
-            add(indent + '.end annotation')
-
-        elif callt == C_EV:
-            tag = codes.et_rtags[bytes_.getRaw(1)]
-            if tag == 'annotation':
-                stack.append((C_ANNOT, prefix, indent + '\t'))
-            else:
-                if tag in ('byte','char','double','int','float','long','short','boolean','string'):
-                    val = poolm.ldc(bytes_.get('>H'))
-                elif tag == 'class':
-                    val = poolm.utfref(bytes_.get('>H'))
-                elif tag == 'enum':
-                    val = poolm.utfref(bytes_.get('>H')) + ' ' + poolm.utfref(bytes_.get('>H'))
-                elif tag == 'array':
-                    val = ''
-
-                add(indent + '{} {} {}'.format(prefix, tag, val))
-                if tag == 'array':
-                    for _ in range(bytes_.get('>H')):
-                        stack.append((C_EV, '', indent+'\t'))
-                    stack.append((C_EV2, None, indent))
-
-        elif callt == C_EV2:
-            add(indent + '.end array')
-
-def disElementValue(bytes_, prefix, add, poolm, indent):
-    _disEVorAnnotationSub(bytes_, add, poolm, False, prefix, indent)
-
-def disAnnotation(bytes_, prefix, add, poolm, indent):
-    _disEVorAnnotationSub(bytes_, add, poolm, True, prefix, indent)
-
-#Todo - make fields automatically unpack this themselves
-def getConstValue(field):
-    if not field.static:
-        return None
-    const_attrs = [attr for attr in field.attributes if attr[0] == 'ConstantValue']
-    if const_attrs:
-        assert(len(const_attrs) == 1)
-        bytes_ = binUnpacker(const_attrs[0][1])
-        return bytes_.get('>H')
-
-_classflags = [(v,k.lower()) for k,v in ClassFile.flagVals.items()]
-def disInnerClassesAttribute(name_ind, length, bytes_, add, poolm):
-    count = bytes_.get('>H')
-
-    if length != 2+8*count:
-        add('.innerlength {}'.format(length))
-
-    for _ in range(count):
-        inner, outer, innername, flagbits = bytes_.get('>HHHH')
-
-        flags = [v for k,v in _classflags if k&flagbits]
-        inner = poolm.classref(inner)
-        outer = poolm.classref(outer)
-        innername = poolm.utfref(innername)
-
-        add('.inner {} {} {} {}'.format(' '.join(flags), innername, inner, outer))
-
-    if not count:
-        add('.attribute InnerClasses "\\0\\0"')
-
-def disOtherClassAttribute(name_ind, name, bytes_, add, poolm):
-    assert(name != 'InnerClasses')
-    if name == 'EnclosingMethod':
-        cls, nat = bytes_.get('>HH')
-        add('.enclosing method {} {}'.format(poolm.classref(cls), poolm.multiref(nat)))
-        return
-    disCFMAttribute(name_ind, name, bytes_, add, poolm)
-
-def disassemble(cls):
-    lines = []
-    add = lines.append
-    poolm = PoolManager(cls.cpool)
-
-    # def add(s): print s
-    add('.version {0[0]} {0[1]}'.format(cls.version))
-
-    class_attributes = getAttributesDict(cls)
-    if 'SourceFile' in class_attributes:
-        bytes_ = binUnpacker(class_attributes['SourceFile'].pop()[1])
-        val_ind = bytes_.get('>H')
-        add('.source {}'.format(poolm.utfref(val_ind)))
-
-    if 'BootstrapMethods' in class_attributes:
-        bytes_ = binUnpacker(class_attributes['BootstrapMethods'].pop()[1])
-        count = bytes_.get('>H')
-        for _ in range(count):
-            arg1, argc = bytes_.get('>HH')
-            args = (arg1,) + bytes_.get('>'+'H'*argc, forceTuple=True)
-            poolm.bootstrap_methods.append(args)
-
-    cflags = ' '.join(map(str.lower, cls.flags))
-    add('.class {} {}'.format(cflags, poolm.classref(cls.this)))
-    add('.super {}'.format(poolm.classref(cls.super)))
-    for ii in cls.interfaces_raw:
-        add('.implements {}'.format(poolm.classref(ii)))
-
-    for name in class_attributes:
-        if name == "InnerClasses":
-            assert(len(class_attributes[name]) == 1)
-            for name_ind, (length, attr) in class_attributes[name]:
-                disInnerClassesAttribute(name_ind, length, binUnpacker(attr), add, poolm)
-        else:
-            for name_ind, attr in class_attributes[name]:
-                disOtherClassAttribute(name_ind, name, binUnpacker(attr), add, poolm)
-
-    add('')
-    for field in cls.fields:
-        fflags = ' '.join(map(str.lower, field.flags))
-        const = getConstValue(field)
-
-        if const is not None:
-            add('.field {} {} {} = {}'.format(fflags, poolm.utfref(field.name_id), poolm.utfref(field.desc_id), poolm.ldc(const)))
-        else:
-            add('.field {} {} {}'.format(fflags, poolm.utfref(field.name_id), poolm.utfref(field.desc_id)))
-
-        facount = 0
-        for name_ind, name, attr in getAttributeTriples(field):
-            if name == 'ConstantValue' and field.static:
-                continue
-            disMethodAttribute(name_ind, name, binUnpacker(attr), add, poolm)
-            facount += 1
-        if facount > 0:
-            add('.end field')
-            add('')
-
-    add('')
-
-    for method in cls.methods:
-        disMethod(method, add, poolm)
-        add('')
-
-    poolm.printConstDefs(add)
-    return '\n'.join(lines)
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/instructions.py b/lib/Krakatau/Krakatau/assembler/instructions.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/instructions.py
+++ /dev/null
@@ -1,11 +0,0 @@
-instrs_noarg = ('nop', 'aconst_null', 'iconst_m1', 'iconst_0', 'iconst_1', 'iconst_2', 'iconst_3', 'iconst_4', 'iconst_5', 'lconst_0', 'lconst_1', 'fconst_0', 'fconst_1', 'fconst_2', 'dconst_0', 'dconst_1', 'iload_0', 'iload_1', 'iload_2', 'iload_3', 'lload_0', 'lload_1', 'lload_2', 'lload_3', 'fload_0', 'fload_1', 'fload_2', 'fload_3', 'dload_0', 'dload_1', 'dload_2', 'dload_3', 'aload_0', 'aload_1', 'aload_2', 'aload_3', 'iaload', 'laload', 'faload', 'daload', 'aaload', 'baload', 'caload', 'saload', 'istore_0', 'istore_1', 'istore_2', 'istore_3', 'lstore_0', 'lstore_1', 'lstore_2', 'lstore_3', 'fstore_0', 'fstore_1', 'fstore_2', 'fstore_3', 'dstore_0', 'dstore_1', 'dstore_2', 'dstore_3', 'astore_0', 'astore_1', 'astore_2', 'astore_3', 'iastore', 'lastore', 'fastore', 'dastore', 'aastore', 'bastore', 'castore', 'sastore', 'pop', 'pop2', 'dup', 'dup_x1', 'dup_x2', 'dup2', 'dup2_x1', 'dup2_x2', 'swap', 'iadd', 'ladd', 'fadd', 'dadd', 'isub', 'lsub', 'fsub', 'dsub', 'imul', 'lmul', 'fmul', 'dmul', 'idiv', 'ldiv', 'fdiv', 'ddiv', 'irem', 'lrem', 'frem', 'drem', 'ineg', 'lneg', 'fneg', 'dneg', 'ishl', 'lshl', 'ishr', 'lshr', 'iushr', 'lushr', 'iand', 'land', 'ior', 'lor', 'ixor', 'lxor', 'i2l', 'i2f', 'i2d', 'l2i', 'l2f', 'l2d', 'f2i', 'f2l', 'f2d', 'd2i', 'd2l', 'd2f', 'i2b', 'i2c', 'i2s', 'lcmp', 'fcmpl', 'fcmpg', 'dcmpl', 'dcmpg', 'ireturn', 'lreturn', 'freturn', 'dreturn', 'areturn', 'return', 'arraylength', 'athrow', 'monitorenter', 'monitorexit')
-
-instrs_int = ('bipush', 'sipush', 'iload', 'lload', 'fload', 'dload', 'aload', 'istore', 'lstore', 'fstore', 'dstore', 'astore', 'ret')
-
-instrs_lbl = ('ifeq', 'ifne', 'iflt', 'ifge', 'ifgt', 'ifle', 'if_icmpeq', 'if_icmpne', 'if_icmplt', 'if_icmpge', 'if_icmpgt', 'if_icmple', 'if_acmpeq', 'if_acmpne', 'goto', 'jsr', 'ifnull', 'ifnonnull', 'goto_w', 'jsr_w')
-
-instrs_cp = ('ldc', 'ldc_w', 'ldc2_w', 'getstatic', 'putstatic', 'getfield', 'putfield', 'invokevirtual', 'invokespecial', 'invokestatic', 'invokedynamic', 'new', 'anewarray', 'checkcast', 'instanceof')
-
-instrs_other = ('iinc', 'tableswitch', 'lookupswitch', 'invokeinterface', 'newarray', 'wide', 'multianewarray')
-
-allinstructions = ('nop', 'aconst_null', 'iconst_m1', 'iconst_0', 'iconst_1', 'iconst_2', 'iconst_3', 'iconst_4', 'iconst_5', 'lconst_0', 'lconst_1', 'fconst_0', 'fconst_1', 'fconst_2', 'dconst_0', 'dconst_1', 'bipush', 'sipush', 'ldc', 'ldc_w', 'ldc2_w', 'iload', 'lload', 'fload', 'dload', 'aload', 'iload_0', 'iload_1', 'iload_2', 'iload_3', 'lload_0', 'lload_1', 'lload_2', 'lload_3', 'fload_0', 'fload_1', 'fload_2', 'fload_3', 'dload_0', 'dload_1', 'dload_2', 'dload_3', 'aload_0', 'aload_1', 'aload_2', 'aload_3', 'iaload', 'laload', 'faload', 'daload', 'aaload', 'baload', 'caload', 'saload', 'istore', 'lstore', 'fstore', 'dstore', 'astore', 'istore_0', 'istore_1', 'istore_2', 'istore_3', 'lstore_0', 'lstore_1', 'lstore_2', 'lstore_3', 'fstore_0', 'fstore_1', 'fstore_2', 'fstore_3', 'dstore_0', 'dstore_1', 'dstore_2', 'dstore_3', 'astore_0', 'astore_1', 'astore_2', 'astore_3', 'iastore', 'lastore', 'fastore', 'dastore', 'aastore', 'bastore', 'castore', 'sastore', 'pop', 'pop2', 'dup', 'dup_x1', 'dup_x2', 'dup2', 'dup2_x1', 'dup2_x2', 'swap', 'iadd', 'ladd', 'fadd', 'dadd', 'isub', 'lsub', 'fsub', 'dsub', 'imul', 'lmul', 'fmul', 'dmul', 'idiv', 'ldiv', 'fdiv', 'ddiv', 'irem', 'lrem', 'frem', 'drem', 'ineg', 'lneg', 'fneg', 'dneg', 'ishl', 'lshl', 'ishr', 'lshr', 'iushr', 'lushr', 'iand', 'land', 'ior', 'lor', 'ixor', 'lxor', 'iinc', 'i2l', 'i2f', 'i2d', 'l2i', 'l2f', 'l2d', 'f2i', 'f2l', 'f2d', 'd2i', 'd2l', 'd2f', 'i2b', 'i2c', 'i2s', 'lcmp', 'fcmpl', 'fcmpg', 'dcmpl', 'dcmpg', 'ifeq', 'ifne', 'iflt', 'ifge', 'ifgt', 'ifle', 'if_icmpeq', 'if_icmpne', 'if_icmplt', 'if_icmpge', 'if_icmpgt', 'if_icmple', 'if_acmpeq', 'if_acmpne', 'goto', 'jsr', 'ret', 'tableswitch', 'lookupswitch', 'ireturn', 'lreturn', 'freturn', 'dreturn', 'areturn', 'return', 'getstatic', 'putstatic', 'getfield', 'putfield', 'invokevirtual', 'invokespecial','invokestatic', 'invokeinterface', 'invokedynamic', 'new', 'newarray', 'anewarray', 'arraylength', 'athrow', 'checkcast', 'instanceof', 'monitorenter', 'monitorexit', 'wide', 'multianewarray', 'ifnull', 'ifnonnull', 'goto_w', 'jsr_w')
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/parse.py b/lib/Krakatau/Krakatau/assembler/parse.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/parse.py
+++ /dev/null
@@ -1,554 +0,0 @@
-import ast, struct
-import itertools
-
-from ..classfile import ClassFile
-from ..method import Method
-from ..field import Field
-
-#Important to import tokens here even though it appears unused, as ply uses it
-from .tokenize import tokens, wordget, flags
-from .assembler import PoolRef
-
-#Specify the starting symbol
-start = 'top'
-
-###############################################################################
-name_counter = itertools.count()
-def addRule(func, name, *rhs_rules):
-    def _inner(p):
-        func(p)
-    _inner.__doc__ = name + ' : ' + '\n| '.join(rhs_rules)
-    fname = 'p_{}'.format(next(name_counter))
-    globals()[fname] = _inner
-
-def list_sub(p):p[0] = p[1] + p[2:]
-def listRule(name): #returns a list
-    name2 = name + 's'
-    addRule(list_sub, name2, '{} {}'.format(name2, name), 'empty')    
-
-def nothing(p):pass
-def assign1(p):p[0] = p[1]
-def assign2(p):p[0] = p[2]
-def upper1(p): p[0] = p[1].upper()
-
-# Common Rules ################################################################
-addRule(nothing, 'sep', 'sep NEWLINE', 'NEWLINE')
-
-def p_empty(p):
-    'empty :'
-    p[0] = []
-
-def p_intl(p):
-    '''intl : INT_LITERAL'''
-    p[0] = ast.literal_eval(p[1])
-
-def p_longl(p):
-    '''longl : LONG_LITERAL'''
-    p[0] = ast.literal_eval(p[1][:-1])
-
-#Todo - find a better way of handling floats
-def parseFloat(s):
-    s = s[:-1]
-    if s.strip('-')[:2].lower() == '0x':
-        f = float.fromhex(s)
-    else:
-        f = float(s)
-    return struct.unpack('>i', struct.pack('>f', f))[0]
-
-def parseDouble(s):
-    if s.strip('-')[:2].lower() == '0x':
-        f = float.fromhex(s)
-    else:
-        f = float(s)
-    return struct.unpack('>q', struct.pack('>d', f))[0]
-
-def p_floatl(p):
-    '''floatl : FLOAT_LITERAL'''
-    p[0] = parseFloat(p[1])
-def p_doublel(p):
-    '''doublel : DOUBLE_LITERAL'''
-    p[0] = parseDouble(p[1])
-
-#We can allow keywords as inline classnames as long as they aren't flag names
-#which would be ambiguous. We don't allow directives to simplfy the grammar
-#rules, since they wouldn't be valid identifiers anyway.
-badwords = frozenset(map(str.lower, flags))
-badwords |= frozenset(k for k in wordget if '.' in k) 
-oktokens = frozenset(v for k,v in wordget.items() if k not in badwords)
-addRule(assign1, 'notflag', 'WORD', 'STRING_LITERAL', *oktokens)
-
-def p_ref(p):
-    '''ref : CPINDEX'''
-    s = p[1][1:-1]
-    try:
-        i = int(s)
-        if 0 <= i <= 0xFFFF:
-            p[0] = PoolRef(index=i)
-        else:
-            p[0] = PoolRef(lbl=s)    
-    except ValueError:
-        p[0] = PoolRef(lbl=s)
-
-def p_utf8_notref(p):
-    '''utf8_notref : notflag'''
-    p[0] = PoolRef('Utf8', p[1])
-
-def p_class_notref(p):
-    '''class_notref : utf8_notref'''
-    p[0] = PoolRef('Class', p[1])
-
-def p_string_notref(p):
-    '''string_notref : utf8_notref'''
-    p[0] = PoolRef('String', p[1])
-
-def p_nat_notref(p):
-    '''nameandtype_notref : utf8ref utf8ref'''
-    p[0] = PoolRef('NameAndType', p[1], p[2])
-
-def p_field_notref(p):
-    '''field_notref : classref nameandtyperef'''
-    p[0] = PoolRef('Field', p[1], p[2])
-
-def p_method_notref(p):
-    '''method_notref : classref nameandtyperef'''
-    p[0] = PoolRef('Method', p[1], p[2])
-
-def p_imethod_notref(p):
-    '''interfacemethod_notref : classref nameandtyperef'''
-    p[0] = PoolRef('InterfaceMethod', p[1], p[2])
-
-#constant pool types related to InvokeDynamic handled later
-
-for _name in ('utf8','class', 'nameandtype', 'method', 'interfacemethod', 'methodhandle'):
-    addRule(assign1, '{}ref'.format(_name), '{}_notref'.format(_name), 'ref')
-
-###############################################################################
-def p_classnoend(p):
-    '''classnoend : version_opt class_directive_lines classdec superdec interfacedecs class_directive_lines topitems'''
-    p[0] = tuple(p[1:])
-
-addRule(assign1, 'classwithend', 'classnoend D_END CLASS sep')
-listRule('classwithend')
-
-def p_top(p):
-    '''top : sep classwithends classnoend'''
-    p[0] = p[2] + [p[3]]
-#case where all classes have an end
-addRule(assign2, 'top', 'sep classwithends')
-
-def p_version(p):
-    '''version_opt : D_VERSION intl intl sep'''
-    p[0] = p[2], p[3]
-addRule(assign1, 'version_opt', 'empty')
-
-###############################################################################
-for c, type_ in zip('cmf', (ClassFile, Method, Field)):
-    _name = "{}flag".format(c)
-    addRule(upper1, _name, *list(type_.flagVals))
-    listRule(_name)
-
-def p_classdec(p):
-    '''classdec : D_CLASS cflags classref sep 
-                | D_INTERFACE cflags classref sep'''
-    #if interface, add interface to flags
-    p[0] = (p[1] == '.interface'), p[2], p[3]
-
-addRule(assign2, 'superdec', 'D_SUPER classref sep')
-addRule(assign2, 'interfacedec', 'D_IMPLEMENTS classref sep')
-listRule('interfacedec')
-
-addRule(assign1, 'class_directive', 'classattribute', 'innerlength_dir')
-addRule(assign1, 'class_directive_line', 'class_directive sep')
-listRule('class_directive_line')
-
-def p_topitem_c(p):
-    '''topitem : const_spec'''
-    p[0] = 'const', p[1]
-def p_topitem_f(p):
-    '''topitem : field_spec'''
-    p[0] = 'field', p[1]
-def p_topitem_m(p):
-    '''topitem : method_spec'''
-    p[0] = 'method', p[1]
-listRule('topitem')
-
-###############################################################################
-#invoke dynamic stuff
-from .codes import handle_codes
-_handle_token_types = set(wordget.get(x, 'WORD') for x in handle_codes)
-def p_handle(p):
-    p[0] = handle_codes[p[1]]
-p_handle.__doc__ = "handlecode : " + '\n| '.join(_handle_token_types)
-
-#The second argument's type depends on the code, so we require an explicit reference for simplicity
-def p_methodhandle_notref(p):
-    '''methodhandle_notref : handlecode ref'''
-    p[0] = PoolRef('MethodHandle', p[1], p[2])
-
-def p_methodtype_notref(p):
-    '''methodtype_notref : utf8_notref'''
-    p[0] = PoolRef('Methodtype', p[1])
-
-addRule(assign1, 'bootstrap_arg', 'ref') #TODO - allow inline constants and strings?
-listRule('bootstrap_arg')
-
-def p_invokedynamic_notref(p):
-    '''invokedynamic_notref : methodhandleref bootstrap_args COLON nameandtyperef'''
-    args = [p[1]] + p[2] + [p[4]]
-    p[0] = PoolRef('InvokeDynamic', *args)
-
-###############################################################################
-def p_const_spec(p):
-    '''const_spec : D_CONST ref EQUALS const_rhs sep'''
-    p[0] = p[2], p[4]
-
-def assignPoolSingle(typen):
-    def inner(p):
-        p[0] = PoolRef(typen, p[2])
-    return inner
-
-addRule(assign1, 'const_rhs', 'ref')
-for tt in ['UTF8', 'CLASS','STRING','NAMEANDTYPE','FIELD','METHOD','INTERFACEMETHOD',
-            'METHODHANDLE','METHODTYPE','INVOKEDYNAMIC']:
-    addRule(assign2, 'const_rhs', '{} {}_notref'.format(tt, tt.lower()))
-
-#these are special cases, since they take a single argument
-#and the notref version can't have a ref as its argument due to ambiguity
-for ptype in ('Class','String','MethodType'):
-    addRule(assignPoolSingle(ptype), 'const_rhs', ptype.upper() + ' ref')
-
-for ptype in ('Int','Float','Long','Double'):
-    addRule(assignPoolSingle(ptype), 'const_rhs', '{} {}l'.format(ptype.upper(), ptype.lower()))
-###############################################################################
-
-
-def p_field_spec(p):
-    '''field_spec : D_FIELD fflags utf8ref utf8ref field_constval fieldattribute_list'''
-    p[0] = p[2:7]
-
-addRule(nothing, 'field_constval', 'empty')
-addRule(assign2, 'field_constval', 'EQUALS ref', 
-                                    'EQUALS ldc1_notref', 
-                                    'EQUALS ldc2_notref')
-
-#Sadly, we must only allow .end field when at least one attribute is specified
-#in order to avoid grammatical ambiguity. JasminXT does not share this problem
-#because it lacks the .end class syntax which causes the conflict
-def p_field_attrlist1(p):
-    '''field_al_nonempty : fieldattribute sep field_al_nonempty'''
-    p[0] = [p[1]]+ p[3]
-def p_field_attrlist2(p):
-    '''field_al_nonempty : fieldattribute sep D_END FIELD sep'''
-    p[0] = [p[1]]
-
-addRule(assign2, 'fieldattribute_list', 'sep field_al_nonempty', 'sep empty')
-
-
-def p_method_spec(p):
-    '''method_spec : defmethod statements endmethod'''
-    p[0] = p[1],p[2]
-
-def p_defmethod_0(p):
-    '''defmethod : D_METHOD mflags jas_meth_namedesc sep'''
-    p[0] = p[2],p[3] 
-def p_defmethod_1(p):
-    '''defmethod : D_METHOD mflags utf8ref COLON utf8ref sep'''
-    p[0] = p[2],(p[3], p[5]) 
-
-def p_jas_meth_namedesc(p):
-    '''jas_meth_namedesc : WORD'''
-    name, paren, desc = p[1].rpartition('(')
-    name = PoolRef('Utf8', name)
-    desc = PoolRef('Utf8', paren+desc)
-    p[0] = name, desc
-addRule(nothing, 'endmethod', 'D_END METHOD sep')
-
-def p_statement_0(p):
-    '''statement : method_directive sep'''
-    p[0] = False, p[1]
-def p_statement_1(p):
-    '''statement : code_directive sep'''
-    p[0] = True, (False, p[1])
-def p_statement_2(p):
-    '''statement : empty instruction sep 
-                | lbldec instruction sep
-                | lbldec sep'''
-    p[0] = True, (True, ((p[1] or None), p[2]))
-listRule('statement')
-
-addRule(assign1, 'lbldec', 'lbl COLON')
-addRule(assign1, 'method_directive', 'methodattribute')
-addRule(assign1, 'code_directive', 'limit_dir', 'except_dir','localvar_dir','linenumber_dir','stack_dir', 'generic_codeattribute_dir')
-
-def p_limit_dir(p):
-    '''limit_dir : D_LIMIT LOCALS intl 
-                | D_LIMIT STACK intl'''
-    p[0] = p[1], (p[2], p[3])
-
-def p_except_dir(p):
-    '''except_dir : D_CATCH classref FROM lbl TO lbl USING lbl'''
-    p[0] = p[1], (p[2], p[4], p[6], p[8])
-
-def p_linenumber_dir(p):
-    '''linenumber_dir : D_LINE intl'''
-    p[0] = p[1], p[2]
-
-def p_localvar_dir(p):
-    '''localvar_dir : D_VAR intl IS utf8ref utf8ref FROM lbl TO lbl'''
-    p[0] = p[1], (p[2], p[4], p[5], p[7], p[9])
-
-def p_instruction(p):
-    '''instruction : OP_NONE
-                    | OP_INT intl
-                    | OP_INT_INT intl intl
-                    | OP_LBL lbl
-                    | OP_FIELD fieldref_or_jas
-                    | OP_METHOD methodref_or_jas
-                    | OP_METHOD_INT imethodref_or_jas intl
-                    | OP_DYNAMIC ref
-                    | OP_CLASS classref
-                    | OP_CLASS_INT classref intl
-                    | OP_LDC1 ldc1_ref
-                    | OP_LDC2 ldc2_ref
-                    | OP_NEWARR nacode
-                    | OP_LOOKUPSWITCH luswitch
-                    | OP_TABLESWITCH tblswitch
-                    | OP_WIDE wide_instr
-                    '''
-    if p[1] == 'invokenonvirtual':
-        p[1] = 'invokespecial'
-    p[0] = tuple(p[1:])
-    #these instructions have 0 padding at the end
-    #this is kind of an ungly hack, but the best way I could think of
-    if p[1] in ('invokeinterface','invokedynamic'):
-        p[0] += (0,)
-
-addRule(assign1, 'lbl', 'WORD')
-addRule(assign1, 'fieldref_or_jas', 'jas_fieldref', 'ref', 'inline_fieldref')
-def p_jas_fieldref(p):
-    '''jas_fieldref : WORD WORD'''
-    class_, sep, name = p[1].replace('.','/').rpartition('/')
-
-    desc = PoolRef('Utf8', p[2])
-    class_ = PoolRef('Class', PoolRef('Utf8', class_))
-    name = PoolRef('Utf8', name)
-    nt = PoolRef('NameAndType', name, desc)
-    p[0] = PoolRef('Field', class_, nt)
-
-#This is an ugly hack to work around the fact that Jasmin syntax would otherwise be impossible to 
-#handle with a LALR(1) parser
-def p_inline_fieldref_1(p):
-    '''inline_fieldref : WORD nameandtyperef
-                        | STRING_LITERAL nameandtyperef'''
-    class_ = PoolRef('Class', PoolRef('Utf8', p[1]))
-    p[0] = PoolRef('Field', class_, p[2])
-def p_inline_fieldref_2(p):
-    '''inline_fieldref : ref nameandtyperef'''
-    p[0] = PoolRef('Field', p[1], p[2])
-
-
-def p_jas_meth_classnamedesc(p):
-    '''jas_methodref : WORD'''
-    name, paren, desc = p[1].rpartition('(')
-    class_, sep, name = name.replace('.','/').rpartition('/')
-    desc = paren + desc
-
-    class_ = PoolRef('Class', PoolRef('Utf8', class_))
-    nt = PoolRef('NameAndType', PoolRef('Utf8', name), PoolRef('Utf8', desc))
-    p[0] = class_, nt
-
-addRule(assign1, 'methodref_or_jas', 'methodref')
-def p_methodref_or_jas(p):
-    '''methodref_or_jas : jas_methodref'''
-    p[0] = PoolRef('Method', *p[1])
-
-addRule(assign1, 'imethodref_or_jas', 'interfacemethodref')
-def p_imethodref_or_jas(p):
-    '''imethodref_or_jas : jas_methodref'''
-    p[0] = PoolRef('InterfaceMethod', *p[1])
-
-
-from .codes import newarr_codes
-_newarr_token_types = set(wordget.get(x, 'WORD') for x in newarr_codes)
-def p_nacode(p):
-    p[0] = newarr_codes[p[1]]
-p_nacode.__doc__ = "nacode : " + '\n| '.join(_newarr_token_types)
-
-addRule(assign1, 'ldc1_ref', 'ldc1_notref', 'ref')
-def p_ldc1_notref_string(p):
-    '''ldc1_notref : STRING_LITERAL'''
-    p[0] = PoolRef('String', PoolRef('Utf8', p[1]))
-def p_ldc1_notref_int(p):
-    '''ldc1_notref : intl'''
-    p[0] = PoolRef('Int', p[1])
-def p_ldc1_notref_float(p):
-    '''ldc1_notref : floatl'''
-    p[0] = PoolRef('Float', p[1])
-
-addRule(assign1, 'ldc2_ref', 'ldc2_notref', 'ref')
-def p_ldc2_notref_long(p):
-    '''ldc2_notref : longl'''
-    p[0] = PoolRef('Long', p[1])
-def p_ldc2_notref_double(p):
-    '''ldc2_notref : doublel'''
-    p[0] = PoolRef('Double', p[1])
-
-def p_defaultentry(p):
-    '''defaultentry : DEFAULT COLON lbl'''
-    p[0] = p[3]
-
-def p_luentry(p):
-    '''luentry : intl COLON lbl sep'''
-    p[0] = p[1], p[3]
-listRule('luentry')
-
-addRule(assign1, 'tblentry', 'lbl sep')
-listRule('tblentry')
-
-def p_lookupswitch(p):
-    '''luswitch : empty sep luentrys defaultentry'''
-    p[0] = p[1], p[3], p[4]
-
-def p_tableswitch(p):
-    '''tblswitch : intl sep tblentrys defaultentry'''
-    p[0] = p[1], p[3], p[4]
-
-def p_wide_instr(p):
-    '''wide_instr : OP_INT intl
-                | OP_INT_INT intl intl'''
-    p[0] = p[1], tuple(p[2:])
-
-#######################################################################
-# Explicit Attributes
-addRule(assign1, 'cfmattribute', 'annotation_dir', 'signature_dir', 'generic_attribute_dir')
-addRule(assign1, 'classattribute', 'cfmattribute', 'sourcefile_dir', 'inner_dir', 'enclosing_dir')
-addRule(assign1, 'fieldattribute', 'cfmattribute')
-addRule(assign1, 'methodattribute', 'cfmattribute', 'throws_dir', 'annotation_param_dir', 'annotation_def_dir')
-
-#Class, field, method
-def p_annotation_dir(p):
-    '''annotation_dir : D_RUNTIMEVISIBLE annotation
-                    | D_RUNTIMEINVISIBLE annotation'''
-    p[0] = p[1], (None, p[2])
-
-def p_signature_dir(p):
-    '''signature_dir : D_SIGNATURE utf8ref'''
-    p[0] = p[1], p[2]
-
-#Class only
-def p_sourcefile_dir(p):
-    '''sourcefile_dir : D_SOURCE utf8ref'''
-    p[0] = p[1], p[2]
-
-def p_inner_dir(p): 
-    '''inner_dir : D_INNER cflags utf8ref classref classref'''
-    p[0] = p[1], (p[4],p[5],p[3],p[2]) #use JasminXT's (flags, name, inner, outer) order but switch internally to correct order
-
-def p_enclosing_dir(p): 
-    '''enclosing_dir : D_ENCLOSING METHOD classref nameandtyperef'''
-    p[0] = p[1], (p[3], p[4])
-
-#This is included here even though strictly speaking, it's not an attribute. Rather it's a directive that affects the assembly
-#of the InnerClasses attribute
-def p_innerlength_dir(p): 
-    '''innerlength_dir : D_INNERLENGTH intl'''
-    p[0] = p[1], p[2]
-
-
-#Method only
-def p_throws_dir(p):
-    '''throws_dir : D_THROWS classref'''
-    p[0] = p[1], p[2]
-
-def p_annotation_param_dir(p):
-    '''annotation_param_dir : D_RUNTIMEVISIBLE PARAMETER intl annotation
-                           | D_RUNTIMEINVISIBLE PARAMETER intl annotation'''
-    p[0] = p[1], (p[3], p[4])
-def p_annotation_def_dir(p):
-    '''annotation_def_dir : D_ANNOTATIONDEFAULT element_value'''
-    p[0] = p[1], p[2]
-
-#Generic
-def p_generic_attribute_dir(p): 
-    '''generic_attribute_dir : D_ATTRIBUTE utf8ref STRING_LITERAL'''
-    p[0] = p[1], (p[2], p[3])
-
-def p_generic_codeattribute_dir(p): 
-    '''generic_codeattribute_dir : D_CODEATTRIBUTE utf8ref STRING_LITERAL'''
-    p[0] = p[1], (p[2], p[3])
-
-#######################################################################
-#Stack map stuff
-addRule(nothing, 'endstack', 'D_END STACK') #directives are not expected to end with a sep
-
-def assign1All(p):p[0] = tuple(p[1:])
-addRule(assign1All, 'verification_type', 'TOP', 'INTEGER', 'FLOAT', 'DOUBLE', 'LONG', 'NULL', 'UNINITIALIZEDTHIS',
-                                        'OBJECT classref', 'UNINITIALIZED lbl')
-listRule('verification_type')
-addRule(assign2, 'locals_vtlist', 'LOCALS verification_types sep')
-addRule(assign2, 'stack_vtlist', 'STACK verification_types sep')
-
-def p_stack_dir(p):
-    '''stack_dir_rest : SAME 
-                    | SAME_EXTENDED
-                    | CHOP intl 
-                    | SAME_LOCALS_1_STACK_ITEM sep stack_vtlist endstack
-                    | SAME_LOCALS_1_STACK_ITEM_EXTENDED sep stack_vtlist endstack
-                    | APPEND sep locals_vtlist endstack
-                    | FULL sep locals_vtlist stack_vtlist endstack'''
-    p[0] = '.stackmap', tuple(p[1:])
-addRule(assign2, 'stack_dir', 'D_STACK stack_dir_rest')
-#######################################################################
-#Annotation stuff
-from .codes import et_tags
-primtags = set(wordget.get(x, 'WORD') for x in 'byte char double int float long short boolean string'.split())
-addRule(assign1, 'primtag', *primtags)
-addRule(assign1, 'ldc_any', 'ldc1_notref', 'ldc2_notref', 'ref')
-
-def p_element_value_0(p):
-    '''element_value : primtag ldc_any
-                    | CLASS utf8ref
-                    | ENUM utf8ref utf8ref
-                    | ARRAY sep element_array'''
-    p[0] = et_tags[p[1]], tuple(p[2:])
-def p_element_value_1(p):
-    '''element_value : annotation'''
-    p[0] = '@', (p[1],)
-
-addRule(assign1, 'element_value_line', 'element_value sep')
-listRule('element_value_line')
-addRule(assign1, 'element_array', 'element_value_lines D_END ARRAY')
-
-def p_key_ev_line(p):
-    '''key_ev_line : utf8ref EQUALS element_value_line'''
-    p[0] = p[1], p[3]
-listRule('key_ev_line')
-
-def p_annotation(p):
-    '''annotation : ANNOTATION utf8ref sep key_ev_lines D_END ANNOTATION'''
-    p[0] = p[2], p[4]
-#######################################################################
-
-def p_error(p):
-    if p is None:
-        print "Syntax error: unexpected EOF"
-    else: #remember to subtract 1 from line number since we had a newline at the start of the file
-        print "Syntax error at line {}: unexpected token {!r}".format(p.lineno-1, p.value)
-    
-    #Ugly hack since Ply doesn't provide any useful error information
-    import inspect
-    frame = inspect.currentframe()
-    cvars = frame.f_back.f_locals
-    print 'Expected:', ', '.join(cvars['actions'][cvars['state']].keys())
-    print 'Found:', cvars['ltype']
-    print 'Current stack:', cvars['symstack']
-
-    #Discard the rest of the input so that Ply doesn't attempt error recovery
-    from ply import yacc
-    tok = yacc.token()
-    while tok is not None:
-        tok = yacc.token()
-
-def makeParser(**kwargs):
-    from ply import yacc
-    return yacc.yacc(**kwargs)
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/tokenize.py b/lib/Krakatau/Krakatau/assembler/tokenize.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/tokenize.py
+++ /dev/null
@@ -1,140 +0,0 @@
-import ast
-
-from ..classfile import ClassFile
-from ..method import Method
-from ..field import Field
-from .. import constant_pool
-from . import instructions as ins
-from . import codes
-
-directives = 'CLASS','INTERFACE','SUPER','IMPLEMENTS','CONST','FIELD','METHOD','END','LIMIT','CATCH','SOURCE','LINE','VAR','THROWS',
-directives += 'VERSION', 'STACK', 'RUNTIMEVISIBLE', 'RUNTIMEINVISIBLE', 'ANNOTATIONDEFAULT', 'INNER', 'ENCLOSING', 'SIGNATURE', 
-directives += 'ATTRIBUTE', 'CODEATTRIBUTE', 'INNERLENGTH'
-keywords = ['CLASS','METHOD','FIELD','LOCALS','STACK','FROM','TO','USING','DEFAULT','IS']
-keywords += ['SAME','SAME_LOCALS_1_STACK_ITEM','SAME_LOCALS_1_STACK_ITEM_EXTENDED','CHOP','SAME_EXTENDED','APPEND','FULL']
-keywords += ['ANNOTATION','ARRAY','PARAMETER']
-flags = ClassFile.flagVals.keys() + Method.flagVals.keys() + Field.flagVals.keys()
-
-lowwords = set().union(keywords, flags)
-casewords = set().union(codes.vt_keywords, constant_pool.name2Type.keys())
-
-wordget = {}
-wordget.update({w.lower():w.upper() for w in lowwords})
-wordget.update({w:w.upper() for w in casewords})
-wordget.update({'.'+w.lower():'D_'+w for w in directives})
-
-assert(set(wordget).isdisjoint(ins.allinstructions))
-for op in ins.instrs_noarg:
-    wordget[op] = 'OP_NONE'
-for op in ins.instrs_int:
-    wordget[op] = 'OP_INT'
-for op in ins.instrs_lbl:
-    wordget[op] = 'OP_LBL'
-for op in ('getstatic', 'putstatic', 'getfield', 'putfield'):
-    wordget[op] = 'OP_FIELD'
-#support invokenonvirtual for backwards compatibility with Jasmin
-for op in ('invokevirtual', 'invokespecial', 'invokestatic', 'invokenonvirtual'): 
-    wordget[op] = 'OP_METHOD'
-for op in ('new', 'anewarray', 'checkcast', 'instanceof'):
-    wordget[op] = 'OP_CLASS'
-for op in ('wide','lookupswitch','tableswitch'):
-    wordget[op] = 'OP_' + op.upper()
-
-wordget['ldc'] = 'OP_LDC1'
-wordget['ldc_w'] = 'OP_LDC1'
-wordget['ldc2_w'] = 'OP_LDC2'
-wordget['iinc'] = 'OP_INT_INT'
-wordget['newarray'] = 'OP_NEWARR'
-wordget['multianewarray'] = 'OP_CLASS_INT'
-wordget['invokeinterface'] = 'OP_METHOD_INT'
-wordget['invokedynamic'] = 'OP_DYNAMIC'
-
-for op in ins.allinstructions:
-    wordget.setdefault(op,op.upper())
-
-#special PLY value
-tokens = ('NEWLINE', 'COLON', 'EQUALS', 'WORD', 'CPINDEX', 
-    'STRING_LITERAL', 'INT_LITERAL', 'LONG_LITERAL', 'FLOAT_LITERAL', 'DOUBLE_LITERAL') + tuple(set(wordget.values()))
-
-def t_ignore_COMMENT(t):
-    r';.*'
-
-# Define a rule so we can track line numbers
-def t_NEWLINE(t):
-    r'\n+'
-    t.lexer.lineno += len(t.value)
-    return t
-
-def t_STRING_LITERAL(t):
-    # See http://stackoverflow.com/questions/430759/regex-for-managing-escaped-characters-for-items-like-string-literals/5455705#5455705
-    r'''[uUbB]?[rR]?(?:
-        """[^"\\]*              # any number of unescaped characters
-            (?:\\.[^"\\]*       # escaped followed by 0 or more unescaped
-                |"[^"\\]+       # single quote followed by at least one unescaped
-                |""[^"\\]+      # two quotes followed by at least one unescaped
-            )*"""
-        |"[^"\n\\]*              # any number of unescaped characters
-            (?:\\.[^"\n\\]*      # escaped followed by 0 or more unescaped
-            )*"
-    '''r"""                     # concatenated string literals
-        |'''[^'\\]*              # any number of unescaped characters
-            (?:\\.[^'\\]*       # escaped followed by 0 or more unescaped
-                |'[^'\\]+       # single quote followed by at least one unescaped
-                |''[^'\\]+      # two quotes followed by at least one unescaped
-            )*'''
-        |'[^'\n\\]*              # any number of unescaped characters
-            (?:\\.[^'\n\\]*      # escaped followed by 0 or more unescaped
-            )*'
-        )"""
-
-    t.value = ast.literal_eval(t.value)
-    return t
-
-#careful here: | is not greedy so hex must come first
-int_base = r'[+-]?(?:0[xX][0-9a-fA-F]+|[0-9]+)'
-float_base = r'''(?:
-    [Nn][Aa][Nn]|                                       #Nan
-    [-+]?(?:                                            #Inf and normal both use sign
-        [Ii][Nn][Ff]|                                   #Inf
-        \d+\.\d*(?:[eE][+-]?\d+)?|                         #decimal float
-        \d+[eE][+-]?\d+|                                   #decimal float with no fraction (exponent mandatory)
-        0[xX][0-9a-fA-F]*\.[0-9a-fA-F]+[pP][+-]?\d+        #hexidecimal float
-        )
-    )
-'''
-
-#Sadly there's no nice way to define these even with reflection hacks
-#Hopefully we can get Ply patched some day or fork it or something so 
-#it's not so much of a pain
-
-#These are matched in order of appearence (specifically, f.func_code.co_firstlineno)
-#So anything that can be a prefix of another must go last
-def t_FLOAT_LITERAL(t): return t
-t_FLOAT_LITERAL.__doc__ = float_base + r'[fF]'
-def t_DOUBLE_LITERAL(t): return t
-t_DOUBLE_LITERAL.__doc__ = float_base
-def t_LONG_LITERAL(t): return t
-t_LONG_LITERAL.__doc__ = int_base + r'[lL]'
-def t_INT_LITERAL(t): return t
-t_INT_LITERAL.__doc__ = int_base
-
-def t_CPINDEX(t): return t
-t_CPINDEX.__doc__ = r'\[[0-9a-z_]+\]'
-
-
-def t_WORD(t):
-    r'''[^\s:="']+'''
-    t.type = wordget.get(t.value, 'WORD')
-    return t
-
-t_COLON = r':'
-t_EQUALS = r'='
-t_ignore = ' \t\r'
-
-def t_error(t):
-    print 'Parser error on line {} at {}'.format(t.lexer.lineno, t.lexer.lexpos)
-    print t.value[:79]
-
-def makeLexer(**kwargs):
-    from ply import lex
-    return lex.lex(**kwargs)
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/attributes_raw.py b/lib/Krakatau/Krakatau/attributes_raw.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/attributes_raw.py
+++ /dev/null
@@ -1,21 +0,0 @@
-def get_attribute_raw(bytestream, ic_indices):
-    name_ind, length = bytestream.get('>HL')
-
-    #Hotspot does not actually check the attribute length of InnerClasses prior to 49.0
-    #so this case requires special handling. We will keep the purported length of the 
-    #attribute so that it can be displayed in the disassembly. For InnerClass attributes
-    #data is actually a (length, bytes) tuple, rather than storing the bytes directly
-    if name_ind in ic_indices:
-        count = bytestream.get('>H', peek=True)
-        data = length, bytestream.getRaw(2+8*count)
-    else:
-        data = bytestream.getRaw(length)
-    
-    return name_ind,data
-
-def get_attributes_raw(bytestream, ic_indices=()):
-    attribute_count = bytestream.get('>H')

From noreply at buildbot.pypy.org  Thu Dec 11 15:40:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 11 Dec 2014 15:40:44 +0100 (CET)
Subject: [pypy-commit] benchmarks default: merge heads
Message-ID: <20141211144044.31BD81C02EE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r297:14fee126c6bd
Date: 2014-12-11 14:41 +0000
http://bitbucket.org/pypy/benchmarks/changeset/14fee126c6bd/

Log:	merge heads

diff --git a/own/bm_krakatau.py b/own/bm_krakatau.py
--- a/own/bm_krakatau.py
+++ b/own/bm_krakatau.py
@@ -47,19 +47,22 @@
                 source = javaclass.generateAST(c, makeGraph).print_()
 
 
+WARMUP_ITERATIONS = 30 # Krakatau needs a number of iterations to warmup...
+
 def main(n):
     l = []
     old_stdout = sys.stdout
     sys.stdout = cStringIO.StringIO()
     try:
-        for i in range(n):
+        for i in range(WARMUP_ITERATIONS + n):
             t0 = time.time()
-            decompileClass()
+            for j in range(4):
+                decompileClass()
             time_elapsed = time.time() - t0
             l.append(time_elapsed)
     finally:
         sys.stdout = old_stdout
-    return l
+    return l[WARMUP_ITERATIONS:]
 
 if __name__ == "__main__":
     parser = optparse.OptionParser(

From noreply at buildbot.pypy.org  Thu Dec 11 15:59:32 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Thu, 11 Dec 2014 15:59:32 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Don't read/write from a cache
	file.
Message-ID: <20141211145932.439531D27FB@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r298:2a17a0177782
Date: 2014-12-11 14:50 +0000
http://bitbucket.org/pypy/benchmarks/changeset/2a17a0177782/

Log:	Don't read/write from a cache file.

	Without this, the benchmark's behaviour is (even more) non-
	deterministic from one run to another.

diff --git a/own/krakatau/Krakatau/Krakatau/stdcache.py b/own/krakatau/Krakatau/Krakatau/stdcache.py
--- a/own/krakatau/Krakatau/Krakatau/stdcache.py
+++ b/own/krakatau/Krakatau/Krakatau/stdcache.py
@@ -6,11 +6,14 @@
         self.env = env
         self.filename = filename
 
-        try:
-            with open(self.filename, 'rb') as f:
-                fdata = f.read()
-        except IOError:
-            fdata = ''
+        # XXX for a benchmark, we don't ever want to use a cache, so we simply
+        # don't load data from a cache file (even if it exists)
+        fdata = ''
+        #try:
+        #    with open(self.filename, 'rb') as f:
+        #        fdata = f.read()
+        #except IOError:
+        #    fdata = ''
 
         #Note, we assume \n will never appear in a class name. This should be true for classes in the Java package,
         #but isn't necessarily true for user defined classes (Which we don't cache anyway)
@@ -24,8 +27,8 @@
         newvals = class_.getSuperclassHierarchy(), class_.flags 
         self.data[class_.name] = newvals 
         writedata = ';'.join(','.join(x) for x in newvals)
-        with open(self.filename, 'ab') as f:
-            f.write(writedata + '\n')
+        #with open(self.filename, 'ab') as f:
+        #    f.write(writedata + '\n')
         print class_.name, 'cached'
 
     def isCached(self, name): return name in self.data
@@ -46,4 +49,4 @@
         class_ = self.env.getClass(name, partial=True)
         if shouldCache(name):
             self._cache_info(class_)
-        return class_.flags
\ No newline at end of file
+        return class_.flags

From noreply at buildbot.pypy.org  Thu Dec 11 22:46:08 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 11 Dec 2014 22:46:08 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: overeager '_immutable_fields_w'
Message-ID: <20141211214608.04E371C02EE@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r74890:e1a050f7eb5d
Date: 2014-12-08 21:21 +0200
http://bitbucket.org/pypy/pypy/changeset/e1a050f7eb5d/

Log:	overeager '_immutable_fields_w'

diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -25,7 +25,6 @@
 
 class W_NDimArray(W_NumpyObject):
     __metaclass__ = extendabletype
-    _immutable_fields_ = ['implementation']
 
     def __init__(self, implementation):
         from pypy.module.micronumpy.concrete import BaseConcreteArray

From noreply at buildbot.pypy.org  Thu Dec 11 22:46:09 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 11 Dec 2014 22:46:09 +0100 (CET)
Subject: [pypy-commit] pypy default: test, fix view of a slice
Message-ID: <20141211214609.40CD91C02EE@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r74891:b75a06bcc48d
Date: 2014-12-11 18:36 +0200
http://bitbucket.org/pypy/pypy/changeset/b75a06bcc48d/

Log:	test, fix view of a slice

diff --git a/pypy/module/micronumpy/flagsobj.py b/pypy/module/micronumpy/flagsobj.py
--- a/pypy/module/micronumpy/flagsobj.py
+++ b/pypy/module/micronumpy/flagsobj.py
@@ -5,46 +5,26 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import constants as NPY
-
+from pypy.module.micronumpy.strides import is_c_contiguous, is_f_contiguous
 
 def enable_flags(arr, flags):
     arr.flags |= flags
 
-
 def clear_flags(arr, flags):
     arr.flags &= ~flags
 
-
- at jit.unroll_safe
 def _update_contiguous_flags(arr):
-    shape = arr.shape
-    strides = arr.strides
-
-    is_c_contig = True
-    sd = arr.dtype.elsize
-    for i in range(len(shape) - 1, -1, -1):
-        dim = shape[i]
-        if strides[i] != sd:
-            is_c_contig = False
-            break
-        if dim == 0:
-            break
-        sd *= dim
+    is_c_contig = is_c_contiguous(arr)
     if is_c_contig:
         enable_flags(arr, NPY.ARRAY_C_CONTIGUOUS)
     else:
         clear_flags(arr, NPY.ARRAY_C_CONTIGUOUS)
 
-    sd = arr.dtype.elsize
-    for i in range(len(shape)):
-        dim = shape[i]
-        if strides[i] != sd:
-            clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
-            return
-        if dim == 0:
-            break
-        sd *= dim
-    enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
+    is_f_contig = is_f_contiguous(arr)
+    if is_f_contig:
+        enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
+    else:
+        clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
 
 
 class W_FlagsObject(W_Root):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -19,7 +19,7 @@
     order_converter, shape_converter, searchside_converter
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.strides import get_shape_from_iterable, \
-    shape_agreement, shape_agreement_multiple
+    shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
 
 
 def _match_dot_shapes(space, left, right):
@@ -837,7 +837,15 @@
                 raise OperationError(space.w_ValueError, space.wrap(
                     "new type not compatible with array."))
         else:
-            if dims == 1 or impl.get_strides()[0] < impl.get_strides()[-1]:
+            if not is_c_contiguous(impl) and not is_f_contiguous(impl):
+                if old_itemsize != new_itemsize:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array."))
+                # Strides, shape does not change
+                v = impl.astype(space, dtype)
+                return wrap_impl(space, w_type, self, v) 
+            strides = impl.get_strides()
+            if dims == 1 or strides[0] <strides[-1]:
                 # Column-major, resize first dimension
                 if new_shape[0] * old_itemsize % new_itemsize != 0:
                     raise OperationError(space.w_ValueError, space.wrap(
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -429,3 +429,35 @@
                     n_old_elems_to_use *= old_shape[oldI]
     assert len(new_strides) == len(new_shape)
     return new_strides[:]
+
+ at jit.unroll_safe
+def is_c_contiguous(arr):
+    shape = arr.get_shape()
+    strides = arr.get_strides()
+    ret = True
+    sd = arr.dtype.elsize
+    for i in range(len(shape) - 1, -1, -1):
+        dim = shape[i]
+        if strides[i] != sd:
+            ret = False
+            break
+        if dim == 0:
+            break
+        sd *= dim
+    return ret
+
+ at jit.unroll_safe
+def is_f_contiguous(arr):
+    shape = arr.get_shape()
+    strides = arr.get_strides()
+    ret = True
+    sd = arr.dtype.elsize
+    for i in range(len(shape)):
+        dim = shape[i]
+        if strides[i] != sd:
+            ret = False
+            break
+        if dim == 0:
+            break
+        sd *= dim
+    return ret
diff --git a/pypy/module/micronumpy/test/test_iterators.py b/pypy/module/micronumpy/test/test_iterators.py
--- a/pypy/module/micronumpy/test/test_iterators.py
+++ b/pypy/module/micronumpy/test/test_iterators.py
@@ -13,6 +13,11 @@
         self.strides = strides
         self.start = start
 
+    def get_shape(self):
+        return self.shape
+
+    def get_strides(self):
+        return self.strides
 
 class TestIterDirect(object):
     def test_iterator_basic(self):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1807,6 +1807,17 @@
         x = array([], dtype=[('a', 'int8'), ('b', 'int8')])
         y = x.view(dtype='int16')
 
+    def test_view_of_slice(self):
+        from numpy import empty
+        x = empty([6], 'uint32')
+        x.fill(0xdeadbeef)
+        s = x[::3]
+        exc = raises(ValueError, s.view, 'uint8')
+        assert exc.value[0] == 'new type not compatible with array.'
+        s[...] = 2
+        v = s.view(x.__class__)
+        assert (v == 2).all()
+    
     def test_tolist_scalar(self):
         from numpy import dtype
         int32 = dtype('int32').type

From noreply at buildbot.pypy.org  Thu Dec 11 22:46:10 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 11 Dec 2014 22:46:10 +0100 (CET)
Subject: [pypy-commit] pypy default: fix test,
 coercing values to records - reduces numpy incompatability
Message-ID: <20141211214610.78C521C02EE@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r74892:218d49b0e351
Date: 2014-12-11 23:30 +0200
http://bitbucket.org/pypy/pypy/changeset/218d49b0e351/

Log:	fix test, coercing values to records - reduces numpy incompatability

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2923,11 +2923,7 @@
         a = empty(10, dtype=[(_, int) for _ in 'abcde'])
         a.fill(123)
         for i in a:
-            import sys
-            if '__pypy__' in sys.builtin_module_names:
-                assert tuple(i) == (123,) + (0,) * 4
-            else:
-                assert tuple(i) == (123,) * 5
+            assert tuple(i) == (123,) * 5
 
         a = zeros(3, dtype=dtype(complex).newbyteorder())
         a.fill(1.5+2.5j)
@@ -3857,7 +3853,7 @@
         a = np.array([b, b, b], dtype=dt)
         assert a.shape == (3, 2)
         for i in a.flat:
-            assert tuple(i) == (True, False)
+            assert tuple(i) == (True, True)
 
         dt = np.dtype([('A', '<i8'), ('B', '<f8'), ('C', '<c16')])
         b = np.array((999999, 1e+20, 1e+20+0j), dtype=dt)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -311,26 +311,26 @@
     BoxType = boxes.W_BoolBox
     format_code = "?"
 
-    True = BoxType(True)
-    False = BoxType(False)
+    _True = BoxType(True)
+    _False = BoxType(False)
 
     @specialize.argtype(1)
     def box(self, value):
         box = Primitive.box(self, value)
         if box.value:
-            return self.True
+            return self._True
         else:
-            return self.False
+            return self._False
 
     @specialize.argtype(1, 2)
     def box_complex(self, real, imag):
         box = Primitive.box(self, real)
         if box.value:
-            return self.True
+            return self._True
         box = Primitive.box(self, imag)
         if box.value:
-            return self.True
-        return self.False
+            return self._True
+        return self._False
 
     def coerce_subtype(self, space, w_subtype, w_item):
         # Doesn't return subclasses so it can return the constants.
@@ -1869,7 +1869,7 @@
                 items_w = space.fixedview(w_item.get_scalar_value())
             else:
                 # XXX support initializing from readable buffers
-                items_w = [w_item]
+                items_w = [w_item] * len(dtype.fields)
         else:
             items_w = [None] * len(dtype.fields)
         arr = VoidBoxStorage(dtype.elsize, dtype)

From noreply at buildbot.pypy.org  Fri Dec 12 07:58:02 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 12 Dec 2014 07:58:02 +0100 (CET)
Subject: [pypy-commit] pypy default: Merged in malor/pypy (pull request #292)
Message-ID: <20141212065802.2E54D1C3688@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74894:fd41db7b1558
Date: 2014-12-12 08:57 +0200
http://bitbucket.org/pypy/pypy/changeset/fd41db7b1558/

Log:	Merged in malor/pypy (pull request #292)

	sqlite3: fix fetching of cursor.description attr

diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -478,6 +478,29 @@
         except TypeError:
             pass
 
+    def CheckCurDescription(self):
+        self.cu.execute("select * from test")
+
+        actual = self.cu.description
+        expected = [
+            ('id', None, None, None, None, None, None),
+            ('name', None, None, None, None, None, None),
+            ('income', None, None, None, None, None, None),
+        ]
+        self.assertEqual(expected, actual)
+
+    def CheckCurDescriptionVoidStatement(self):
+        self.cu.execute("insert into test(name) values (?)", ("foo",))
+        self.assertIsNone(self.cu.description)
+
+    def CheckCurDescriptionWithoutStatement(self):
+        cu = self.cx.cursor()
+        try:
+            self.assertIsNone(cu.description)
+        finally:
+            cu.close()
+
+
 @unittest.skipUnless(threading, 'This test requires threading.')
 class ThreadTests(unittest.TestCase):
     def setUp(self):
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1175,8 +1175,11 @@
         try:
             return self.__description
         except AttributeError:
-            self.__description = self.__statement._get_description()
-            return self.__description
+            try:
+                self.__description = self.__statement._get_description()
+                return self.__description
+            except AttributeError:
+                return None
     description = property(__get_description)
 
     def __get_lastrowid(self):

From noreply at buildbot.pypy.org  Fri Dec 12 07:58:00 2014
From: noreply at buildbot.pypy.org (Roman Podoliaka)
Date: Fri, 12 Dec 2014 07:58:00 +0100 (CET)
Subject: [pypy-commit] pypy default: sqlite3: fix fetching of
	cursor.description attr
Message-ID: <20141212065800.C16AC1C3688@cobra.cs.uni-duesseldorf.de>

Author: Roman Podoliaka <roman.podoliaka at gmail.com>
Branch: 
Changeset: r74893:fa1e46300cb1
Date: 2014-12-11 15:13 +0200
http://bitbucket.org/pypy/pypy/changeset/fa1e46300cb1/

Log:	sqlite3: fix fetching of cursor.description attr

	According to PEP-249 (DB API) cursor attribute 'description' will be
	None, if the cursor has not had an operation invoked via the
	.execute*() method yet, but this snippet currently fails on PyPy:

	 import sqlite3

	 db = sqlite3.connect(':memory:') cur = db.cursor() try:
	print(cur.description) finally: cur.close()

	 This commit fixes the existing behaviour to comply with the PEP.

diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -478,6 +478,29 @@
         except TypeError:
             pass
 
+    def CheckCurDescription(self):
+        self.cu.execute("select * from test")
+
+        actual = self.cu.description
+        expected = [
+            ('id', None, None, None, None, None, None),
+            ('name', None, None, None, None, None, None),
+            ('income', None, None, None, None, None, None),
+        ]
+        self.assertEqual(expected, actual)
+
+    def CheckCurDescriptionVoidStatement(self):
+        self.cu.execute("insert into test(name) values (?)", ("foo",))
+        self.assertIsNone(self.cu.description)
+
+    def CheckCurDescriptionWithoutStatement(self):
+        cu = self.cx.cursor()
+        try:
+            self.assertIsNone(cu.description)
+        finally:
+            cu.close()
+
+
 @unittest.skipUnless(threading, 'This test requires threading.')
 class ThreadTests(unittest.TestCase):
     def setUp(self):
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1175,8 +1175,11 @@
         try:
             return self.__description
         except AttributeError:
-            self.__description = self.__statement._get_description()
-            return self.__description
+            try:
+                self.__description = self.__statement._get_description()
+                return self.__description
+            except AttributeError:
+                return None
     description = property(__get_description)
 
     def __get_lastrowid(self):

From noreply at buildbot.pypy.org  Fri Dec 12 11:54:06 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 12 Dec 2014 11:54:06 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: start updating numpy task ideas
Message-ID: <20141212105406.58C3B1C0188@cobra.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r5482:16701bc31bcb
Date: 2014-12-12 12:52 +0200
http://bitbucket.org/pypy/extradoc/changeset/16701bc31bcb/

Log:	start updating numpy task ideas

diff --git a/planning/micronumpy.txt b/planning/micronumpy.txt
--- a/planning/micronumpy.txt
+++ b/planning/micronumpy.txt
@@ -1,15 +1,13 @@
 NEW TASKS
 ---------
 
-- astype
-
 - a good sort function
 
 - indexing by arrays and lists
 
 - endianness
 
-- frompyfunc to create ufuncs from python functions
+- frompyfunc to create ufuncs from python functions (wip on ufuncapi branch)
 
 - more ufuncs
 
@@ -37,11 +35,11 @@
 - bridges are a tad too complicated - we need to store optimizer info
   in order to improve though
 
-- iterators should possibly not require reinstantiating them.
-
 - things like take/item/fancy indexing can use some knowledge about the density
   of data and either evaluate interesting points (without forcing) or
   do what they do now.
 
 - counting by element_size instead of by 1 and then multiply sounds
   like a much faster option sometimes
+
+- use ndarray.flags to cache values, only recompute when state changes

From noreply at buildbot.pypy.org  Fri Dec 12 12:47:30 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Dec 2014 12:47:30 +0100 (CET)
Subject: [pypy-commit] creflect default: Allow multiple declarations of a
 function with the same name.
Message-ID: <20141212114730.CABED1D3915@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r198:1676e7d1646f
Date: 2014-12-12 11:47 +0000
http://bitbucket.org/cffi/creflect/changeset/1676e7d1646f/

Log:	Allow multiple declarations of a function with the same name.

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -65,6 +65,7 @@
     outerblock = CodeBlock(parent=None)
     outerblock.writeline('#include "creflect.h"')
     outerblock.crx_func_name = crx_func_name
+    outerblock.func_names_seen = set()
 
     funcblock = CodeBlock(outerblock)
 
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -348,7 +348,7 @@
     def shadow_global_var(self, top_level_block, fnname):
         if self.ellipsis:
             return fnname
-        shadowname = '%s__d_%s' % (top_level_block.crx_func_name, fnname)
+        shadowname = _unique_name(top_level_block, '%s__d%s_%s', fnname)
         wrline = top_level_block.writeline
         args = [arg.get_c_name('a%d' % i) for i, arg in enumerate(self.args)]
         decl = '%s(%s)' % (shadowname, ', '.join(args) or 'void')
@@ -365,7 +365,7 @@
         # a special-case for global function declarations
         assert not self.ellipsis
         toplevel = block.crx_top_level
-        crx_func_name = '%s__c_%s' % (toplevel.crx_func_name, fnname)
+        crx_func_name = _unique_name(toplevel, '%s__c%s_%s', fnname)
         wrline = toplevel.writeline
         wrline('static void %s(void *args[], void *result) {' % (
             crx_func_name,))
@@ -378,6 +378,16 @@
             fnname, t1, a2, len(self.args), crx_func_name, shadow))
 
 
+def _unique_name(toplevel, pattern, fnname):
+    result = pattern % (toplevel.crx_func_name, '', fnname)
+    n = 0
+    while result in toplevel.func_names_seen:
+        n += 1
+        result = pattern % (toplevel.crx_func_name, n, fnname)
+    toplevel.func_names_seen.add(result)
+    return result
+
+
 class PointerType(BaseType):
     _attrs_ = ('toqualtype',)
 
diff --git a/creflect/test/codegen/func-006.c b/creflect/test/codegen/func-006.c
new file mode 100644
--- /dev/null
+++ b/creflect/test/codegen/func-006.c
@@ -0,0 +1,51 @@
+/* Multiple declarations of a function with the same name:
+   a way to work around "..." in signatures */
+
+int f(int);
+int f(int, long);
+
+# ____________________________________________________________
+# drop
+
+int f(int a, ...) { return 42 * a; }
+
+# ____________________________________________________________
+
+static void testfunc_006__c_f(void *args[], void *result) {
+    *(int *)result = f(*(int *)args[0]);
+}
+
+static int testfunc_006__d_f(int a0) {
+    return f(a0);
+}
+
+static void testfunc_006__c1_f(void *args[], void *result) {
+    *(int *)result = f(*(int *)args[0], *(long *)args[1]);
+}
+
+static int testfunc_006__d1_f(int a0, long a1) {
+    return f(a0, a1);
+}
+
+void testfunc_006(_crx_builder_t *cb)
+{
+    _crx_type_t *t1, *t2;
+    _crx_qual_type a1[1];
+    _crx_qual_type a2[2];
+    {
+        t1 = cb->get_signed_type(cb, sizeof(int), "int");
+        a1[0].type = t1;
+        a1[0].qualifiers = 0;
+        cb->define_func(cb, "f", t1, a1, 1, &testfunc_006__c_f, &testfunc_006__d_f);
+#expect FUNC f: int -> int
+    }
+    {
+        t2 = cb->get_signed_type(cb, sizeof(long), "long");
+        a2[0].type = t1;
+        a2[0].qualifiers = 0;
+        a2[1].type = t2;
+        a2[1].qualifiers = 0;
+        cb->define_func(cb, "f", t1, a2, 2, &testfunc_006__c1_f, &testfunc_006__d1_f);
+#expect FUNC f: int -> long -> int
+    }
+}
diff --git a/creflect/test/test_cgcompile.py b/creflect/test/test_cgcompile.py
--- a/creflect/test/test_cgcompile.py
+++ b/creflect/test/test_cgcompile.py
@@ -25,11 +25,15 @@
 #include "creflect.h"
 #include "creflect_debug_print.h"
 ''')
+    start_pos = f.tell()
     for line in inputlines:
         if not (line.startswith('# ') or line.startswith('#expect')):
             if line.startswith('typedef ...'):
                 line = 'typedef struct _missing ' + line[11:]
             f.write(line)
+        elif line.startswith('# drop'):
+            f.seek(start_pos)
+            f.truncate()
     f.write('''
 int main(void)
 {
@@ -48,13 +52,7 @@
     lines = g.readlines()
     err = g.close()
     assert not err
-    if any('/*' in line for line in expected):
-        # only remove the comments that are addresses
-        r_remove_addr = re.compile(r"/[*]0x[-0-9a-f]+[*]/")
-    else:
-        # remove the comments that are addresses or sizes
-        r_remove_addr = re.compile(r"/[*][-0-9a-fx,]+[*]/")
-    got = [r_remove_addr.sub('', line.rstrip()) for line in lines]
+    got = [line.rstrip() for line in lines]
     compare_lists(got, expected)
 
 

From noreply at buildbot.pypy.org  Fri Dec 12 13:01:44 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 12 Dec 2014 13:01:44 +0100 (CET)
Subject: [pypy-commit] pypy default: small steps - get rid of a confusing
	loop
Message-ID: <20141212120144.E97791D3973@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74895:dcf1b6b1b8a5
Date: 2014-12-12 14:01 +0200
http://bitbucket.org/pypy/pypy/changeset/dcf1b6b1b8a5/

Log:	small steps - get rid of a confusing loop

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -146,7 +146,7 @@
     loop.quasi_immutable_deps = {}
     if part.quasi_immutable_deps:
         loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
-    while part.operations[-1].getopnum() == rop.LABEL:
+    if part.operations[-1].getopnum() == rop.LABEL:
         inliner = Inliner(inputargs, jumpargs)
         part.quasi_immutable_deps = None
         part.operations = [part.operations[-1]] + \
@@ -167,6 +167,7 @@
         loop.operations = loop.operations[:-1] + part.operations
         if part.quasi_immutable_deps:
             loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
+    assert part.operations[-1].getopnum() != rop.LABEL
 
     if not loop.quasi_immutable_deps:
         loop.quasi_immutable_deps = None

From noreply at buildbot.pypy.org  Fri Dec 12 15:23:03 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Dec 2014 15:23:03 +0100 (CET)
Subject: [pypy-commit] pypy default: (cfbolz, arigo)
Message-ID: <20141212142303.4A1191C0188@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74896:3fe17318bbdb
Date: 2014-12-12 14:22 +0000
http://bitbucket.org/pypy/pypy/changeset/3fe17318bbdb/

Log:	(cfbolz, arigo)

	Fix for prebuilt r_ordereddicts.

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -19,7 +19,7 @@
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
 from rpython.annotator.argument import simple_args
-from rpython.rlib.objectmodel import r_dict, Symbolic
+from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper import extregistry
 
@@ -260,21 +260,23 @@
                     result.listdef.generalize(self.immutablevalue(e))
                 result.const_box = key
                 return result
-        elif tp is dict or tp is r_dict or tp is SomeOrderedDict.knowntype:
-            if tp is SomeOrderedDict.knowntype:
-                cls = SomeOrderedDict
-            else:
-                cls = SomeDict
+        elif (tp is dict or tp is r_dict or
+              tp is SomeOrderedDict.knowntype or tp is r_ordereddict):
             key = Constant(x)
             try:
                 return self.immutable_cache[key]
             except KeyError:
+                if tp is SomeOrderedDict.knowntype or tp is r_ordereddict:
+                    cls = SomeOrderedDict
+                else:
+                    cls = SomeDict
+                is_r_dict = issubclass(tp, r_dict)
                 result = cls(DictDef(self,
                                         s_ImpossibleValue,
                                         s_ImpossibleValue,
-                                        is_r_dict = tp is r_dict))
+                                        is_r_dict = is_r_dict))
                 self.immutable_cache[key] = result
-                if tp is r_dict:
+                if is_r_dict:
                     s_eqfn = self.immutablevalue(x.key_eq)
                     s_hashfn = self.immutablevalue(x.key_hash)
                     result.dictdef.dictkey.update_rdict_annotations(s_eqfn,
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -217,7 +217,7 @@
         #dictobj = getattr(dictobj, '__self__', dictobj)
         if dictobj is None:
             return lltype.nullptr(self.DICT)
-        if not isinstance(dictobj, (dict, objectmodel.r_dict)):
+        if not isinstance(dictobj, (dict, objectmodel.r_ordereddict)):
             raise TypeError("expected a dict: %r" % (dictobj,))
         try:
             key = Constant(dictobj)
@@ -231,7 +231,8 @@
             if r_key.lowleveltype == llmemory.Address:
                 raise TypeError("No prebuilt dicts of address keys")
             r_value = self.value_repr
-            if isinstance(dictobj, objectmodel.r_dict):
+            if isinstance(dictobj, objectmodel.r_ordereddict):
+                
                 if self.r_rdict_eqfn.lowleveltype != lltype.Void:
                     l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq)
                     l_dict.fnkeyeq = l_fn
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -878,6 +878,81 @@
         res = self.interpret(func, [])
         assert lltype.typeOf(res.item0) == lltype.typeOf(res.item1)
 
+    def test_r_dict(self):
+        class FooError(Exception):
+            pass
+        def myeq(n, m):
+            return n == m
+        def myhash(n):
+            if n < 0:
+                raise FooError
+            return -n
+        def f(n):
+            d = self.new_r_dict(myeq, myhash)
+            for i in range(10):
+                d[i] = i*i
+            try:
+                value1 = d[n]
+            except FooError:
+                value1 = 99
+            try:
+                value2 = n in d
+            except FooError:
+                value2 = 99
+            try:
+                value3 = d[-n]
+            except FooError:
+                value3 = 99
+            try:
+                value4 = (-n) in d
+            except FooError:
+                value4 = 99
+            return (value1 * 1000000 +
+                    value2 * 10000 +
+                    value3 * 100 +
+                    value4)
+        res = self.interpret(f, [5])
+        assert res == 25019999
+
+    def test_r_dict_popitem_hash(self):
+        def deq(n, m):
+            return n == m
+        def dhash(n):
+            return ~n
+        def func():
+            d = self.new_r_dict(deq, dhash)
+            d[5] = 2
+            d[6] = 3
+            k1, v1 = d.popitem()
+            assert len(d) == 1
+            k2, v2 = d.popitem()
+            try:
+                d.popitem()
+            except KeyError:
+                pass
+            else:
+                assert 0, "should have raised KeyError"
+            assert len(d) == 0
+            return k1*1000 + v1*100 + k2*10 + v2
+
+        res = self.interpret(func, [])
+        assert res in [5263, 6352]
+
+    def test_prebuilt_r_dict(self):
+        def deq(n, m):
+            return (n & 3) == (m & 3)
+        def dhash(n):
+            return n & 3
+        d = self.new_r_dict(deq, dhash)
+        d[0x123] = "abcd"
+        d[0x231] = "efgh"
+        def func():
+            return d[0x348973] + d[0x12981]
+
+        res = self.interpret(func, [])
+        res = self.ll_to_string(res)
+        assert res == "abcdefgh"
+
 
 class TestRDict(BaseTestRDict):
     @staticmethod
@@ -888,6 +963,10 @@
     def newdict2():
         return {}
 
+    @staticmethod
+    def new_r_dict(myeq, myhash):
+        return r_dict(myeq, myhash)
+
     def test_two_dicts_with_different_value_types(self):
         def func(i):
             d1 = {}
@@ -1043,66 +1122,6 @@
         assert lltype.typeOf(res.item1) == lltype.typeOf(res.item2)
         assert lltype.typeOf(res.item1) == lltype.typeOf(res.item3)
 
-    def test_r_dict(self):
-        class FooError(Exception):
-            pass
-        def myeq(n, m):
-            return n == m
-        def myhash(n):
-            if n < 0:
-                raise FooError
-            return -n
-        def f(n):
-            d = r_dict(myeq, myhash)
-            for i in range(10):
-                d[i] = i*i
-            try:
-                value1 = d[n]
-            except FooError:
-                value1 = 99
-            try:
-                value2 = n in d
-            except FooError:
-                value2 = 99
-            try:
-                value3 = d[-n]
-            except FooError:
-                value3 = 99
-            try:
-                value4 = (-n) in d
-            except FooError:
-                value4 = 99
-            return (value1 * 1000000 +
-                    value2 * 10000 +
-                    value3 * 100 +
-                    value4)
-        res = self.interpret(f, [5])
-        assert res == 25019999
-
-    def test_dict_popitem_hash(self):
-        def deq(n, m):
-            return n == m
-        def dhash(n):
-            return ~n
-        def func():
-            d = r_dict(deq, dhash)
-            d[5] = 2
-            d[6] = 3
-            k1, v1 = d.popitem()
-            assert len(d) == 1
-            k2, v2 = d.popitem()
-            try:
-                d.popitem()
-            except KeyError:
-                pass
-            else:
-                assert 0, "should have raised KeyError"
-            assert len(d) == 0
-            return k1*1000 + v1*100 + k2*10 + v2
-
-        res = self.interpret(func, [])
-        assert res in [5263, 6352]
-
     def test_nonnull_hint(self):
         def eq(a, b):
             return a == b
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -268,6 +268,10 @@
     def newdict2():
         return OrderedDict()
 
+    @staticmethod
+    def new_r_dict(myeq, myhash):
+        return objectmodel.r_ordereddict(myeq, myhash)
+
     def test_two_dicts_with_different_value_types(self):
         def func(i):
             d1 = OrderedDict()
@@ -283,64 +287,3 @@
 
     def test_memoryerror_should_not_insert(self):
         py.test.skip("I don't want to edit this file on two branches")
-
-
-    def test_r_dict(self):
-        class FooError(Exception):
-            pass
-        def myeq(n, m):
-            return n == m
-        def myhash(n):
-            if n < 0:
-                raise FooError
-            return -n
-        def f(n):
-            d = objectmodel.r_ordereddict(myeq, myhash)
-            for i in range(10):
-                d[i] = i*i
-            try:
-                value1 = d[n]
-            except FooError:
-                value1 = 99
-            try:
-                value2 = n in d
-            except FooError:
-                value2 = 99
-            try:
-                value3 = d[-n]
-            except FooError:
-                value3 = 99
-            try:
-                value4 = (-n) in d
-            except FooError:
-                value4 = 99
-            return (value1 * 1000000 +
-                    value2 * 10000 +
-                    value3 * 100 +
-                    value4)
-        res = self.interpret(f, [5])
-        assert res == 25019999
-
-    def test_dict_popitem_hash(self):
-        def deq(n, m):
-            return n == m
-        def dhash(n):
-            return ~n
-        def func():
-            d = objectmodel.r_ordereddict(deq, dhash)
-            d[5] = 2
-            d[6] = 3
-            k1, v1 = d.popitem()
-            assert len(d) == 1
-            k2, v2 = d.popitem()
-            try:
-                d.popitem()
-            except KeyError:
-                pass
-            else:
-                assert 0, "should have raised KeyError"
-            assert len(d) == 0
-            return k1*1000 + v1*100 + k2*10 + v2
-
-        res = self.interpret(func, [])
-        assert res in [5263, 6352]

From noreply at buildbot.pypy.org  Fri Dec 12 15:48:37 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Dec 2014 15:48:37 +0100 (CET)
Subject: [pypy-commit] pypy default: (cfbolz, arigo)
Message-ID: <20141212144837.084011C320E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74897:2b51137c528f
Date: 2014-12-12 14:48 +0000
http://bitbucket.org/pypy/pypy/changeset/2b51137c528f/

Log:	(cfbolz, arigo)

	Copy support for prepare_dict_update() from rdict to rordereddict.

diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -329,6 +329,18 @@
         res = self.interpret(g, [3])
         assert res == 42     # "did not crash"
 
+    def test_prepare_dict_update_2(self):
+        try:
+            from collections import OrderedDict
+        except ImportError:     # Python 2.6
+            py.test.skip("requires collections.OrderedDict")
+        def g(n):
+            d = OrderedDict()
+            prepare_dict_update(d, n)
+            return 42
+        res = self.interpret(g, [3])
+        assert res == 42     # "did not crash"
+
     def test_compute_hash(self):
         class Foo(object):
             pass
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -290,6 +290,11 @@
         hop.exception_cannot_occur()
         return hop.gendirectcall(ll_dict_update, v_dic1, v_dic2)
 
+    def rtype_method__prepare_dict_update(self, hop):
+        v_dict, v_num = hop.inputargs(self, lltype.Signed)
+        hop.exception_cannot_occur()
+        hop.gendirectcall(ll_prepare_dict_update, v_dict, v_num)
+
     def _rtype_method_kvi(self, hop, ll_func):
         v_dic, = hop.inputargs(self)
         r_list = hop.r_result
@@ -654,11 +659,14 @@
     # make a 'new_size' estimate and shrink it if there are many
     # deleted entry markers.  See CPython for why it is a good idea to
     # quadruple the dictionary size as long as it's not too big.
-    num_items = d.num_items
-    if num_items > 50000:
-        new_estimate = num_items * 2
-    else:
-        new_estimate = num_items * 4
+    # (Quadrupling comes from '(d.num_items + d.num_items + 1) * 2'
+    # as long as num_items is not too large.)
+    num_extra = min(d.num_items + 1, 30000)
+    _ll_dict_resize_to(d, num_extra)
+ll_dict_resize.oopspec = 'odict.resize(d)'
+
+def _ll_dict_resize_to(d, num_extra):
+    new_estimate = (d.num_items + num_extra) * 2
     new_size = DICT_INITSIZE
     while new_size <= new_estimate:
         new_size *= 2
@@ -667,7 +675,6 @@
         ll_dict_remove_deleted_items(d)
     else:
         ll_dict_reindex(d, new_size)
-ll_dict_resize.oopspec = 'odict.resize(d)'
 
 def ll_dict_reindex(d, new_size):
     ll_malloc_indexes_and_choose_lookup(d, new_size)
@@ -982,6 +989,9 @@
 ll_dict_clear.oopspec = 'odict.clear(d)'
 
 def ll_dict_update(dic1, dic2):
+    if dic1 == dic2:
+        return
+    ll_prepare_dict_update(dic1, dic2.num_items)
     i = 0
     while i < dic2.num_used_items:
         entries = dic2.entries
@@ -995,6 +1005,16 @@
         i += 1
 ll_dict_update.oopspec = 'odict.update(dic1, dic2)'
 
+def ll_prepare_dict_update(d, num_extra):
+    # Prescale 'd' for 'num_extra' items, assuming that most items don't
+    # collide.  If this assumption is false, 'd' becomes too large by at
+    # most 'num_extra'.  The logic is based on:
+    #      (d.resize_counter - 1) // 3 = room left in d
+    #  so, if num_extra == 1, we need d.resize_counter > 3
+    #      if num_extra == 2, we need d.resize_counter > 6  etc.
+    jit.conditional_call(d.resize_counter <= num_extra * 3,
+                         _ll_dict_resize_to, d, num_extra)
+
 # this is an implementation of keys(), values() and items()
 # in a single function.
 # note that by specialization on func, three different

From noreply at buildbot.pypy.org  Fri Dec 12 16:35:30 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 12 Dec 2014 16:35:30 +0100 (CET)
Subject: [pypy-commit] pypy default: kill exported_state as a thing living
 on a label, instead pass it explicitely
Message-ID: <20141212153530.464891C0675@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74898:99e731e67c69
Date: 2014-12-12 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/99e731e67c69/

Log:	kill exported_state as a thing living on a label, instead pass it
	explicitely when it's necessary (when unrolling *or* retracing)

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -74,7 +74,6 @@
             # for a CALL_ASSEMBLER: record it as a potential jump.
             if descr is not original_jitcell_token:
                 original_jitcell_token.record_jump_to(descr)
-            descr.exported_state = None
             op.cleardescr()    # clear reference, mostly for tests
         elif isinstance(descr, TargetToken):
             # for a JUMP: record it as a potential jump.
@@ -84,10 +83,6 @@
             if descr.original_jitcell_token is not original_jitcell_token:
                 assert descr.original_jitcell_token is not None
                 original_jitcell_token.record_jump_to(descr.original_jitcell_token)
-            # exported_state is clear by optimizeopt when the short preamble is
-            # constrcucted. if that did not happen the label should not show up
-            # in a trace that will be used
-            assert descr.exported_state is None
             if not we_are_translated():
                 op._descr_wref = weakref.ref(op._descr)
             op.cleardescr()    # clear reference to prevent the history.Stats
@@ -133,7 +128,7 @@
                       [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
 
     try:
-        optimize_trace(metainterp_sd, part, enable_opts)
+        start_state = optimize_trace(metainterp_sd, part, enable_opts)
     except InvalidLoop:
         return None
     target_token = part.operations[0].getdescr()
@@ -160,7 +155,8 @@
         jumpargs = part.operations[-1].getarglist()
 
         try:
-            optimize_trace(metainterp_sd, part, enable_opts)
+            optimize_trace(metainterp_sd, part, enable_opts,
+                           start_state=start_state)
         except InvalidLoop:
             return None
 
@@ -187,7 +183,7 @@
 
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,
-                    partial_trace, resumekey):
+                    partial_trace, resumekey, start_state):
     """Try to compile a new procedure by closing the current history back
     to the first operation.
     """
@@ -212,18 +208,19 @@
     orignial_label = label.clone()
     assert label.getopnum() == rop.LABEL
     try:
-        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts)
+        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
+                       start_state=start_state)
     except InvalidLoop:
         # Fall back on jumping to preamble
         target_token = label.getdescr()
         assert isinstance(target_token, TargetToken)
-        assert target_token.exported_state
         part.operations = [orignial_label] + \
                           [ResOperation(rop.JUMP, inputargs[:],
                                         None, descr=loop_jitcell_token)]
         try:
-            optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
-                           inline_short_preamble=False)
+            optimize_trace(metainterp_sd, part,
+                           jitdriver_sd.warmstate.enable_opts,
+                           inline_short_preamble=False, start_state=start_state)
         except InvalidLoop:
             return None
     assert part.operations[-1].getopnum() != rop.LABEL
@@ -792,7 +789,7 @@
     else:
         inline_short_preamble = True
     try:
-        optimize_trace(metainterp_sd, new_trace, state.enable_opts, inline_short_preamble)
+        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts, inline_short_preamble)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
@@ -808,7 +805,7 @@
         record_loop_or_bridge(metainterp_sd, new_trace)
         return target_token
     else:
-        metainterp.retrace_needed(new_trace)
+        metainterp.retrace_needed(new_trace, state)
         return None
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -617,7 +617,6 @@
         self.original_jitcell_token = None
 
         self.virtual_state = None
-        self.exported_state = None
         self.short_preamble = None
 
     def repr_of_descr(self):
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -178,13 +178,6 @@
             fail_args = ''
         return s_offset + res + op.getopname() + '(' + args + ')' + fail_args
 
-    def _log_inputarg_setup_ops(self, op):
-        target_token = op.getdescr()
-        if isinstance(target_token, TargetToken):
-            if target_token.exported_state:
-                for op in target_token.exported_state.inputarg_setup_ops:
-                    debug_print('    ' + self.repr_of_resop(op))
-
     def _log_operations(self, inputargs, operations, ops_offset):
         if not have_debug_prints():
             return
@@ -194,10 +187,10 @@
             args = ", ".join([self.repr_of_arg(arg) for arg in inputargs])
             debug_print('[' + args + ']')
         for i in range(len(operations)):
-            op = operations[i]
+            #op = operations[i]
             debug_print(self.repr_of_resop(operations[i], ops_offset))
-            if op.getopnum() == rop.LABEL:
-                self._log_inputarg_setup_ops(op)
+            #if op.getopnum() == rop.LABEL:
+            #    self._log_inputarg_setup_ops(op)
         if ops_offset and None in ops_offset:
             offset = ops_offset[None]
             debug_print("+%d: --end of the loop--" % offset)
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -47,7 +47,8 @@
 
     return optimizations, unroll
 
-def optimize_trace(metainterp_sd, loop, enable_opts, inline_short_preamble=True):
+def optimize_trace(metainterp_sd, loop, enable_opts,
+                   inline_short_preamble=True, start_state=None):
     """Optimize loop.operations to remove internal overheadish operations.
     """
 
@@ -57,7 +58,8 @@
                                                           loop.operations)
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
-            optimize_unroll(metainterp_sd, loop, optimizations, inline_short_preamble)
+            return optimize_unroll(metainterp_sd, loop, optimizations,
+                                   inline_short_preamble, start_state)
         else:
             optimizer = Optimizer(metainterp_sd, loop, optimizations)
             optimizer.propagate_all_forward()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -32,6 +32,7 @@
                   if op.getopnum()==rop.LABEL]
         prv = 0
         last_label = []
+        state = None
         for nxt in labels + [len(loop.operations)]:
             assert prv != nxt
             operations = last_label + loop.operations[prv:nxt]
@@ -44,7 +45,7 @@
             part.operations = operations
 
             self.add_guard_future_condition(part)
-            self._do_optimize_loop(part, None)
+            state = self._do_optimize_loop(part, None, state)
             if part.operations[-1].getopnum() == rop.LABEL:
                 last_label = [part.operations.pop()]
             else:
@@ -496,7 +497,7 @@
 
 class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel):
 
-    def _do_optimize_loop(self, loop, call_pure_results):
+    def _do_optimize_loop(self, loop, call_pure_results, state):
         from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
         from rpython.jit.metainterp.optimizeopt.util import args_dict
         from rpython.jit.metainterp.optimizeopt.pure import OptPure
@@ -504,7 +505,7 @@
         self.loop = loop
         loop.call_pure_results = args_dict()
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True)
+        return optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True, state)
 
     def test_optimizer_renaming_boxes1(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -390,7 +390,7 @@
         assert equaloplists(optimized.operations,
                             expected.operations, False, remap, text_right)
 
-    def _do_optimize_loop(self, loop, call_pure_results):
+    def _do_optimize_loop(self, loop, call_pure_results, start_state=None):
         from rpython.jit.metainterp.optimizeopt import optimize_trace
         from rpython.jit.metainterp.optimizeopt.util import args_dict
 
@@ -405,7 +405,8 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        optimize_trace(metainterp_sd, loop, self.enable_opts)
+        return optimize_trace(metainterp_sd, loop, self.enable_opts,
+                              start_state=start_state)
 
     def unroll_and_optimize(self, loop, call_pure_results=None):
         self.add_guard_future_condition(loop)
@@ -425,7 +426,7 @@
         preamble.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(token))] + \
                               operations +  \
                               [ResOperation(rop.LABEL, jump_args, None, descr=token)]
-        self._do_optimize_loop(preamble, call_pure_results)
+        start_state = self._do_optimize_loop(preamble, call_pure_results)
 
         assert preamble.operations[-1].getopnum() == rop.LABEL
 
@@ -439,7 +440,7 @@
         assert loop.operations[0].getopnum() == rop.LABEL
         loop.inputargs = loop.operations[0].getarglist()
 
-        self._do_optimize_loop(loop, call_pure_results)
+        self._do_optimize_loop(loop, call_pure_results, start_state)
         extra_same_as = []
         while loop.operations[0].getopnum() != rop.LABEL:
             extra_same_as.append(loop.operations[0])
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -15,10 +15,11 @@
 
 # FIXME: Introduce some VirtualOptimizer super class instead
 
-def optimize_unroll(metainterp_sd, loop, optimizations, inline_short_preamble=True):
+def optimize_unroll(metainterp_sd, loop, optimizations,
+                    inline_short_preamble=True, start_state=None):
     opt = UnrollOptimizer(metainterp_sd, loop, optimizations)
     opt.inline_short_preamble = inline_short_preamble
-    opt.propagate_all_forward()
+    return opt.propagate_all_forward(start_state)
 
 
 class UnrollableOptimizer(Optimizer):
@@ -69,7 +70,7 @@
         prev = self.fix_snapshot(jump_args, snapshot.prev)
         return Snapshot(prev, new_snapshot_args)
 
-    def propagate_all_forward(self):
+    def propagate_all_forward(self, starting_state):
         loop = self.optimizer.loop
         self.optimizer.clear_newoperations()
 
@@ -94,7 +95,7 @@
         else:
             jumpop = None
 
-        self.import_state(start_label)
+        self.import_state(start_label, starting_state)
         self.optimizer.propagate_all_forward(clear=False)
 
         if not jumpop:
@@ -147,8 +148,9 @@
         KillHugeIntBounds(self.optimizer).apply()
 
         loop.operations = self.optimizer.get_newoperations()
-        self.export_state(stop_label)
+        final_state = self.export_state(stop_label)
         loop.operations.append(stop_label)
+        return final_state
 
     def jump_to_start_label(self, start_label, stop_label):
         if not start_label or not stop_label:
@@ -202,10 +204,9 @@
                 box = op.result
                 exported_values[box] = self.optimizer.getvalue(box)
 
-        target_token.exported_state = ExportedState(short_boxes, inputarg_setup_ops,
-                                                    exported_values)
+        return ExportedState(short_boxes, inputarg_setup_ops, exported_values)
 
-    def import_state(self, targetop):
+    def import_state(self, targetop, exported_state):
         if not targetop: # Trace did not start with a label
             self.inputargs = self.optimizer.loop.inputargs
             self.short = None
@@ -215,7 +216,6 @@
         self.inputargs = targetop.getarglist()
         target_token = targetop.getdescr()
         assert isinstance(target_token, TargetToken)
-        exported_state = target_token.exported_state
         if not exported_state:
             # No state exported, construct one without virtuals
             self.short = None
@@ -413,7 +413,6 @@
             if op.result:
                 op.result.forget_value()
         target_token.short_preamble = self.short
-        target_token.exported_state = None
 
     def ensure_short_op_emitted(self, op, optimizer, seen):
         if op is None:
@@ -561,7 +560,9 @@
 
             try:
                 # NB: the short_preamble ends with a jump
-                self._inline_short_preamble(target.short_preamble, inliner, patchguardop, target.assumed_classes)
+                self._inline_short_preamble(target.short_preamble, inliner,
+                                            patchguardop,
+                                            target.assumed_classes)
             except InvalidLoop:
                 #debug_print("Inlining failed unexpectedly",
                 #            "jumping to preamble instead")
@@ -572,7 +573,8 @@
         debug_stop('jit-log-virtualstate')
         return False
 
-    def _inline_short_preamble(self, short_preamble, inliner, patchguardop, assumed_classes):
+    def _inline_short_preamble(self, short_preamble, inliner, patchguardop,
+                               assumed_classes):
         i = 1
         # XXX this is intentiontal :-(. short_preamble can change during the
         # loop in some cases
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1730,6 +1730,7 @@
 class MetaInterp(object):
     portal_call_depth = 0
     cancel_count = 0
+    exported_state = None
 
     def __init__(self, staticdata, jitdriver_sd):
         self.staticdata = staticdata
@@ -1750,9 +1751,10 @@
         self.call_ids = []
         self.current_call_id = 0
 
-    def retrace_needed(self, trace):
+    def retrace_needed(self, trace, exported_state):
         self.partial_trace = trace
         self.retracing_from = len(self.history.operations) - 1
+        self.exported_state = exported_state
         self.heapcache.reset()
 
 
@@ -2249,7 +2251,9 @@
                         raise SwitchToBlackhole(Counters.ABORT_BAD_LOOP) # For now
                 # Found!  Compile it as a loop.
                 # raises in case it works -- which is the common case
-                self.compile_loop(original_boxes, live_arg_boxes, start)
+                self.compile_loop(original_boxes, live_arg_boxes, start,
+                                  exported_state=self.exported_state)
+                self.exported_state = None
                 # creation of the loop was cancelled!
                 self.cancel_count += 1
                 if self.staticdata.warmrunnerdesc:
@@ -2363,7 +2367,7 @@
         return token
 
     def compile_loop(self, original_boxes, live_arg_boxes, start,
-                     try_disabling_unroll=False):
+                     try_disabling_unroll=False, exported_state=None):
         num_green_args = self.jitdriver_sd.num_green_args
         greenkey = original_boxes[:num_green_args]
         if not self.partial_trace:
@@ -2377,7 +2381,8 @@
                                                    original_boxes[num_green_args:],
                                                    live_arg_boxes[num_green_args:],
                                                    self.partial_trace,
-                                                   self.resumekey)
+                                                   self.resumekey,
+                                                   exported_state)
         else:
             target_token = compile.compile_loop(self, greenkey, start,
                                                 original_boxes[num_green_args:],

From noreply at buildbot.pypy.org  Fri Dec 12 16:35:31 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 12 Dec 2014 16:35:31 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20141212153531.885F91C0675@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74899:ac553d5be503
Date: 2014-12-12 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/ac553d5be503/

Log:	merge

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -19,7 +19,7 @@
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
 from rpython.annotator.argument import simple_args
-from rpython.rlib.objectmodel import r_dict, Symbolic
+from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper import extregistry
 
@@ -260,21 +260,23 @@
                     result.listdef.generalize(self.immutablevalue(e))
                 result.const_box = key
                 return result
-        elif tp is dict or tp is r_dict or tp is SomeOrderedDict.knowntype:
-            if tp is SomeOrderedDict.knowntype:
-                cls = SomeOrderedDict
-            else:
-                cls = SomeDict
+        elif (tp is dict or tp is r_dict or
+              tp is SomeOrderedDict.knowntype or tp is r_ordereddict):
             key = Constant(x)
             try:
                 return self.immutable_cache[key]
             except KeyError:
+                if tp is SomeOrderedDict.knowntype or tp is r_ordereddict:
+                    cls = SomeOrderedDict
+                else:
+                    cls = SomeDict
+                is_r_dict = issubclass(tp, r_dict)
                 result = cls(DictDef(self,
                                         s_ImpossibleValue,
                                         s_ImpossibleValue,
-                                        is_r_dict = tp is r_dict))
+                                        is_r_dict = is_r_dict))
                 self.immutable_cache[key] = result
-                if tp is r_dict:
+                if is_r_dict:
                     s_eqfn = self.immutablevalue(x.key_eq)
                     s_hashfn = self.immutablevalue(x.key_hash)
                     result.dictdef.dictkey.update_rdict_annotations(s_eqfn,
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -329,6 +329,18 @@
         res = self.interpret(g, [3])
         assert res == 42     # "did not crash"
 
+    def test_prepare_dict_update_2(self):
+        try:
+            from collections import OrderedDict
+        except ImportError:     # Python 2.6
+            py.test.skip("requires collections.OrderedDict")
+        def g(n):
+            d = OrderedDict()
+            prepare_dict_update(d, n)
+            return 42
+        res = self.interpret(g, [3])
+        assert res == 42     # "did not crash"
+
     def test_compute_hash(self):
         class Foo(object):
             pass
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -217,7 +217,7 @@
         #dictobj = getattr(dictobj, '__self__', dictobj)
         if dictobj is None:
             return lltype.nullptr(self.DICT)
-        if not isinstance(dictobj, (dict, objectmodel.r_dict)):
+        if not isinstance(dictobj, (dict, objectmodel.r_ordereddict)):
             raise TypeError("expected a dict: %r" % (dictobj,))
         try:
             key = Constant(dictobj)
@@ -231,7 +231,8 @@
             if r_key.lowleveltype == llmemory.Address:
                 raise TypeError("No prebuilt dicts of address keys")
             r_value = self.value_repr
-            if isinstance(dictobj, objectmodel.r_dict):
+            if isinstance(dictobj, objectmodel.r_ordereddict):
+                
                 if self.r_rdict_eqfn.lowleveltype != lltype.Void:
                     l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq)
                     l_dict.fnkeyeq = l_fn
@@ -289,6 +290,11 @@
         hop.exception_cannot_occur()
         return hop.gendirectcall(ll_dict_update, v_dic1, v_dic2)
 
+    def rtype_method__prepare_dict_update(self, hop):
+        v_dict, v_num = hop.inputargs(self, lltype.Signed)
+        hop.exception_cannot_occur()
+        hop.gendirectcall(ll_prepare_dict_update, v_dict, v_num)
+
     def _rtype_method_kvi(self, hop, ll_func):
         v_dic, = hop.inputargs(self)
         r_list = hop.r_result
@@ -653,11 +659,14 @@
     # make a 'new_size' estimate and shrink it if there are many
     # deleted entry markers.  See CPython for why it is a good idea to
     # quadruple the dictionary size as long as it's not too big.
-    num_items = d.num_items
-    if num_items > 50000:
-        new_estimate = num_items * 2
-    else:
-        new_estimate = num_items * 4
+    # (Quadrupling comes from '(d.num_items + d.num_items + 1) * 2'
+    # as long as num_items is not too large.)
+    num_extra = min(d.num_items + 1, 30000)
+    _ll_dict_resize_to(d, num_extra)
+ll_dict_resize.oopspec = 'odict.resize(d)'
+
+def _ll_dict_resize_to(d, num_extra):
+    new_estimate = (d.num_items + num_extra) * 2
     new_size = DICT_INITSIZE
     while new_size <= new_estimate:
         new_size *= 2
@@ -666,7 +675,6 @@
         ll_dict_remove_deleted_items(d)
     else:
         ll_dict_reindex(d, new_size)
-ll_dict_resize.oopspec = 'odict.resize(d)'
 
 def ll_dict_reindex(d, new_size):
     ll_malloc_indexes_and_choose_lookup(d, new_size)
@@ -981,6 +989,9 @@
 ll_dict_clear.oopspec = 'odict.clear(d)'
 
 def ll_dict_update(dic1, dic2):
+    if dic1 == dic2:
+        return
+    ll_prepare_dict_update(dic1, dic2.num_items)
     i = 0
     while i < dic2.num_used_items:
         entries = dic2.entries
@@ -994,6 +1005,16 @@
         i += 1
 ll_dict_update.oopspec = 'odict.update(dic1, dic2)'
 
+def ll_prepare_dict_update(d, num_extra):
+    # Prescale 'd' for 'num_extra' items, assuming that most items don't
+    # collide.  If this assumption is false, 'd' becomes too large by at
+    # most 'num_extra'.  The logic is based on:
+    #      (d.resize_counter - 1) // 3 = room left in d
+    #  so, if num_extra == 1, we need d.resize_counter > 3
+    #      if num_extra == 2, we need d.resize_counter > 6  etc.
+    jit.conditional_call(d.resize_counter <= num_extra * 3,
+                         _ll_dict_resize_to, d, num_extra)
+
 # this is an implementation of keys(), values() and items()
 # in a single function.
 # note that by specialization on func, three different
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -878,6 +878,81 @@
         res = self.interpret(func, [])
         assert lltype.typeOf(res.item0) == lltype.typeOf(res.item1)
 
+    def test_r_dict(self):
+        class FooError(Exception):
+            pass
+        def myeq(n, m):
+            return n == m
+        def myhash(n):
+            if n < 0:
+                raise FooError
+            return -n
+        def f(n):
+            d = self.new_r_dict(myeq, myhash)
+            for i in range(10):
+                d[i] = i*i
+            try:
+                value1 = d[n]
+            except FooError:
+                value1 = 99
+            try:
+                value2 = n in d
+            except FooError:
+                value2 = 99
+            try:
+                value3 = d[-n]
+            except FooError:
+                value3 = 99
+            try:
+                value4 = (-n) in d
+            except FooError:
+                value4 = 99
+            return (value1 * 1000000 +
+                    value2 * 10000 +
+                    value3 * 100 +
+                    value4)
+        res = self.interpret(f, [5])
+        assert res == 25019999
+
+    def test_r_dict_popitem_hash(self):
+        def deq(n, m):
+            return n == m
+        def dhash(n):
+            return ~n
+        def func():
+            d = self.new_r_dict(deq, dhash)
+            d[5] = 2
+            d[6] = 3
+            k1, v1 = d.popitem()
+            assert len(d) == 1
+            k2, v2 = d.popitem()
+            try:
+                d.popitem()
+            except KeyError:
+                pass
+            else:
+                assert 0, "should have raised KeyError"
+            assert len(d) == 0
+            return k1*1000 + v1*100 + k2*10 + v2
+
+        res = self.interpret(func, [])
+        assert res in [5263, 6352]
+
+    def test_prebuilt_r_dict(self):
+        def deq(n, m):
+            return (n & 3) == (m & 3)
+        def dhash(n):
+            return n & 3
+        d = self.new_r_dict(deq, dhash)
+        d[0x123] = "abcd"
+        d[0x231] = "efgh"
+        def func():
+            return d[0x348973] + d[0x12981]
+
+        res = self.interpret(func, [])
+        res = self.ll_to_string(res)
+        assert res == "abcdefgh"
+
 
 class TestRDict(BaseTestRDict):
     @staticmethod
@@ -888,6 +963,10 @@
     def newdict2():
         return {}
 
+    @staticmethod
+    def new_r_dict(myeq, myhash):
+        return r_dict(myeq, myhash)
+
     def test_two_dicts_with_different_value_types(self):
         def func(i):
             d1 = {}
@@ -1043,66 +1122,6 @@
         assert lltype.typeOf(res.item1) == lltype.typeOf(res.item2)
         assert lltype.typeOf(res.item1) == lltype.typeOf(res.item3)
 
-    def test_r_dict(self):
-        class FooError(Exception):
-            pass
-        def myeq(n, m):
-            return n == m
-        def myhash(n):
-            if n < 0:
-                raise FooError
-            return -n
-        def f(n):
-            d = r_dict(myeq, myhash)
-            for i in range(10):
-                d[i] = i*i
-            try:
-                value1 = d[n]
-            except FooError:
-                value1 = 99
-            try:
-                value2 = n in d
-            except FooError:
-                value2 = 99
-            try:
-                value3 = d[-n]
-            except FooError:
-                value3 = 99
-            try:
-                value4 = (-n) in d
-            except FooError:
-                value4 = 99
-            return (value1 * 1000000 +
-                    value2 * 10000 +
-                    value3 * 100 +
-                    value4)
-        res = self.interpret(f, [5])
-        assert res == 25019999
-
-    def test_dict_popitem_hash(self):
-        def deq(n, m):
-            return n == m
-        def dhash(n):
-            return ~n
-        def func():
-            d = r_dict(deq, dhash)
-            d[5] = 2
-            d[6] = 3
-            k1, v1 = d.popitem()
-            assert len(d) == 1
-            k2, v2 = d.popitem()
-            try:
-                d.popitem()
-            except KeyError:
-                pass
-            else:
-                assert 0, "should have raised KeyError"
-            assert len(d) == 0
-            return k1*1000 + v1*100 + k2*10 + v2
-
-        res = self.interpret(func, [])
-        assert res in [5263, 6352]
-
     def test_nonnull_hint(self):
         def eq(a, b):
             return a == b
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -268,6 +268,10 @@
     def newdict2():
         return OrderedDict()
 
+    @staticmethod
+    def new_r_dict(myeq, myhash):
+        return objectmodel.r_ordereddict(myeq, myhash)
+
     def test_two_dicts_with_different_value_types(self):
         def func(i):
             d1 = OrderedDict()
@@ -283,64 +287,3 @@
 
     def test_memoryerror_should_not_insert(self):
         py.test.skip("I don't want to edit this file on two branches")
-
-
-    def test_r_dict(self):
-        class FooError(Exception):
-            pass
-        def myeq(n, m):
-            return n == m
-        def myhash(n):
-            if n < 0:
-                raise FooError
-            return -n
-        def f(n):
-            d = objectmodel.r_ordereddict(myeq, myhash)
-            for i in range(10):
-                d[i] = i*i
-            try:
-                value1 = d[n]
-            except FooError:
-                value1 = 99
-            try:
-                value2 = n in d
-            except FooError:
-                value2 = 99
-            try:
-                value3 = d[-n]
-            except FooError:
-                value3 = 99
-            try:
-                value4 = (-n) in d
-            except FooError:
-                value4 = 99
-            return (value1 * 1000000 +
-                    value2 * 10000 +
-                    value3 * 100 +
-                    value4)
-        res = self.interpret(f, [5])
-        assert res == 25019999
-
-    def test_dict_popitem_hash(self):
-        def deq(n, m):
-            return n == m
-        def dhash(n):
-            return ~n
-        def func():
-            d = objectmodel.r_ordereddict(deq, dhash)
-            d[5] = 2
-            d[6] = 3
-            k1, v1 = d.popitem()
-            assert len(d) == 1
-            k2, v2 = d.popitem()
-            try:
-                d.popitem()
-            except KeyError:
-                pass
-            else:
-                assert 0, "should have raised KeyError"
-            assert len(d) == 0
-            return k1*1000 + v1*100 + k2*10 + v2
-
-        res = self.interpret(func, [])
-        assert res in [5263, 6352]

From noreply at buildbot.pypy.org  Fri Dec 12 16:48:31 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 12 Dec 2014 16:48:31 +0100 (CET)
Subject: [pypy-commit] pypy default: fix typo
Message-ID: <20141212154831.3A74E1C0675@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74900:59fae35bdef8
Date: 2014-12-12 17:48 +0200
http://bitbucket.org/pypy/pypy/changeset/59fae35bdef8/

Log:	fix typo

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -585,13 +585,13 @@
 
 
 class ShortBoxes(object):
-    def __init__(self, optimizer, surviving_boxes, availible_boxes=None):
+    def __init__(self, optimizer, surviving_boxes, available_boxes=None):
         self.potential_ops = {}
         self.alternatives = {}
         self.synthetic = {}
         self.rename = {}
         self.optimizer = optimizer
-        self.availible_boxes = availible_boxes
+        self.available_boxes = available_boxes
         self.assumed_classes = {}
 
         if surviving_boxes is not None:
@@ -663,7 +663,7 @@
             return
         if box in self.short_boxes_in_production:
             raise BoxNotProducable
-        if self.availible_boxes is not None and box not in self.availible_boxes:
+        if self.available_boxes is not None and box not in self.available_boxes:
             raise BoxNotProducable
         self.short_boxes_in_production[box] = None
 

From noreply at buildbot.pypy.org  Fri Dec 12 17:24:07 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Dec 2014 17:24:07 +0100 (CET)
Subject: [pypy-commit] pypy default: clarify _sqlite3 change
Message-ID: <20141212162407.819F11D3875@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74901:ff868c93a1a5
Date: 2014-12-12 11:23 -0500
http://bitbucket.org/pypy/pypy/changeset/ff868c93a1a5/

Log:	clarify _sqlite3 change

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1175,11 +1175,9 @@
         try:
             return self.__description
         except AttributeError:
-            try:
+            if self.__statement:
                 self.__description = self.__statement._get_description()
                 return self.__description
-            except AttributeError:
-                return None
     description = property(__get_description)
 
     def __get_lastrowid(self):

From noreply at buildbot.pypy.org  Fri Dec 12 17:43:24 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 12 Dec 2014 17:43:24 +0100 (CET)
Subject: [pypy-commit] pypy optresult: small progress
Message-ID: <20141212164324.B7FD21C0188@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74902:c9f5a43ea3c1
Date: 2014-12-12 18:39 +0200
http://bitbucket.org/pypy/pypy/changeset/c9f5a43ea3c1/

Log:	small progress

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -847,6 +847,7 @@
     calls back the interpreter.  Used temporarily: a fully compiled
     version of the code may end up replacing it.
     """
+    xxx
     jitcell_token = make_jitcell_token(jitdriver_sd)
     nb_red_args = jitdriver_sd.num_red_args
     assert len(redargtypes) == nb_red_args
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -185,7 +185,7 @@
                 for op in target_token.exported_state.inputarg_setup_ops:
                     debug_print('    ' + self.repr_of_resop(op))
 
-    def _log_operations(self, inputargs, operations, ops_offset):
+    def _log_operations(self, inputargs, operations, ops_offset=None):
         if not have_debug_prints():
             return
         if ops_offset is None:
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -161,7 +161,7 @@
                     getop = ResOperation(rop.GETARRAYITEM_GC, [op.getarg(0), op.getarg(1)],
                                          result, op.getdescr())
                     shortboxes.add_potential(getop, synthetic=True)
-                elif op.result is not None:
+                elif op.type != 'v':
                     xxxx
                     shortboxes.add_potential(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -1,11 +1,13 @@
 from rpython.jit.metainterp import jitprof, resume, compile
 from rpython.jit.metainterp.executor import execute_nonspec_const
+from rpython.jit.metainterp.logger import LogOperations
 from rpython.jit.metainterp.history import Const, ConstInt, REF
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded, \
                                                      ImmutableIntUnbounded, \
                                                      IntLowerBound, MININT, MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractResOp
+from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+     AbstractResOp, AbstractInputArg, DONT_CHANGE
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.tool.pairtype import extendabletype
 from rpython.rlib.debug import debug_print
@@ -375,6 +377,7 @@
         self.metainterp_sd = metainterp_sd
         self.cpu = metainterp_sd.cpu
         self.loop = loop
+        self.logops = LogOperations(metainterp_sd, False)
         self.values = {}
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.interned_ints = {}
@@ -506,8 +509,15 @@
 
     def replace_op_with(self, op, newopnum, args=None, descr=None):
         v = self.getvalue(op)
-        newop = op.copy_and_change(newopnum, args=args, descr=descr)
-        v.box = newop
+        if isinstance(op, AbstractInputArg):
+            if descr is DONT_CHANGE:
+                descr = None
+            newop = ResOperation(newopnum, args, descr)
+            newop.source_op = op
+        else:
+            newop = op.copy_and_change(newopnum, args=args, descr=descr)
+        if v.box is op: # don't replace if it's already replaced
+            v.box = newop
         return newop
 
     def make_constant_int(self, box, intvalue):
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -586,10 +586,10 @@
         self.pure(rop.CAST_PTR_TO_INT, [op], op.getarg(0))
         self.emit_operation(op)
 
-    def optimize_SAME_AS_i(self, op):
-        self.make_equal_to(op.result, self.getvalue(op.getarg(0)))
-    optimize_SAME_AS_r = optimize_SAME_AS_i
-    optimize_SAME_AS_f = optimize_SAME_AS_i
+    def optimize_SAME_AS_I(self, op):
+        self.make_equal_to(op, self.getvalue(op.getarg(0)))
+    optimize_SAME_AS_R = optimize_SAME_AS_I
+    optimize_SAME_AS_F = optimize_SAME_AS_I
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -256,14 +256,14 @@
         ops = """
         [i0]
         p0 = new_with_vtable(ConstClass(node_vtable))
-        escape(p0)
+        escape_n(p0)
         guard_class(p0, ConstClass(node_vtable)) []
         jump(i0)
         """
         expected = """
         [i0]
         p0 = new_with_vtable(ConstClass(node_vtable))
-        escape(p0)
+        escape_n(p0)
         jump(i0)
         """
         self.optimize_loop(ops, expected)
@@ -271,7 +271,7 @@
     def test_remove_guard_class_constant(self):
         ops = """
         [i0]
-        p0 = same_as(ConstPtr(myptr))
+        p0 = same_as_r(ConstPtr(myptr))
         guard_class(p0, ConstClass(node_vtable)) []
         jump(i0)
         """
@@ -368,19 +368,19 @@
     def test_remove_consecutive_guard_value_constfold(self):
         ops = """
         []
-        i0 = escape()
+        i0 = escape_i()
         guard_value(i0, 0) []
         i1 = int_add(i0, 1)
         guard_value(i1, 1) []
         i2 = int_add(i1, 2)
-        escape(i2)
+        escape_n(i2)
         jump()
         """
         expected = """
         []
-        i0 = escape()
+        i0 = escape_i()
         guard_value(i0, 0) []
-        escape(3)
+        escape_n(3)
         jump()
         """
         self.optimize_loop(ops, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -1,5 +1,6 @@
 import py, random
 
+from rpython.rlib.debug import debug_print
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper import rclass
 from rpython.rtyper.rclass import (
@@ -10,6 +11,7 @@
                                             JitCellToken, TargetToken)
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
 from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.metainterp.logger import LogOperations
 from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.tool.oparser import parse, pure_parse
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
@@ -406,6 +408,8 @@
         optimize_trace(metainterp_sd, loop, self.enable_opts)
 
     def unroll_and_optimize(self, loop, call_pure_results=None):
+        metainterp_sd = FakeMetaInterpStaticData(self.cpu)
+        logops = LogOperations(metainterp_sd, False)
         self.add_guard_future_condition(loop)
         operations =  loop.operations
         jumpop = operations[-1]
@@ -435,11 +439,18 @@
                           [ResOperation(rop.JUMP, [memo.get(a, a) for a in jump_args],
                                         descr=token)]
                           #[inliner.inline_op(jumpop)]
+        
         assert loop.operations[-1].getopnum() == rop.JUMP
         assert loop.operations[0].getopnum() == rop.LABEL
         loop.inputargs = loop.operations[0].getarglist()
 
+        
+        debug_print("   noopt loop")
+        logops._log_operations(loop.inputargs, loop.operations)
+        
         self._do_optimize_loop(loop, call_pure_results)
+        debug_print("   opt loop")
+        logops._log_operations(loop.inputargs, loop.operations)
         extra_same_as = []
         while loop.operations[0].getopnum() != rop.LABEL:
             extra_same_as.append(loop.operations[0])
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -2,12 +2,14 @@
 
 from rpython.jit.metainterp.compile import Memo
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
+from rpython.jit.metainterp.logger import LogOperations
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
         ShortBoxes, BadVirtualState, VirtualStatesCantMatch)
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, DONT_CHANGE,\
+     OpHelpers, AbstractInputArg
 from rpython.jit.metainterp.resume import Snapshot
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
 
@@ -183,7 +185,10 @@
             if values[i].is_virtual():
                 srcbox = values[i].force_box(self.optimizer)
             if original_jump_args[i] is not srcbox:
-                op = ResOperation(rop.SAME_AS, [srcbox], original_jump_args[i])
+                opnum = OpHelpers.same_as_for_type(original_jump_args[i].type)
+                op = self.optimizer.replace_op_with(original_jump_args[i],
+                                                    opnum, [srcbox],
+                                                    descr=DONT_CHANGE)
                 self.optimizer.emit_operation(op)
         inputarg_setup_ops = self.optimizer.get_newoperations()
 
@@ -197,12 +202,11 @@
         for box in inputargs:
             exported_values[box] = self.optimizer.getvalue(box)
         for op in short_boxes.operations():
-            if op and op.result:
-                box = op.result
-                exported_values[box] = self.optimizer.getvalue(box)
+            if op and op.type != 'v':
+                exported_values[op] = self.optimizer.getvalue(op)
 
-        target_token.exported_state = ExportedState(short_boxes, inputarg_setup_ops,
-                                                    exported_values)
+        es = ExportedState(short_boxes, inputarg_setup_ops, exported_values)
+        target_token.exported_state = es
 
     def import_state(self, targetop):
         if not targetop: # Trace did not start with a label
@@ -245,20 +249,21 @@
         seen = {}
         for op in self.short_boxes.operations():
             self.ensure_short_op_emitted(op, self.optimizer, seen)
-            if op and op.result:
-                preamble_value = exported_state.exported_values[op.result]
-                value = self.optimizer.getvalue(op.result)
+            if op and op.type != 'v':
+                preamble_value = exported_state.exported_values[op]
+                value = self.optimizer.getvalue(op)
                 if not value.is_virtual() and not value.is_constant():
                     imp = ValueImporter(self, preamble_value, op)
                     self.optimizer.importable_values[value] = imp
-                newvalue = self.optimizer.getvalue(op.result)
+                newvalue = self.optimizer.getvalue(op)
                 newresult = newvalue.get_key_box()
                 # note that emitting here SAME_AS should not happen, but
                 # in case it does, we would prefer to be suboptimal in asm
                 # to a fatal RPython exception.
-                if newresult is not op.result and \
+                if newresult is not op and \
                    not self.short_boxes.has_producer(newresult) and \
                    not newvalue.is_constant():
+                    xxx
                     op = ResOperation(rop.SAME_AS, [op.result], newresult)
                     self.optimizer._newoperations.append(op)
                     #if self.optimizer.loop.logops:
@@ -395,7 +400,7 @@
             if a in boxmap:
                 newargs[i] = boxmap[a]
             else:
-                newargs[i] = a.clonebox()
+                newargs[i] = a.clone_input_arg()
                 boxmap[a] = newargs[i]
         memo = Memo(short_inputargs, newargs)
         target_token.assumed_classes = {}
@@ -417,10 +422,10 @@
     def ensure_short_op_emitted(self, op, optimizer, seen):
         if op is None:
             return
-        if op.result is not None and op.result in seen:
+        if op.type != 'v' and op in seen:
             return
         for a in op.getarglist():
-            if not isinstance(a, Const) and a not in seen:
+            if not isinstance(a, Const) and not isinstance(a, AbstractInputArg) and a not in seen:
                 self.ensure_short_op_emitted(self.short_boxes.producer(a), optimizer,
                                              seen)
 
@@ -429,15 +434,15 @@
         #                self.optimizer.loop.logops.repr_of_resop(op))
 
         optimizer.send_extra_operation(op)
-        seen[op.result] = None
+        seen[op] = None
         if op.is_ovf():
-            guard = ResOperation(rop.GUARD_NO_OVERFLOW, [], None)
+            guard = ResOperation(rop.GUARD_NO_OVERFLOW, [])
             optimizer.send_extra_operation(guard)
 
     def add_op_to_short(self, op, emit=True, guards_needed=False):
         if op is None:
             return None
-        if op.result is not None and op.result in self.short_seen:
+        if op is not None and op in self.short_seen:
             if emit and self.short_inliner:
                 return self.short_inliner.inline_arg(op.result)
             else:
@@ -609,3 +614,14 @@
         self.short_boxes = short_boxes
         self.inputarg_setup_ops = inputarg_setup_ops
         self.exported_values = exported_values
+
+    def dump(self):
+        debug_start("jit-exported-state")
+        logops = LogOperations()
+        debug_print("   inputarg setup")
+        logops._log_operations(self.inputarg_setup_ops)
+        debug_print("   short boxes")
+        self.short_boxes.dump(logops)
+        debug_print("   exported values")
+        self.exported_values.dump(logops)
+        debug_stop("jit-exported-state")
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -1,13 +1,15 @@
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
-from rpython.jit.metainterp.history import (BoxInt, ConstInt, BoxPtr, Const,
+from rpython.jit.metainterp.history import (ConstInt, Const,
         ConstPtr, ConstFloat)
 from rpython.jit.metainterp.optimizeopt import virtualize
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 from rpython.jit.metainterp.optimizeopt.optimizer import (LEVEL_CONSTANT,
     LEVEL_KNOWNCLASS, LEVEL_NONNULL, LEVEL_UNKNOWN, OptValue)
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+     AbstractInputArg
+from rpython.jit.metainterp.compile import Memo
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
-from rpython.rlib.objectmodel import we_are_translated, import_from_mixin
+from rpython.rlib.objectmodel import we_are_translated
 
 
 class BadVirtualState(Exception):
@@ -589,7 +591,7 @@
         self.potential_ops = {}
         self.alternatives = {}
         self.synthetic = {}
-        self.rename = {}
+        self.memo = Memo()
         self.optimizer = optimizer
         self.availible_boxes = availible_boxes
         self.assumed_classes = {}
@@ -602,9 +604,9 @@
             self.short_boxes = {}
             self.short_boxes_in_production = {}
 
-            for box in self.potential_ops.keys():
+            for op in self.potential_ops.keys():
                 try:
-                    self.produce_short_preamble_box(box)
+                    self.produce_short_preamble_op(op)
                 except BoxNotProducable:
                     pass
 
@@ -614,7 +616,7 @@
 
     def prioritized_alternatives(self, box):
         if box not in self.alternatives:
-            return [self.potential_ops[box]]
+            return [box]
         alts = self.alternatives[box]
         hi, lo = 0, len(alts) - 1
         while hi < lo:
@@ -628,16 +630,10 @@
                 lo -= 1
         return alts
 
-    def renamed(self, box):
-        if box in self.rename:
-            return self.rename[box]
-        return box
-
     def add_to_short(self, box, op):
         if op:
-            op = op.clone()
-            for i in range(op.numargs()):
-                op.setarg(i, self.renamed(op.getarg(i)))
+            op = op.clone(self.memo)
+            op.is_source_op = True
         if box in self.short_boxes:
             if op is None:
                 oldop = self.short_boxes[box].clone()
@@ -656,30 +652,30 @@
         else:
             self.short_boxes[box] = op
 
-    def produce_short_preamble_box(self, box):
-        if box in self.short_boxes:
+    def produce_short_preamble_op(self, op):
+        if op in self.short_boxes:
             return
-        if isinstance(box, Const):
+        if isinstance(op, Const) or isinstance(op, AbstractInputArg):
             return
-        if box in self.short_boxes_in_production:
+        if op in self.short_boxes_in_production:
             raise BoxNotProducable
-        if self.availible_boxes is not None and box not in self.availible_boxes:
+        if self.availible_boxes is not None and op not in self.availible_boxes:
             raise BoxNotProducable
-        self.short_boxes_in_production[box] = None
+        self.short_boxes_in_production[op] = None
 
-        if box in self.potential_ops:
-            ops = self.prioritized_alternatives(box)
+        if op in self.potential_ops:
+            ops = self.prioritized_alternatives(op)
             produced_one = False
-            for op in ops:
+            for newop in ops:
                 try:
-                    if op:
-                        for arg in op.getarglist():
-                            self.produce_short_preamble_box(arg)
+                    if newop:
+                        for arg in newop.getarglist():
+                            self.produce_short_preamble_op(arg)
                 except BoxNotProducable:
                     pass
                 else:
                     produced_one = True
-                    self.add_to_short(box, op)
+                    self.add_to_short(op, newop)
             if not produced_one:
                 raise BoxNotProducable
         else:
@@ -693,10 +689,10 @@
                 if classbox:
                     self.assumed_classes[op] = classbox
         if op not in self.potential_ops:
-            self.potential_ops[op] = op
+            self.potential_ops[op] = None
         else:
             if op not in self.alternatives:
-                self.alternatives[op] = [self.potential_ops[op]]
+                self.alternatives[op] = [op]
             self.alternatives[op].append(op)
         if synthetic:
             self.synthetic[op] = True
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -387,19 +387,31 @@
 
     def getdescr(self):
         return None
-        
+
+    def forget_value(self):
+        pass
+    
 class InputArgInt(IntOp, AbstractInputArg):
     def __init__(self, intval=0):
-        self.setint(intval)            
+        self.setint(intval)
+
+    def clone_input_arg(self):
+        return InputArgInt()
 
 class InputArgFloat(FloatOp, AbstractInputArg):
     def __init__(self, f=0.0):
         self.setfloatstorage(f)
 
+    def clone_input_arg(self):
+        return InputArgFloat()
+
 class InputArgRef(RefOp, AbstractInputArg):
     def __init__(self, r=lltype.nullptr(llmemory.GCREF.TO)):
         self.setref_base(r)
 
+    def clone_input_arg(self):
+        return InputArgRef()
+
 # ============
 # arity mixins
 # ============

From noreply at buildbot.pypy.org  Fri Dec 12 17:43:30 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 12 Dec 2014 17:43:30 +0100 (CET)
Subject: [pypy-commit] pypy optresult: merge default
Message-ID: <20141212164330.0982B1C0188@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74903:30f34cca91d4
Date: 2014-12-12 18:43 +0200
http://bitbucket.org/pypy/pypy/changeset/30f34cca91d4/

Log:	merge default

diff too long, truncating to 2000 out of 14889 lines

diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -478,6 +478,29 @@
         except TypeError:
             pass
 
+    def CheckCurDescription(self):
+        self.cu.execute("select * from test")
+
+        actual = self.cu.description
+        expected = [
+            ('id', None, None, None, None, None, None),
+            ('name', None, None, None, None, None, None),
+            ('income', None, None, None, None, None, None),
+        ]
+        self.assertEqual(expected, actual)
+
+    def CheckCurDescriptionVoidStatement(self):
+        self.cu.execute("insert into test(name) values (?)", ("foo",))
+        self.assertIsNone(self.cu.description)
+
+    def CheckCurDescriptionWithoutStatement(self):
+        cu = self.cx.cursor()
+        try:
+            self.assertIsNone(cu.description)
+        finally:
+            cu.close()
+
+
 @unittest.skipUnless(threading, 'This test requires threading.')
 class ThreadTests(unittest.TestCase):
     def setUp(self):
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1175,8 +1175,11 @@
         try:
             return self.__description
         except AttributeError:
-            self.__description = self.__statement._get_description()
-            return self.__description
+            try:
+                self.__description = self.__statement._get_description()
+                return self.__description
+            except AttributeError:
+                return None
     description = property(__get_description)
 
     def __get_lastrowid(self):
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -205,23 +205,28 @@
 The above is true both in CPython and in PyPy.  Differences
 can occur about whether a built-in function or method will
 call an overridden method of *another* object than ``self``.
-In PyPy, they are generally always called, whereas not in
-CPython.  For example, in PyPy, ``dict1.update(dict2)``
-considers that ``dict2`` is just a general mapping object, and
-will thus call overridden ``keys()``  and ``__getitem__()``
-methods on it.  So the following code prints ``42`` on PyPy
-but ``foo`` on CPython::
+In PyPy, they are often called in cases where CPython would not.
+Two examples::
 
-    >>>> class D(dict):
-    ....     def __getitem__(self, key):
-    ....         return 42
-    ....
-    >>>>
-    >>>> d1 = {}
-    >>>> d2 = D(a='foo')
-    >>>> d1.update(d2)
-    >>>> print d1['a']
-    42
+    class D(dict):
+        def __getitem__(self, key):
+            return "%r from D" % (key,)
+
+    class A(object):
+        pass
+
+    a = A()
+    a.__dict__ = D()
+    a.foo = "a's own foo"
+    print a.foo
+    # CPython => a's own foo
+    # PyPy => 'foo' from D
+
+    glob = D(foo="base item")
+    loc = {}
+    exec "print foo" in glob, loc
+    # CPython => base item
+    # PyPy => 'foo' from D
 
 
 Mutating classes of objects which are already used as dictionary keys
@@ -292,6 +297,9 @@
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
+Notably missing from the list above are ``str`` and ``unicode``.  If your
+code relies on comparing strings with ``is``, then it might break in PyPy.
+
 
 Miscellaneous
 -------------
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,6 +6,10 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
+**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
+on linux, linux64 and windows. We will make it the default on all platforms
+by the time of the next release.
+
 The first thing that you need is to compile PyPy yourself with the option
 ``--shared``. We plan to make ``--shared`` the default in the future. Consult
 the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -35,6 +35,13 @@
 PyPy's bytearray type is very inefficient. It would be an interesting
 task to look into possible optimizations on this.
 
+Implement AF_XXX packet types for PyPy
+--------------------------------------
+
+PyPy is missing AF_XXX types of sockets. Implementing it is easy-to-medium
+task. `bug report`_
+
+.. _`bug report`: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets#more
 
 Implement copy-on-write list slicing
 ------------------------------------
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,15 +1,13 @@
-"""
-Python control flow graph generation and bytecode assembly.
-"""
+"""Python control flow graph generation and bytecode assembly."""
 
-from pypy.interpreter.astcompiler import ast, symtable
-from pypy.interpreter import pycode
+from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import we_are_translated
+
+from pypy.interpreter.astcompiler import ast, misc, symtable
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pycode import PyCode
 from pypy.tool import stdlib_opcode as ops
 
-from pypy.interpreter.error import OperationError
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import rfloat
-
 
 class Instruction(object):
     """Represents a single opcode."""
@@ -21,14 +19,12 @@
         self.has_jump = False
 
     def size(self):
-        """Return the size of bytes of this instruction when it is encoded."""
+        """Return the size of bytes of this instruction when it is
+        encoded.
+        """
         if self.opcode >= ops.HAVE_ARGUMENT:
-            if self.arg > 0xFFFF:
-                return 6
-            else:
-                return 3
-        else:
-            return 1
+            return (6 if self.arg > 0xFFFF else 3)
+        return 1
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -54,9 +50,9 @@
 class Block(object):
     """A basic control flow block.
 
-    It has one entry point and several possible exit points.  Its instructions
-    may be jumps to other blocks, or if control flow reaches the end of the
-    block, it continues to next_block.
+    It has one entry point and several possible exit points.  Its
+    instructions may be jumps to other blocks, or if control flow
+    reaches the end of the block, it continues to next_block.
     """
 
     def __init__(self):
@@ -71,10 +67,10 @@
             stack.append(nextblock)
 
     def post_order(self):
-        """Return this block and its children in post order.
-        This means that the graph of blocks is first cleaned up to
-        ignore back-edges, thus turning it into a DAG.  Then the DAG
-        is linearized.  For example:
+        """Return this block and its children in post order.  This means
+        that the graph of blocks is first cleaned up to ignore
+        back-edges, thus turning it into a DAG.  Then the DAG is
+        linearized.  For example:
 
                    A --> B -\           =>     [A, D, B, C]
                      \-> D ---> C
@@ -105,7 +101,9 @@
         return resultblocks
 
     def code_size(self):
-        """Return the encoded size of all the instructions in this block."""
+        """Return the encoded size of all the instructions in this
+        block.
+        """
         i = 0
         for instr in self.instructions:
             i += instr.size()
@@ -141,6 +139,7 @@
             i += 1
     return result
 
+
 def _list_to_dict(l, offset=0):
     result = {}
     index = offset
@@ -300,11 +299,11 @@
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
         last_extended_arg_count = 0
-        # The reason for this loop is extended jumps.  EXTENDED_ARG extends the
-        # bytecode size, so it might invalidate the offsets we've already given.
-        # Thus we have to loop until the number of extended args is stable.  Any
-        # extended jump at all is extremely rare, so performance is not too
-        # concerning.
+        # The reason for this loop is extended jumps.  EXTENDED_ARG
+        # extends the bytecode size, so it might invalidate the offsets
+        # we've already given.  Thus we have to loop until the number of
+        # extended args is stable.  Any extended jump at all is
+        # extremely rare, so performance is not too concerning.
         while True:
             extended_arg_count = 0
             offset = 0
@@ -330,7 +329,8 @@
                                     instr.opcode = ops.JUMP_ABSOLUTE
                                     absolute = True
                                 elif target_op == ops.RETURN_VALUE:
-                                    # Replace JUMP_* to a RETURN into just a RETURN
+                                    # Replace JUMP_* to a RETURN into
+                                    # just a RETURN
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
@@ -345,7 +345,8 @@
                         instr.arg = jump_arg
                         if jump_arg > 0xFFFF:
                             extended_arg_count += 1
-            if extended_arg_count == last_extended_arg_count and not force_redo:
+            if (extended_arg_count == last_extended_arg_count and
+                not force_redo):
                 break
             else:
                 last_extended_arg_count = extended_arg_count
@@ -360,12 +361,14 @@
         while True:
             try:
                 w_key = space.next(w_iter)
-            except OperationError, e:
+            except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise
                 break
             w_index = space.getitem(w_consts, w_key)
-            consts_w[space.int_w(w_index)] = space.getitem(w_key, first)
+            w_constant = space.getitem(w_key, first)
+            w_constant = misc.intern_if_common_string(space, w_constant)
+            consts_w[space.int_w(w_index)] = w_constant
         return consts_w
 
     def _get_code_flags(self):
@@ -433,15 +436,16 @@
                         continue
                     addr = offset - current_off
                     # Python assumes that lineno always increases with
-                    # increasing bytecode address (lnotab is unsigned char).
-                    # Depending on when SET_LINENO instructions are emitted this
-                    # is not always true.  Consider the code:
+                    # increasing bytecode address (lnotab is unsigned
+                    # char).  Depending on when SET_LINENO instructions
+                    # are emitted this is not always true.  Consider the
+                    # code:
                     #     a = (1,
                     #          b)
-                    # In the bytecode stream, the assignment to "a" occurs after
-                    # the loading of "b".  This works with the C Python compiler
-                    # because it only generates a SET_LINENO instruction for the
-                    # assignment.
+                    # In the bytecode stream, the assignment to "a"
+                    # occurs after the loading of "b".  This works with
+                    # the C Python compiler because it only generates a
+                    # SET_LINENO instruction for the assignment.
                     if line or addr:
                         while addr > 255:
                             push(chr(255))
@@ -484,22 +488,22 @@
         free_names = _list_from_dict(self.free_vars, len(cell_names))
         flags = self._get_code_flags() | self.compile_info.flags
         bytecode = ''.join([block.get_code() for block in blocks])
-        return pycode.PyCode(self.space,
-                             self.argcount,
-                             len(self.var_names),
-                             stack_depth,
-                             flags,
-                             bytecode,
-                             list(consts_w),
-                             names,
-                             var_names,
-                             self.compile_info.filename,
-                             self.name,
-                             self.first_lineno,
-                             lnotab,
-                             free_names,
-                             cell_names,
-                             self.compile_info.hidden_applevel)
+        return PyCode(self.space,
+                      self.argcount,
+                      len(self.var_names),
+                      stack_depth,
+                      flags,
+                      bytecode,
+                      list(consts_w),
+                      names,
+                      var_names,
+                      self.compile_info.filename,
+                      self.name,
+                      self.first_lineno,
+                      lnotab,
+                      free_names,
+                      cell_names,
+                      self.compile_info.hidden_applevel)
 
 
 def _list_from_dict(d, offset=0):
@@ -510,134 +514,134 @@
 
 
 _static_opcode_stack_effects = {
-    ops.NOP : 0,
-    ops.STOP_CODE : 0,
+    ops.NOP: 0,
+    ops.STOP_CODE: 0,
 
-    ops.POP_TOP : -1,
-    ops.ROT_TWO : 0,
-    ops.ROT_THREE : 0,
-    ops.ROT_FOUR : 0,
-    ops.DUP_TOP : 1,
+    ops.POP_TOP: -1,
+    ops.ROT_TWO: 0,
+    ops.ROT_THREE: 0,
+    ops.ROT_FOUR: 0,
+    ops.DUP_TOP: 1,
 
-    ops.UNARY_POSITIVE : 0,
-    ops.UNARY_NEGATIVE : 0,
-    ops.UNARY_NOT : 0,
-    ops.UNARY_CONVERT : 0,
-    ops.UNARY_INVERT : 0,
+    ops.UNARY_POSITIVE: 0,
+    ops.UNARY_NEGATIVE: 0,
+    ops.UNARY_NOT: 0,
+    ops.UNARY_CONVERT: 0,
+    ops.UNARY_INVERT: 0,
 
-    ops.LIST_APPEND : -1,
-    ops.SET_ADD : -1,
-    ops.MAP_ADD : -2,
-    ops.STORE_MAP : -2,
+    ops.LIST_APPEND: -1,
+    ops.SET_ADD: -1,
+    ops.MAP_ADD: -2,
+    ops.STORE_MAP: -2,
 
-    ops.BINARY_POWER : -1,
-    ops.BINARY_MULTIPLY : -1,
-    ops.BINARY_DIVIDE : -1,
-    ops.BINARY_MODULO : -1,
-    ops.BINARY_ADD : -1,
-    ops.BINARY_SUBTRACT : -1,
-    ops.BINARY_SUBSCR : -1,
-    ops.BINARY_FLOOR_DIVIDE : -1,
-    ops.BINARY_TRUE_DIVIDE : -1,
-    ops.BINARY_LSHIFT : -1,
-    ops.BINARY_RSHIFT : -1,
-    ops.BINARY_AND : -1,
-    ops.BINARY_OR : -1,
-    ops.BINARY_XOR : -1,
+    ops.BINARY_POWER: -1,
+    ops.BINARY_MULTIPLY: -1,
+    ops.BINARY_DIVIDE: -1,
+    ops.BINARY_MODULO: -1,
+    ops.BINARY_ADD: -1,
+    ops.BINARY_SUBTRACT: -1,
+    ops.BINARY_SUBSCR: -1,
+    ops.BINARY_FLOOR_DIVIDE: -1,
+    ops.BINARY_TRUE_DIVIDE: -1,
+    ops.BINARY_LSHIFT: -1,
+    ops.BINARY_RSHIFT: -1,
+    ops.BINARY_AND: -1,
+    ops.BINARY_OR: -1,
+    ops.BINARY_XOR: -1,
 
-    ops.INPLACE_FLOOR_DIVIDE : -1,
-    ops.INPLACE_TRUE_DIVIDE : -1,
-    ops.INPLACE_ADD : -1,
-    ops.INPLACE_SUBTRACT : -1,
-    ops.INPLACE_MULTIPLY : -1,
-    ops.INPLACE_DIVIDE : -1,
-    ops.INPLACE_MODULO : -1,
-    ops.INPLACE_POWER : -1,
-    ops.INPLACE_LSHIFT : -1,
-    ops.INPLACE_RSHIFT : -1,
-    ops.INPLACE_AND : -1,
-    ops.INPLACE_OR : -1,
-    ops.INPLACE_XOR : -1,
+    ops.INPLACE_FLOOR_DIVIDE: -1,
+    ops.INPLACE_TRUE_DIVIDE: -1,
+    ops.INPLACE_ADD: -1,
+    ops.INPLACE_SUBTRACT: -1,
+    ops.INPLACE_MULTIPLY: -1,
+    ops.INPLACE_DIVIDE: -1,
+    ops.INPLACE_MODULO: -1,
+    ops.INPLACE_POWER: -1,
+    ops.INPLACE_LSHIFT: -1,
+    ops.INPLACE_RSHIFT: -1,
+    ops.INPLACE_AND: -1,
+    ops.INPLACE_OR: -1,
+    ops.INPLACE_XOR: -1,
 
-    ops.SLICE+0 : 1,
-    ops.SLICE+1 : 0,
-    ops.SLICE+2 : 0,
-    ops.SLICE+3 : -1,
-    ops.STORE_SLICE+0 : -2,
-    ops.STORE_SLICE+1 : -3,
-    ops.STORE_SLICE+2 : -3,
-    ops.STORE_SLICE+3 : -4,
-    ops.DELETE_SLICE+0 : -1,
-    ops.DELETE_SLICE+1 : -2,
-    ops.DELETE_SLICE+2 : -2,
-    ops.DELETE_SLICE+3 : -3,
+    ops.SLICE+0: 1,
+    ops.SLICE+1: 0,
+    ops.SLICE+2: 0,
+    ops.SLICE+3: -1,
+    ops.STORE_SLICE+0: -2,
+    ops.STORE_SLICE+1: -3,
+    ops.STORE_SLICE+2: -3,
+    ops.STORE_SLICE+3: -4,
+    ops.DELETE_SLICE+0: -1,
+    ops.DELETE_SLICE+1: -2,
+    ops.DELETE_SLICE+2: -2,
+    ops.DELETE_SLICE+3: -3,
 
-    ops.STORE_SUBSCR : -2,
-    ops.DELETE_SUBSCR : -2,
+    ops.STORE_SUBSCR: -2,
+    ops.DELETE_SUBSCR: -2,
 
-    ops.GET_ITER : 0,
-    ops.FOR_ITER : 1,
-    ops.BREAK_LOOP : 0,
-    ops.CONTINUE_LOOP : 0,
-    ops.SETUP_LOOP : 0,
+    ops.GET_ITER: 0,
+    ops.FOR_ITER: 1,
+    ops.BREAK_LOOP: 0,
+    ops.CONTINUE_LOOP: 0,
+    ops.SETUP_LOOP: 0,
 
-    ops.PRINT_EXPR : -1,
-    ops.PRINT_ITEM : -1,
-    ops.PRINT_NEWLINE : 0,
-    ops.PRINT_ITEM_TO : -2,
-    ops.PRINT_NEWLINE_TO : -1,
+    ops.PRINT_EXPR: -1,
+    ops.PRINT_ITEM: -1,
+    ops.PRINT_NEWLINE: 0,
+    ops.PRINT_ITEM_TO: -2,
+    ops.PRINT_NEWLINE_TO: -1,
 
-    ops.WITH_CLEANUP : -1,
-    ops.POP_BLOCK : 0,
-    ops.END_FINALLY : -1,
-    ops.SETUP_WITH : 1,
-    ops.SETUP_FINALLY : 0,
-    ops.SETUP_EXCEPT : 0,
+    ops.WITH_CLEANUP: -1,
+    ops.POP_BLOCK: 0,
+    ops.END_FINALLY: -1,
+    ops.SETUP_WITH: 1,
+    ops.SETUP_FINALLY: 0,
+    ops.SETUP_EXCEPT: 0,
 
-    ops.LOAD_LOCALS : 1,
-    ops.RETURN_VALUE : -1,
-    ops.EXEC_STMT : -3,
-    ops.YIELD_VALUE : 0,
-    ops.BUILD_CLASS : -2,
-    ops.BUILD_MAP : 1,
-    ops.BUILD_SET : 1,
-    ops.COMPARE_OP : -1,
+    ops.LOAD_LOCALS: 1,
+    ops.RETURN_VALUE: -1,
+    ops.EXEC_STMT: -3,
+    ops.YIELD_VALUE: 0,
+    ops.BUILD_CLASS: -2,
+    ops.BUILD_MAP: 1,
+    ops.BUILD_SET: 1,
+    ops.COMPARE_OP: -1,
 
-    ops.LOOKUP_METHOD : 1,
+    ops.LOOKUP_METHOD: 1,
 
-    ops.LOAD_NAME : 1,
-    ops.STORE_NAME : -1,
-    ops.DELETE_NAME : 0,
+    ops.LOAD_NAME: 1,
+    ops.STORE_NAME: -1,
+    ops.DELETE_NAME: 0,
 
-    ops.LOAD_FAST : 1,
-    ops.STORE_FAST : -1,
-    ops.DELETE_FAST : 0,
+    ops.LOAD_FAST: 1,
+    ops.STORE_FAST: -1,
+    ops.DELETE_FAST: 0,
 
-    ops.LOAD_ATTR : 0,
-    ops.STORE_ATTR : -2,
-    ops.DELETE_ATTR : -1,
+    ops.LOAD_ATTR: 0,
+    ops.STORE_ATTR: -2,
+    ops.DELETE_ATTR: -1,
 
-    ops.LOAD_GLOBAL : 1,
-    ops.STORE_GLOBAL : -1,
-    ops.DELETE_GLOBAL : 0,
+    ops.LOAD_GLOBAL: 1,
+    ops.STORE_GLOBAL: -1,
+    ops.DELETE_GLOBAL: 0,
 
-    ops.LOAD_CLOSURE : 1,
-    ops.LOAD_DEREF : 1,
-    ops.STORE_DEREF : -1,
+    ops.LOAD_CLOSURE: 1,
+    ops.LOAD_DEREF: 1,
+    ops.STORE_DEREF: -1,
 
-    ops.LOAD_CONST : 1,
+    ops.LOAD_CONST: 1,
 
-    ops.IMPORT_STAR : -1,
-    ops.IMPORT_NAME : -1,
-    ops.IMPORT_FROM : 1,
+    ops.IMPORT_STAR: -1,
+    ops.IMPORT_NAME: -1,
+    ops.IMPORT_FROM: 1,
 
-    ops.JUMP_FORWARD : 0,
-    ops.JUMP_ABSOLUTE : 0,
-    ops.JUMP_IF_TRUE_OR_POP : 0,
-    ops.JUMP_IF_FALSE_OR_POP : 0,
-    ops.POP_JUMP_IF_TRUE : -1,
-    ops.POP_JUMP_IF_FALSE : -1,
-    ops.JUMP_IF_NOT_DEBUG : 0,
+    ops.JUMP_FORWARD: 0,
+    ops.JUMP_ABSOLUTE: 0,
+    ops.JUMP_IF_TRUE_OR_POP: 0,
+    ops.JUMP_IF_FALSE_OR_POP: 0,
+    ops.POP_JUMP_IF_TRUE: -1,
+    ops.POP_JUMP_IF_FALSE: -1,
+    ops.JUMP_IF_NOT_DEBUG: 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -106,3 +106,13 @@
     except IndexError:
         return name
     return "_%s%s" % (klass[i:], name)
+
+
+def intern_if_common_string(space, w_const):
+    # only intern identifier-like strings
+    if not space.is_w(space.type(w_const), space.w_str):
+        return w_const
+    for c in space.str_w(w_const):
+        if not (c.isalnum() or c == '_'):
+            return w_const
+    return space.new_interned_w_str(w_const)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -272,6 +272,11 @@
                 if w_const is None:
                     return tup
                 consts_w[i] = w_const
+            # intern the string constants packed into the tuple here,
+            # because assemble.py will see the result as just a tuple constant
+            for i in range(len(consts_w)):
+                consts_w[i] = misc.intern_if_common_string(
+                    self.space, consts_w[i])
         else:
             consts_w = []
         w_consts = self.space.newtuple(consts_w)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -14,7 +14,7 @@
     UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
-from pypy.interpreter.miscutils import ThreadLocals
+from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
 
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
@@ -384,7 +384,7 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        self.interned_strings = {}
+        self.interned_strings = make_weak_value_dictionary(self, str, W_Root)
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
@@ -777,25 +777,30 @@
             return self.w_False
 
     def new_interned_w_str(self, w_s):
+        assert isinstance(w_s, W_Root)   # and is not None
         s = self.str_w(w_s)
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        self.interned_strings[s] = w_s
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = w_s
+            self.interned_strings.set(s, w_s1)
+        return w_s1
 
     def new_interned_str(self, s):
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        w_s = self.interned_strings[s] = self.wrap(s)
-        return w_s
+        w_s1 = self.interned_strings.get(s)
+        if w_s1 is None:
+            w_s1 = self.wrap(s)
+            self.interned_strings.set(s, w_s1)
+        return w_s1
+
+    def is_interned_str(self, s):
+        # interface for marshal_impl
+        if not we_are_translated():
+            assert type(s) is str
+        return self.interned_strings.get(s) is not None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -31,3 +31,19 @@
 
     def getallvalues(self):
         return {0: self._value}
+
+
+def make_weak_value_dictionary(space, keytype, valuetype):
+    "NOT_RPYTHON"
+    if space.config.translation.rweakref:
+        from rpython.rlib.rweakref import RWeakValueDictionary
+        return RWeakValueDictionary(keytype, valuetype)
+    else:
+        class FakeWeakValueDict(object):
+            def __init__(self):
+                self._dict = {}
+            def get(self, key):
+                return self._dict.get(key, None)
+            def set(self, key, value):
+                self._dict[key] = value
+        return FakeWeakValueDict()
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -125,13 +125,14 @@
         else:
             return self.space.builtin
 
+    _NO_CELLS = []
+
     @jit.unroll_safe
     def initialize_frame_scopes(self, outer_func, code):
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
         # class bodies only have CO_NEWLOCALS.
         # CO_NEWLOCALS: make a locals dict unless optimized is also set
         # CO_OPTIMIZED: no locals dict needed at all
-        # NB: this method is overridden in nestedscope.py
         flags = code.co_flags
         if not (flags & pycode.CO_OPTIMIZED):
             if flags & pycode.CO_NEWLOCALS:
@@ -144,7 +145,7 @@
         nfreevars = len(code.co_freevars)
         if not nfreevars:
             if not ncellvars:
-                self.cells = []
+                self.cells = self._NO_CELLS
                 return            # no self.cells needed - fast path
         elif outer_func is None:
             space = self.space
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -970,7 +970,12 @@
             sys.stdout = out
         output = s.getvalue()
         assert "CALL_METHOD" in output
-            
+
+    def test_interned_strings(self):
+        source = """x = ('foo_bar42', 5); y = 'foo_bar42'; z = x[0]"""
+        exec source
+        assert y is z
+
 
 class AppTestExceptions:
     def test_indentation_error(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -378,3 +378,41 @@
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+
+    def test_interned_strings_are_weak(self):
+        import weakref, gc, random
+        space = self.space
+        assert space.config.translation.rweakref
+        w1 = space.new_interned_str("abcdef")
+        w2 = space.new_interned_str("abcdef")
+        assert w2 is w1
+        #
+        # check that 'w1' goes away if we don't hold a reference to it
+        rw1 = weakref.ref(w1)
+        del w1, w2
+        i = 10
+        while rw1() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
+        #
+        s = "foobar%r" % random.random()
+        w0 = space.wrap(s)
+        w1 = space.new_interned_w_str(w0)
+        assert w1 is w0
+        w2 = space.new_interned_w_str(w0)
+        assert w2 is w0
+        w3 = space.wrap(s)
+        assert w3 is not w0
+        w4 = space.new_interned_w_str(w3)
+        assert w4 is w0
+        #
+        # check that 'w0' goes away if we don't hold a reference to it
+        # (even if we hold a reference to 'w3')
+        rw0 = weakref.ref(w0)
+        del w0, w1, w2, w4
+        i = 10
+        while rw0() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -158,21 +158,14 @@
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
-    _attrs_            = ['value_fits_long', 'vmin', 'vrangemax']
-    _immutable_fields_ = ['value_fits_long', 'vmin', 'vrangemax']
+    _attrs_            = ['value_fits_long', 'value_smaller_than_long']
+    _immutable_fields_ = ['value_fits_long', 'value_smaller_than_long']
     is_primitive_integer = True
 
     def __init__(self, *args):
         W_CTypePrimitive.__init__(self, *args)
         self.value_fits_long = self.size <= rffi.sizeof(lltype.Signed)
-        if self.size < rffi.sizeof(lltype.Signed):
-            assert self.value_fits_long
-            sh = self.size * 8
-            self.vmin = r_uint(-1) << (sh - 1)
-            self.vrangemax = (r_uint(1) << sh) - 1
-        else:
-            self.vmin = r_uint(0)
-            self.vrangemax = r_uint(-1)
+        self.value_smaller_than_long = self.size < rffi.sizeof(lltype.Signed)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -192,8 +185,17 @@
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
             value = misc.as_long(self.space, w_ob)
-            if self.size < rffi.sizeof(lltype.Signed):
-                if r_uint(value) - self.vmin > self.vrangemax:
+            if self.value_smaller_than_long:
+                size = self.size
+                if size == 1:
+                    signextended = misc.signext(value, 1)
+                elif size == 2:
+                    signextended = misc.signext(value, 2)
+                elif size == 4:
+                    signextended = misc.signext(value, 4)
+                else:
+                    raise AssertionError("unsupported size")
+                if value != signextended:
                     self._overflow(w_ob)
             misc.write_raw_signed_data(cdata, value, self.size)
         else:
@@ -221,7 +223,7 @@
             length = w_cdata.get_array_length()
             populate_list_from_raw_array(res, buf, length)
             return res
-        elif self.value_fits_long:
+        elif self.value_smaller_than_long:
             res = [0] * w_cdata.get_array_length()
             misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
             return res
@@ -235,8 +237,8 @@
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
-                overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                overflowed = misc.pack_list_to_raw_array_bounds_signed(
+                    int_list, cdata, self.size)
                 if overflowed != 0:
                     self._overflow(self.space.wrap(overflowed))
             return True
@@ -314,8 +316,8 @@
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
-            overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
+                int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.wrap(overflowed))
             return True
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -216,6 +216,19 @@
 neg_msg = "can't convert negative number to unsigned"
 ovf_msg = "long too big to convert"
 
+ at specialize.arg(1)
+def signext(value, size):
+    # 'value' is sign-extended from 'size' bytes to a full integer.
+    # 'size' should be a constant smaller than a full integer size.
+    if size == rffi.sizeof(rffi.SIGNEDCHAR):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SIGNEDCHAR, value))
+    elif size == rffi.sizeof(rffi.SHORT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SHORT, value))
+    elif size == rffi.sizeof(rffi.INT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+    else:
+        raise AssertionError("unsupported size")
+
 # ____________________________________________________________
 
 class _NotStandardObject(Exception):
@@ -334,13 +347,26 @@
 
 # ____________________________________________________________
 
-def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+def pack_list_to_raw_array_bounds_signed(int_list, target, size):
     for TP, TPP in _prim_signed_types:
         if size == rffi.sizeof(TP):
             ptr = rffi.cast(TPP, target)
             for i in range(len(int_list)):
                 x = int_list[i]
-                if r_uint(x) - vmin > vrangemax:
+                y = rffi.cast(TP, x)
+                if x != rffi.cast(lltype.Signed, y):
+                    return x      # overflow
+                ptr[i] = y
+            return 0
+    raise NotImplementedError("bad integer size")
+
+def pack_list_to_raw_array_bounds_unsigned(int_list, target, size, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) > vrangemax:
                     return x      # overflow
                 ptr[i] = rffi.cast(TP, x)
             return 0
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -304,7 +304,7 @@
             py.test.skip("works with internals of _file impl on py.py")
         state = [0]
         def read(fd, n=None):
-            if fd != 42:
+            if fd != 424242:
                 return cls.old_read(fd, n)
             if state[0] == 0:
                 state[0] += 1
@@ -315,7 +315,7 @@
             return ''
         os.read = read
         stdin = W_File(cls.space)
-        stdin.file_fdopen(42, 'rb', 1)
+        stdin.file_fdopen(424242, 'rb', 1)
         stdin.name = '<stdin>'
         cls.w_stream = stdin
 
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -14,6 +14,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.module._rawffi.alt.type_converter import FromAppLevelConverter, ToAppLevelConverter
 from pypy.module._rawffi.interp_rawffi import got_libffi_error, wrap_dlopenerror
+from pypy.module._rawffi import lasterror
 
 import os
 if os.name == 'nt':
@@ -201,11 +202,23 @@
         self.func = func
         self.argchain = argchain
 
+    def before(self):
+        lasterror.restore_last_error(self.space)
+
+    def after(self):
+        lasterror.save_last_error(self.space)
+
     def get_longlong(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.LONGLONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.LONGLONG)
+        self.after()
+        return x
 
     def get_ulonglong(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.ULONGLONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.ULONGLONG)
+        self.after()
+        return x
 
     def get_signed(self, w_ffitype):
         # if the declared return type of the function is smaller than LONG,
@@ -216,64 +229,94 @@
         # to space.wrap in order to get a nice applevel <int>.
         #
         restype = w_ffitype.get_ffitype()
+        self.before()
         call = self.func.call
         if restype is libffi.types.slong:
-            return call(self.argchain, rffi.LONG)
+            x = call(self.argchain, rffi.LONG)
         elif restype is libffi.types.sint:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.INT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.INT))
         elif restype is libffi.types.sshort:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT))
         elif restype is libffi.types.schar:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR))
         else:
-            self.error(w_ffitype)
+            raise self.error(w_ffitype)
+        self.after()
+        return x
 
     def get_unsigned(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.ULONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.ULONG)
+        self.after()
+        return x
 
     def get_unsigned_which_fits_into_a_signed(self, w_ffitype):
         # the same comment as get_signed apply
         restype = w_ffitype.get_ffitype()
+        self.before()
         call = self.func.call
         if restype is libffi.types.uint:
             assert not libffi.IS_32_BIT
             # on 32bit machines, we should never get here, because it's a case
             # which has already been handled by get_unsigned above.
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT))
         elif restype is libffi.types.ushort:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT))
         elif restype is libffi.types.uchar:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR))
         else:
-            self.error(w_ffitype)
+            raise self.error(w_ffitype)
+        self.after()
+        return x
 
 
     def get_pointer(self, w_ffitype):
+        self.before()
         ptrres = self.func.call(self.argchain, rffi.VOIDP)
+        self.after()
         return rffi.cast(rffi.ULONG, ptrres)
 
     def get_char(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.UCHAR)
+        self.before()
+        x = self.func.call(self.argchain, rffi.UCHAR)
+        self.after()
+        return x
 
     def get_unichar(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.WCHAR_T)
+        self.before()
+        x = self.func.call(self.argchain, rffi.WCHAR_T)
+        self.after()
+        return x
 
     def get_float(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.DOUBLE)
+        self.before()
+        x = self.func.call(self.argchain, rffi.DOUBLE)
+        self.after()
+        return x
 
     def get_singlefloat(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.FLOAT)
+        self.before()
+        x = self.func.call(self.argchain, rffi.FLOAT)
+        self.after()
+        return x
 
     def get_struct(self, w_ffitype, w_structdescr):
+        self.before()
         addr = self.func.call(self.argchain, rffi.LONG, is_struct=True)
+        self.after()
         return w_structdescr.fromaddress(self.space, addr)
 
     def get_struct_rawffi(self, w_ffitype, w_structdescr):
+        self.before()
         uintval = self.func.call(self.argchain, rffi.ULONG, is_struct=True)
+        self.after()
         return w_structdescr.fromaddress(self.space, uintval)
 
     def get_void(self, w_ffitype):
-        return self.func.call(self.argchain, lltype.Void)
+        self.before()
+        x = self.func.call(self.argchain, lltype.Void)
+        self.after()
+        return x
 
 
 def unpack_argtypes(space, w_argtypes, w_restype):
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -18,6 +18,7 @@
 from rpython.rlib.rarithmetic import intmask, r_uint
 from pypy.module._rawffi.buffer import RawFFIBuffer
 from pypy.module._rawffi.tracker import tracker
+from pypy.module._rawffi import lasterror
 
 TYPEMAP = {
     # XXX A mess with unsigned/signed/normal chars :-/
@@ -495,10 +496,14 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                lasterror.restore_last_error(space)
                 self.ptr.call(args_ll, result.ll_buffer)
+                lasterror.save_last_error(space)
                 return space.wrap(result)
             else:
+                lasterror.restore_last_error(space)
                 self.ptr.call(args_ll, lltype.nullptr(rffi.VOIDP.TO))
+                lasterror.save_last_error(space)
                 return space.w_None
         except StackCheckError, e:
             raise OperationError(space.w_ValueError, space.wrap(e.message))
@@ -615,12 +620,10 @@
 
 if sys.platform == 'win32':
     def get_last_error(space):
-        from rpython.rlib.rwin32 import GetLastError
-        return space.wrap(GetLastError())
+        return space.wrap(lasterror.fetch_last_error(space))
     @unwrap_spec(error=int)
     def set_last_error(space, error):
-        from rpython.rlib.rwin32 import SetLastError
-        SetLastError(error)
+        lasterror.store_last_error(space, error)
 else:
     # always have at least a dummy version of these functions
     # (https://bugs.pypy.org/issue1242)
diff --git a/pypy/module/_rawffi/lasterror.py b/pypy/module/_rawffi/lasterror.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_rawffi/lasterror.py
@@ -0,0 +1,40 @@
+# For Windows only.
+# https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
+
+import os
+
+_MS_WINDOWS = os.name == "nt"
+
+
+if _MS_WINDOWS:
+    from rpython.rlib import rwin32
+    from pypy.interpreter.executioncontext import ExecutionContext
+
+
+    ExecutionContext._rawffi_last_error = 0
+
+    def fetch_last_error(space):
+        ec = space.getexecutioncontext()
+        return ec._rawffi_last_error
+
+    def store_last_error(space, last_error):
+        ec = space.getexecutioncontext()
+        ec._rawffi_last_error = last_error
+
+    def restore_last_error(space):
+        ec = space.getexecutioncontext()
+        lasterror = ec._rawffi_last_error
+        rwin32.SetLastError(lasterror)
+
+    def save_last_error(space):
+        lasterror = rwin32.GetLastError()
+        ec = space.getexecutioncontext()
+        ec._rawffi_last_error = lasterror
+
+else:
+
+    def restore_last_error(space):
+        pass
+
+    def save_last_error(space):
+        pass
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -16,6 +16,7 @@
         #include "src/precommondefs.h"
         #include <stdlib.h>
         #include <stdio.h>
+        #include <errno.h>
 
         struct x
         {
@@ -204,6 +205,24 @@
             return inp;
         }
 
+        RPY_EXPORTED
+        int check_errno(int incoming)
+        {
+            int old_errno = errno;
+            errno = incoming;
+            return old_errno;
+        }
+
+        #ifdef _WIN32
+        #include <Windows.h>
+        RPY_EXPORTED
+        int check_last_error(int incoming)
+        {
+            int old_errno = GetLastError();
+            SetLastError(incoming);
+            return old_errno;
+        }
+        #endif
         '''))
         eci = ExternalCompilationInfo(include_dirs=[cdir])
         return str(platform.compile([c_file], eci, 'x', standalone=False))
@@ -1150,6 +1169,37 @@
         raises(OverflowError, "arg1[0] = 10**900")
         arg1.free()
 
+    def test_errno(self):
+        import _rawffi
+        lib = _rawffi.CDLL(self.lib_name)
+        A = _rawffi.Array('i')
+        f = lib.ptr('check_errno', ['i'], 'i')
+        _rawffi.set_errno(42)
+        arg = A(1)
+        arg[0] = 43
+        res = f(arg)
+        assert res[0] == 42
+        z = _rawffi.get_errno()
+        assert z == 43
+        arg.free()
+
+    def test_last_error(self):
+        import sys
+        if sys.platform != 'win32':
+            skip("Windows test")
+        import _rawffi
+        lib = _rawffi.CDLL(self.lib_name)
+        A = _rawffi.Array('i')
+        f = lib.ptr('check_last_error', ['i'], 'i')
+        _rawffi.set_last_error(42)
+        arg = A(1)
+        arg[0] = 43
+        res = f(arg)
+        assert res[0] == 42
+        z = _rawffi.get_last_error()
+        assert z == 43
+        arg.free()
+
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -52,7 +52,8 @@
 
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
-constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
+if not OPENSSL_NO_SSL3:
+    constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 
@@ -656,7 +657,7 @@
     # set up context
     if protocol == PY_SSL_VERSION_TLS1:
         method = libssl_TLSv1_method()
-    elif protocol == PY_SSL_VERSION_SSL3:
+    elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
         method = libssl_SSLv3_method()
     elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
         method = libssl_SSLv2_method()
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -24,12 +24,19 @@
 
 separate_module_source = """
 #include <openssl/crypto.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 
 static unsigned int _ssl_locks_count = 0;
 static struct RPyOpaque_ThreadLock *_ssl_locks;
 
 static unsigned long _ssl_thread_id_function(void) {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (unsigned long)GetCurrentThreadId();
+#else
+    return (unsigned long)pthread_self();
+#endif
 }
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -58,7 +58,7 @@
     w_globals = from_ref(space, py_frame.c_f_globals)
 
     frame = space.FrameClass(space, code, w_globals, outer_func=None)
-    frame.f_lineno = py_frame.c_f_lineno
+    frame.f_lineno = rffi.getintfield(py_frame, 'c_f_lineno')
     w_obj = space.wrap(frame)
     track_reference(space, py_obj, w_obj)
     return w_obj
diff --git a/pypy/module/cpyext/src/pythread.c b/pypy/module/cpyext/src/pythread.c
--- a/pypy/module/cpyext/src/pythread.c
+++ b/pypy/module/cpyext/src/pythread.c
@@ -1,11 +1,18 @@
 #include <Python.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 #include "pythread.h"
 #include "src/thread.h"
 
 long
 PyThread_get_thread_ident(void)
 {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (long)GetCurrentThreadId();
+#else
+    return (long)pthread_self();
+#endif
 }
 
 PyThread_type_lock
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -144,7 +144,6 @@
     atom_int(tc, int)           puts code and int
     atom_int64(tc, int64)       puts code and int64
     atom_str(tc, str)           puts code, len and string
-    atom_strlist(tc, strlist)   puts code, len and list of strings
 
     building blocks for compound types:
 
@@ -198,15 +197,6 @@
         self.atom_int(typecode, len(x))
         self.put(x)
 
-    def atom_strlist(self, typecode, tc2, x):
-        self.atom_int(typecode, len(x))
-        atom_str = self.atom_str
-        for item in x:
-            # type(str) seems to be forbidden
-            #if type(item) is not str:
-            #    self.raise_exc('object with wrong type in strlist')
-            atom_str(tc2, item)
-
     def start(self, typecode):
         # type(char) not supported
         self.put(typecode)
@@ -379,16 +369,6 @@
         self.start(typecode)
         return self.get_lng()
 
-    def atom_strlist(self, typecode, tc2):
-        self.start(typecode)
-        lng = self.get_lng()
-        res = [None] * lng
-        idx = 0
-        while idx < lng:
-            res[idx] = self.atom_str(tc2)
-            idx += 1
-        return res
-
     def start(self, typecode):
         tc = self.get1()
         if tc != typecode:
@@ -436,7 +416,6 @@
 
     def get_w_obj(self, allow_null=False):
         space = self.space
-        w_ret = space.w_None # something not None
         tc = self.get1()
         w_ret = self._dispatch[ord(tc)](space, self, tc)
         if w_ret is None and not allow_null:
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from rpython.tool.pairtype import extendabletype
-
+from pypy.module.micronumpy import support
 
 def wrap_impl(space, w_cls, w_instance, impl):
     if w_cls is None or space.is_w(w_cls, space.gettypefor(W_NDimArray)):
@@ -44,11 +44,32 @@
         return W_NDimArray(impl)
 
     @staticmethod
-    def from_shape_and_storage(space, shape, storage, dtype, order='C', owning=False,
-                               w_subtype=None, w_base=None, writable=True):
+    def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1,
+                               order='C', owning=False, w_subtype=None,
+                               w_base=None, writable=True, strides=None):
         from pypy.module.micronumpy import concrete
-        from pypy.module.micronumpy.strides import calc_strides
-        strides, backstrides = calc_strides(shape, dtype, order)
+        from pypy.module.micronumpy.strides import (calc_strides,
+                                                    calc_backstrides)
+        isize = dtype.elsize
+        if storage_bytes > 0 :
+            totalsize = support.product(shape) * isize
+            if totalsize > storage_bytes:
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "buffer is too small for requested array"))
+        else:
+            storage_bytes = support.product(shape) * isize
+        if strides is None:
+            strides, backstrides = calc_strides(shape, dtype, order)
+        else:
+            if len(strides) != len(shape):
+                raise oefmt(space.w_ValueError,
+                    'strides, if given, must be the same length as shape')
+            for i in range(len(strides)):
+                if strides[i] < 0 or strides[i]*shape[i] > storage_bytes:
+                    raise oefmt(space.w_ValueError,
+                        'strides is incompatible with shape of requested '
+                        'array and size of buffer')
+            backstrides = calc_backstrides(strides, shape)
         if w_base is not None:
             if owning:
                 raise OperationError(space.w_ValueError,
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -59,9 +59,9 @@
     _mixin_ = True
 
     def reduce(self, space):
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
+        _numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(_numpypy, MixedModule)
+        multiarray = _numpypy.get("multiarray")
         assert isinstance(multiarray, MixedModule)
         scalar = multiarray.get("scalar")
 
@@ -167,7 +167,7 @@
         if len(args_w) >= 1:
             for w_arg in args_w:
                 try:
-                    idx = support.index_w(space, w_arg)
+                    support.index_w(space, w_arg)
                 except OperationError:
                     raise oefmt(space.w_TypeError, "an integer is required")
             raise oefmt(space.w_ValueError, "axes don't match array")
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -34,8 +34,8 @@
 
 
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
-                        "unegative", "flat", "tostring","count_nonzero",
-                        "argsort"]
+                        "unegative", "flat", "tostring", "count_nonzero",
+                        "argsort", "cumsum", "logical_xor_reduce"]
 TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted']
 TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype']
 THREE_ARG_FUNCTIONS = ['where']
@@ -559,6 +559,11 @@
                 w_res = arr.descr_any(interp.space)
             elif self.name == "all":
                 w_res = arr.descr_all(interp.space)
+            elif self.name == "cumsum":
+                w_res = arr.descr_cumsum(interp.space)
+            elif self.name == "logical_xor_reduce":
+                logical_xor = ufuncs.get(interp.space).logical_xor
+                w_res = logical_xor.reduce(interp.space, arr, None)
             elif self.name == "unegative":
                 neg = ufuncs.get(interp.space).negative
                 w_res = neg.call(interp.space, [arr])
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -11,7 +11,7 @@
 from pypy.module.micronumpy.iterators import ArrayIter
 from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
     RecordChunk, calc_strides, calc_new_strides, shape_agreement,
-    calculate_broadcast_strides)
+    calculate_broadcast_strides, calc_backstrides)
 
 
 class BaseConcreteArray(object):
@@ -47,6 +47,7 @@
     def setitem(self, index, value):
         self.dtype.itemtype.store(self, index, 0, value)
 
+    @jit.unroll_safe
     def setslice(self, space, arr):
         if len(arr.get_shape()) > 0 and len(self.get_shape()) == 0:
             raise oefmt(space.w_ValueError,
@@ -78,10 +79,7 @@
                                                self.get_strides(), self.order)
         if new_strides is not None:
             # We can create a view, strides somehow match up.
-            ndims = len(new_shape)
-            new_backstrides = [0] * ndims
-            for nd in range(ndims):
-                new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd]
+            new_backstrides = calc_backstrides(new_strides, new_shape)
             assert isinstance(orig_array, W_NDimArray) or orig_array is None
             return SliceArray(self.start, new_strides, new_backstrides,
                               new_shape, self, orig_array)
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -302,5 +302,5 @@
         return a
     else:
         writable = not buf.readonly
-    return W_NDimArray.from_shape_and_storage(space, [n], storage, dtype=dtype,
-                                              w_base=w_buffer, writable=writable)
+    return W_NDimArray.from_shape_and_storage(space, [n], storage, storage_bytes=s, 
+                                dtype=dtype, w_base=w_buffer, writable=writable)
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -286,7 +286,6 @@
     def descr_hash(self, space):
         return space.wrap(self._compute_hash(space, 0x345678))
 
-
     def descr_str(self, space):
         if self.fields:
             return space.str(self.descr_get_descr(space))
diff --git a/pypy/module/micronumpy/flagsobj.py b/pypy/module/micronumpy/flagsobj.py
--- a/pypy/module/micronumpy/flagsobj.py
+++ b/pypy/module/micronumpy/flagsobj.py
@@ -1,47 +1,30 @@
+from rpython.rlib import jit
+
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import constants as NPY
-
+from pypy.module.micronumpy.strides import is_c_contiguous, is_f_contiguous
 
 def enable_flags(arr, flags):
     arr.flags |= flags
 
-
 def clear_flags(arr, flags):
     arr.flags &= ~flags
 
-
 def _update_contiguous_flags(arr):
-    shape = arr.shape
-    strides = arr.strides
-
-    is_c_contig = True
-    sd = arr.dtype.elsize
-    for i in range(len(shape) - 1, -1, -1):
-        dim = shape[i]
-        if strides[i] != sd:
-            is_c_contig = False
-            break
-        if dim == 0:
-            break
-        sd *= dim
+    is_c_contig = is_c_contiguous(arr)
     if is_c_contig:
         enable_flags(arr, NPY.ARRAY_C_CONTIGUOUS)
     else:
         clear_flags(arr, NPY.ARRAY_C_CONTIGUOUS)
 
-    sd = arr.dtype.elsize
-    for i in range(len(shape)):
-        dim = shape[i]
-        if strides[i] != sd:
-            clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
-            return
-        if dim == 0:
-            break
-        sd *= dim
-    enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
+    is_f_contig = is_f_contiguous(arr)
+    if is_f_contig:
+        enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
+    else:
+        clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
 
 
 class W_FlagsObject(W_Root):
diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py
--- a/pypy/module/micronumpy/flatiter.py
+++ b/pypy/module/micronumpy/flatiter.py
@@ -22,6 +22,9 @@
     def get_shape(self):
         return self.shape
 
+    def get_size(self):
+        return self.base().get_size()
+
     def create_iter(self, shape=None, backward_broadcast=False):
         assert isinstance(self.base(), W_NDimArray)
         return self.base().create_iter()
@@ -41,8 +44,8 @@
         return space.wrap(self.state.index)
 
     def descr_coords(self, space):
-        self.state = self.iter.update(self.state)
-        return space.newtuple([space.wrap(c) for c in self.state.indices])
+        coords = self.iter.indices(self.state)
+        return space.newtuple([space.wrap(c) for c in coords])
 
     def descr_iter(self):
         return self
@@ -54,7 +57,7 @@
         if self.iter.done(self.state):
             raise OperationError(space.w_StopIteration, space.w_None)
         w_res = self.iter.getitem(self.state)
-        self.state = self.iter.next(self.state)
+        self.iter.next(self.state, mutate=True)
         return w_res
 
     def descr_getitem(self, space, w_idx):
@@ -71,7 +74,7 @@
                                          base.get_order(), w_instance=base)
             return loop.flatiter_getitem(res, self.iter, state, step)
         finally:
-            self.state = self.iter.reset(self.state)
+            self.iter.reset(self.state, mutate=True)
 
     def descr_setitem(self, space, w_idx, w_value):
         if not (space.isinstance_w(w_idx, space.w_int) or
@@ -91,7 +94,7 @@
             arr = convert_to_array(space, w_value)
             loop.flatiter_setitem(space, dtype, arr, self.iter, state, step, length)
         finally:
-            self.state = self.iter.reset(self.state)
+            self.iter.reset(self.state, mutate=True)
 
 
 W_FlatIterator.typedef = TypeDef("numpy.flatiter",
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -41,16 +41,6 @@
 from pypy.module.micronumpy.base import W_NDimArray
 from pypy.module.micronumpy.flagsobj import _update_contiguous_flags
 
-class OpFlag(object):
-    def __init__(self):
-        self.rw = ''
-        self.broadcast = True
-        self.force_contig = False
-        self.force_align = False
-        self.native_byte_order = False
-        self.tmp_copy = ''
-        self.allocate = False
-
 
 class PureShapeIter(object):
     def __init__(self, shape, idx_w):
@@ -87,25 +77,24 @@
 
 
 class IterState(object):
-    _immutable_fields_ = ['iterator', 'index', 'indices', 'offset']
+    _immutable_fields_ = ['iterator', '_indices']
 
     def __init__(self, iterator, index, indices, offset):
         self.iterator = iterator
         self.index = index
-        self.indices = indices
+        self._indices = indices
         self.offset = offset
 
 
 class ArrayIter(object):
     _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
                           'strides[*]', 'backstrides[*]', 'factors[*]',
-                          'slice_shape', 'slice_stride', 'slice_backstride',
-                          'track_index', 'operand_type', 'slice_operand_type']
+                          'track_index']
 
     track_index = True
 
-    def __init__(self, array, size, shape, strides, backstrides, op_flags=OpFlag()):
-        from pypy.module.micronumpy import concrete
+    @jit.unroll_safe
+    def __init__(self, array, size, shape, strides, backstrides):
         assert len(shape) == len(strides) == len(backstrides)
         _update_contiguous_flags(array)
         self.contiguous = (array.flags & NPY.ARRAY_C_CONTIGUOUS and
@@ -117,12 +106,6 @@
         self.shape_m1 = [s - 1 for s in shape]
         self.strides = strides
         self.backstrides = backstrides
-        self.slice_shape = 1
-        self.slice_stride = -1
-        if strides:
-            self.slice_stride = strides[-1]
-        self.slice_backstride = 1
-        self.slice_operand_type = concrete.SliceArray
 
         ndim = len(shape)
         factors = [0] * ndim
@@ -132,32 +115,35 @@
             else:
                 factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i]
         self.factors = factors
-        if op_flags.rw == 'r':
-            self.operand_type = concrete.ConcreteNonWritableArrayWithBase
-        else:
-            self.operand_type = concrete.ConcreteArrayWithBase
 
     @jit.unroll_safe
-    def reset(self, state=None):
+    def reset(self, state=None, mutate=False):
+        index = 0
         if state is None:
             indices = [0] * len(self.shape_m1)
         else:
             assert state.iterator is self
-            indices = state.indices
+            indices = state._indices
             for i in xrange(self.ndim_m1, -1, -1):
                 indices[i] = 0
-        return IterState(self, 0, indices, self.array.start)
+        offset = self.array.start
+        if not mutate:
+            return IterState(self, index, indices, offset)
+        state.index = index
+        state.offset = offset
 
     @jit.unroll_safe
-    def next(self, state):
+    def next(self, state, mutate=False):
         assert state.iterator is self
         index = state.index
         if self.track_index:
             index += 1
-        indices = state.indices
+        indices = state._indices
         offset = state.offset
         if self.contiguous:
             offset += self.array.dtype.elsize
+        elif self.ndim_m1 == 0:
+            offset += self.strides[0]
         else:
             for i in xrange(self.ndim_m1, -1, -1):
                 idx = indices[i]
@@ -168,13 +154,18 @@
                 else:
                     indices[i] = 0
                     offset -= self.backstrides[i]
-        return IterState(self, index, indices, offset)
+        if not mutate:
+            return IterState(self, index, indices, offset)
+        state.index = index
+        state.offset = offset
 
     @jit.unroll_safe
     def goto(self, index):
         offset = self.array.start
         if self.contiguous:
             offset += index * self.array.dtype.elsize
+        elif self.ndim_m1 == 0:
+            offset += index * self.strides[0]
         else:
             current = index
             for i in xrange(len(self.shape_m1)):
@@ -183,20 +174,20 @@
         return IterState(self, index, None, offset)
 
     @jit.unroll_safe
-    def update(self, state):
+    def indices(self, state):
         assert state.iterator is self
         assert self.track_index
-        if not self.contiguous:
-            return state
+        indices = state._indices
+        if not (self.contiguous or self.ndim_m1 == 0):
+            return indices
         current = state.index
-        indices = state.indices
         for i in xrange(len(self.shape_m1)):
             if self.factors[i] != 0:
                 indices[i] = current / self.factors[i]
                 current %= self.factors[i]
             else:
                 indices[i] = 0
-        return IterState(self, state.index, indices, state.offset)
+        return indices
 
     def done(self, state):
         assert state.iterator is self
@@ -215,12 +206,6 @@
         assert state.iterator is self
         self.array.setitem(state.offset, elem)
 
-    def getoperand(self, st, base):
-        impl = self.operand_type
-        res = impl([], self.array.dtype, self.array.order, [], [],
-                   self.array.storage, base)
-        res.start = st.offset
-        return res
 
 def AxisIter(array, shape, axis, cumulative):
     strides = array.get_strides()
@@ -244,42 +229,3 @@
         size /= shape[axis]
     shape[axis] = backstrides[axis] = 0
     return ArrayIter(array, size, shape, array.strides, backstrides)
-
-class SliceIter(ArrayIter):
-    '''
-    used with external loops, getitem and setitem return a SliceArray
-    view into the original array
-    '''
-    _immutable_fields_ = ['base', 'slice_shape[*]', 'slice_stride[*]', 'slice_backstride[*]']
-
-    def __init__(self, array, size, shape, strides, backstrides, slice_shape,
-                 slice_stride, slice_backstride, op_flags, base):
-        from pypy.module.micronumpy import concrete
-        ArrayIter.__init__(self, array, size, shape, strides, backstrides, op_flags)
-        self.slice_shape = slice_shape
-        self.slice_stride = slice_stride
-        self.slice_backstride = slice_backstride
-        self.base = base
-        if op_flags.rw == 'r':
-            self.slice_operand_type = concrete.NonWritableSliceArray
-        else:
-            self.slice_operand_type = concrete.SliceArray
-
-    def getitem(self, state):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def getitem_bool(self, state):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def setitem(self, state, elem):
-        # XXX cannot be called - must return a boxed value
-        assert False
-
-    def getoperand(self, state, base):
-        assert state.iterator is self
-        impl = self.slice_operand_type
-        arr = impl(state.offset, [self.slice_stride], [self.slice_backstride],
-                   [self.slice_shape], self.array, self.base)
-        return arr
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -42,23 +42,38 @@
 
     # TODO handle __array_priorities__ and maybe flip the order
 
+    if w_lhs.get_size() == 1:
+        w_left = w_lhs.get_scalar_value().convert_to(space, calc_dtype)
+        left_iter = left_state = None
+    else:
+        w_left = None
+        left_iter, left_state = w_lhs.create_iter(shape)
+        left_iter.track_index = False
+
+    if w_rhs.get_size() == 1:
+        w_right = w_rhs.get_scalar_value().convert_to(space, calc_dtype)
+        right_iter = right_state = None
+    else:
+        w_right = None
+        right_iter, right_state = w_rhs.create_iter(shape)
+        right_iter.track_index = False
+
     if out is None:
         out = W_NDimArray.from_shape(space, shape, res_dtype,
                                      w_instance=lhs_for_subtype)
-    left_iter, left_state = w_lhs.create_iter(shape)
-    right_iter, right_state = w_rhs.create_iter(shape)
     out_iter, out_state = out.create_iter(shape)
-    left_iter.track_index = right_iter.track_index = False
     shapelen = len(shape)
     while not out_iter.done(out_state):
         call2_driver.jit_merge_point(shapelen=shapelen, func=func,
                                      calc_dtype=calc_dtype, res_dtype=res_dtype)
-        w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
-        w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
+        if left_iter:
+            w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
+            left_state = left_iter.next(left_state)
+        if right_iter:
+            w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
+            right_state = right_iter.next(right_state)
         out_iter.setitem(out_state, func(calc_dtype, w_left, w_right).convert_to(
             space, res_dtype))
-        left_state = left_iter.next(left_state)
-        right_state = right_iter.next(right_state)
         out_state = out_iter.next(out_state)
     return out
 
@@ -68,11 +83,12 @@
     reds='auto')
 
 def call1(space, shape, func, calc_dtype, res_dtype, w_obj, out):
+    obj_iter, obj_state = w_obj.create_iter(shape)
+    obj_iter.track_index = False
+
     if out is None:
         out = W_NDimArray.from_shape(space, shape, res_dtype, w_instance=w_obj)
-    obj_iter, obj_state = w_obj.create_iter(shape)
     out_iter, out_state = out.create_iter(shape)
-    obj_iter.track_index = False
     shapelen = len(shape)
     while not out_iter.done(out_state):
         call1_driver.jit_merge_point(shapelen=shapelen, func=func,
@@ -88,10 +104,23 @@
                                 reds = 'auto')
 
 def setslice(space, shape, target, source):
+    if not shape:
+        dtype = target.dtype
+        val = source.getitem(source.start)
+        if dtype.is_str_or_unicode():
+            val = dtype.coerce(space, val)
+        else:
+            val = val.convert_to(space, dtype)
+        target.setitem(target.start, val)
+        return target

From noreply at buildbot.pypy.org  Fri Dec 12 17:45:36 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 12 Dec 2014 17:45:36 +0100 (CET)
Subject: [pypy-commit] pypy optresult: fix the merge
Message-ID: <20141212164536.0823F1C0675@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74904:d75fb78484b4
Date: 2014-12-12 18:45 +0200
http://bitbucket.org/pypy/pypy/changeset/d75fb78484b4/

Log:	fix the merge

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -351,10 +351,10 @@
         stop = 1 << (numbits - 1)
         bounds = IntBound(start, stop - 1)
         if bounds.contains_bound(value.intbound):
-            self.make_equal_to(op.result, value)
+            self.make_equal_to(op, value)
         else:
             self.emit_operation(op)
-            vres = self.getvalue(op.result)
+            vres = self.getvalue(op)
             vres.intbound.intersect(bounds)
 
     def optimize_ARRAYLEN_GC(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -659,11 +659,7 @@
             return
         if op in self.short_boxes_in_production:
             raise BoxNotProducable
-<<<<<<< local
-        if self.availible_boxes is not None and op not in self.availible_boxes:
-=======
-        if self.available_boxes is not None and box not in self.available_boxes:
->>>>>>> other
+        if self.availible_boxes is not None and op not in self.available_boxes:
             raise BoxNotProducable
         self.short_boxes_in_production[op] = None
 

From noreply at buildbot.pypy.org  Fri Dec 12 18:00:07 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Dec 2014 18:00:07 +0100 (CET)
Subject: [pypy-commit] pypy default: run sqlite3 tests against both cpython
 version and lib_pypy version
Message-ID: <20141212170007.24AAC1C3688@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74905:853ab473ce93
Date: 2014-12-12 11:59 -0500
http://bitbucket.org/pypy/pypy/changeset/853ab473ce93/

Log:	run sqlite3 tests against both cpython version and lib_pypy version

diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -1,267 +1,284 @@
 # -*- coding: utf-8 -*-
 """Tests for _sqlite3.py"""
 
-import pytest, sys
+import pytest
+import sys
 
-if sys.version_info < (2, 7):
-    pytest.skip("_sqlite3 requires Python 2.7")
-try:
-    import _cffi_backend
-except ImportError:
-    # On CPython, "pip install cffi".  On old PyPy's, no chance
-    pytest.skip("_sqlite3 requires _cffi_backend to be installed")
-
-from lib_pypy import _sqlite3
 
 def pytest_funcarg__con(request):
     con = _sqlite3.connect(':memory:')
     request.addfinalizer(lambda: con.close())
     return con
 
-def test_list_ddl(con):
-    """From issue996.  Mostly just looking for lack of exceptions."""
-    cursor = con.cursor()
-    cursor.execute('CREATE TABLE foo (bar INTEGER)')
-    result = list(cursor)
-    assert result == []
-    cursor.execute('INSERT INTO foo (bar) VALUES (42)')
-    result = list(cursor)
-    assert result == []
-    cursor.execute('SELECT * FROM foo')
-    result = list(cursor)
-    assert result == [(42,)]
 
-def test_connect_takes_same_positional_args_as_Connection(con):
-    from inspect import getargspec
-    clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
-    conargs = getargspec(_sqlite3.connect).args
-    assert clsargs == conargs
+class BaseTestSQLite:
+    def test_list_ddl(self, con):
+        """From issue996.  Mostly just looking for lack of exceptions."""
+        cursor = con.cursor()
+        cursor.execute('CREATE TABLE foo (bar INTEGER)')
+        result = list(cursor)
+        assert result == []
+        cursor.execute('INSERT INTO foo (bar) VALUES (42)')
+        result = list(cursor)
+        assert result == []
+        cursor.execute('SELECT * FROM foo')
+        result = list(cursor)
+        assert result == [(42,)]
 
-def test_total_changes_after_close(con):
-    con.close()
-    pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
+    def test_connect_takes_same_positional_args_as_Connection(self, con):
+        if not hasattr(_sqlite3, '_ffi'):
+            pytest.skip("only works for lib_pypy _sqlite3")
+        from inspect import getargspec
+        clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
+        conargs = getargspec(_sqlite3.connect).args
+        assert clsargs == conargs
 
-def test_connection_check_init():
-    class Connection(_sqlite3.Connection):
-        def __init__(self, name):
+    def test_total_changes_after_close(self, con):
+        con.close()
+        pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
+
+    def test_connection_check_init(self):
+        class Connection(_sqlite3.Connection):
+            def __init__(self, name):
+                pass
+
+        con = Connection(":memory:")
+        e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()")
+        assert '__init__' in e.value.message
+
+    def test_cursor_check_init(self, con):
+        class Cursor(_sqlite3.Cursor):
+            def __init__(self, name):
+                pass
+
+        cur = Cursor(con)
+        e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')")
+        assert '__init__' in e.value.message
+
+    def test_connection_after_close(self, con):
+        pytest.raises(TypeError, "con()")
+        con.close()
+        # raises ProgrammingError because should check closed before check args
+        pytest.raises(_sqlite3.ProgrammingError, "con()")
+
+    def test_cursor_iter(self, con):
+        cur = con.cursor()
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        next(cur)
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        con.commit()
+        next(cur)
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        with pytest.raises(_sqlite3.ProgrammingError):
+            cur.executemany('select 1', [])
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        cur.execute('create table test(ing)')
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        cur.execute('insert into test values(1)')
+        con.commit()
+        with pytest.raises(StopIteration):
+            next(cur)
+
+    def test_cursor_after_close(self, con):
+        cur = con.execute('select 1')
+        cur.close()
+        con.close()
+        pytest.raises(_sqlite3.ProgrammingError, "cur.close()")
+        # raises ProgrammingError because should check closed before check args
+        pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)")
+        pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)")
+
+    @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')")
+    def test_connection_del(self, tmpdir):
+        """For issue1325."""
+        import os
+        import gc
+        try:
+            import resource
+        except ImportError:
+            pytest.skip("needs resource module")
+
+        limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+        try:
+            fds = 0
+            while True:
+                fds += 1
+                resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
+                try:
+                    for p in os.pipe(): os.close(p)
+                except OSError:
+                    assert fds < 100
+                else:
+                    break
+
+            def open_many(cleanup):
+                con = []
+                for i in range(3):
+                    con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
+                    if cleanup:
+                        con[i] = None
+                        gc.collect(); gc.collect()
+
+            pytest.raises(_sqlite3.OperationalError, open_many, False)
+            gc.collect(); gc.collect()
+            open_many(True)
+        finally:
+            resource.setrlimit(resource.RLIMIT_NOFILE, limit)
+
+    def test_on_conflict_rollback_executemany(self, con):
+        major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+        if (int(major), int(minor), int(micro)) < (3, 2, 2):
+            pytest.skip("requires sqlite3 version >= 3.2.2")
+        con.execute("create table foo(x, unique(x) on conflict rollback)")
+        con.execute("insert into foo(x) values (1)")
+        try:
+            con.executemany("insert into foo(x) values (?)", [[1]])
+        except _sqlite3.DatabaseError:
             pass
+        con.execute("insert into foo(x) values (2)")
+        try:
+            con.commit()
+        except _sqlite3.OperationalError:
+            pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
 
-    con = Connection(":memory:")
-    e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()")
-    assert '__init__' in e.value.message
+    def test_statement_arg_checking(self, con):
+        with pytest.raises(_sqlite3.Warning) as e:
+            con(123)
+        assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.'
+        with pytest.raises(ValueError) as e:
+            con.execute(123)
+        assert str(e.value) == 'operation parameter must be str or unicode'
+        with pytest.raises(ValueError) as e:
+            con.executemany(123, 123)
+        assert str(e.value) == 'operation parameter must be str or unicode'
+        with pytest.raises(ValueError) as e:
+            con.executescript(123)
+        assert str(e.value) == 'script argument must be unicode or string.'
 
-def test_cursor_check_init(con):
-    class Cursor(_sqlite3.Cursor):
-        def __init__(self, name):
-            pass
+    def test_statement_param_checking(self, con):
+        con.execute('create table foo(x)')
+        con.execute('insert into foo(x) values (?)', [2])
+        con.execute('insert into foo(x) values (?)', (2,))
+        class seq(object):
+            def __len__(self):
+                return 1
+            def __getitem__(self, key):
+                return 2
+        con.execute('insert into foo(x) values (?)', seq())
+        del seq.__len__
+        with pytest.raises(_sqlite3.ProgrammingError):
+            con.execute('insert into foo(x) values (?)', seq())
+        with pytest.raises(_sqlite3.ProgrammingError):
+            con.execute('insert into foo(x) values (?)', {2:2})
+        with pytest.raises(ValueError) as e:
+            con.execute('insert into foo(x) values (?)', 2)
+        assert str(e.value) == 'parameters are of unsupported type'
 
-    cur = Cursor(con)
-    e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')")
-    assert '__init__' in e.value.message
+    def test_explicit_begin(self, con):
+        con.execute('BEGIN')
+        con.execute('BEGIN ')
+        con.execute('BEGIN')
+        con.commit()
+        con.execute('BEGIN')
+        con.commit()
 
-def test_connection_after_close(con):
-    pytest.raises(TypeError, "con()")
-    con.close()
-    # raises ProgrammingError because should check closed before check args
-    pytest.raises(_sqlite3.ProgrammingError, "con()")
+    def test_row_factory_use(self, con):
+        con.row_factory = 42
+        con.execute('select 1')
 
-def test_cursor_iter(con):
-    cur = con.cursor()
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_returning_blob_must_own_memory(self, con):
+        import gc
+        con.create_function("returnblob", 0, lambda: buffer("blob"))
+        cur = con.execute("select returnblob()")
+        val = cur.fetchone()[0]
+        for i in range(5):
+            gc.collect()
+            got = (val[0], val[1], val[2], val[3])
+            assert got == ('b', 'l', 'o', 'b')
+        # in theory 'val' should be a read-write buffer
+        # but it's not right now
+        if not hasattr(_sqlite3, '_ffi'):
+            val[1] = 'X'
+            got = (val[0], val[1], val[2], val[3])
+            assert got == ('b', 'X', 'o', 'b')
 
-    cur.execute('select 1')
-    next(cur)
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_description_after_fetchall(self, con):
+        cur = con.cursor()
+        assert cur.description is None
+        cur.execute("select 42").fetchall()
+        assert cur.description is not None
 
-    cur.execute('select 1')
-    con.commit()
-    next(cur)
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_executemany_lastrowid(self, con):
+        cur = con.cursor()
+        cur.execute("create table test(a)")
+        cur.executemany("insert into test values (?)", [[1], [2], [3]])
+        assert cur.lastrowid is None
 
-    with pytest.raises(_sqlite3.ProgrammingError):
-        cur.executemany('select 1', [])
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_authorizer_bad_value(self, con):
+        def authorizer_cb(action, arg1, arg2, dbname, source):
+            return 42
+        con.set_authorizer(authorizer_cb)
+        with pytest.raises(_sqlite3.OperationalError) as e:
+            con.execute('select 123')
+        major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+        if (int(major), int(minor), int(micro)) >= (3, 6, 14):
+            assert str(e.value) == 'authorizer malfunction'
+        else:
+            assert str(e.value) == \
+                ("illegal return value (1) from the authorization function - "
+                 "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
 
-    cur.execute('select 1')
-    cur.execute('create table test(ing)')
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_issue1573(self, con):
+        cur = con.cursor()
+        cur.execute(u'SELECT 1 as m&#233;il')
+        assert cur.description[0][0] == u"m&#233;il".encode('utf-8')
 
-    cur.execute('select 1')
-    cur.execute('insert into test values(1)')
-    con.commit()
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_adapter_exception(self, con):
+        def cast(obj):
+            raise ZeroDivisionError
 
-def test_cursor_after_close(con):
-    cur = con.execute('select 1')
-    cur.close()
-    con.close()
-    pytest.raises(_sqlite3.ProgrammingError, "cur.close()")
-    # raises ProgrammingError because should check closed before check args
-    pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)")
-    pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)")
+        _sqlite3.register_adapter(int, cast)
+        try:
+            cur = con.cursor()
+            cur.execute("select ?", (4,))
+            val = cur.fetchone()[0]
+            # Adapter error is ignored, and parameter is passed as is.
+            assert val == 4
+            assert type(val) is int
+        finally:
+            del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
 
- at pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')")
-def test_connection_del(tmpdir):
-    """For issue1325."""
-    import os
-    import gc
-    try:
-        import resource
-    except ImportError:
-        pytest.skip("needs resource module")
 
-    limit = resource.getrlimit(resource.RLIMIT_NOFILE)
-    try:
-        fds = 0
-        while True:
-            fds += 1
-            resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
-            try:
-                for p in os.pipe(): os.close(p)
-            except OSError:
-                assert fds < 100
-            else:
-                break
-        def open_many(cleanup):
-            con = []
-            for i in range(3):
-                con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
-                if cleanup:
-                    con[i] = None
-                    gc.collect(); gc.collect()
+class TestSQLiteCPython(BaseTestSQLite):
+    def setup_class(cls):
+        global _sqlite3
+        import _sqlite3
 
-        pytest.raises(_sqlite3.OperationalError, open_many, False)
-        gc.collect(); gc.collect()
-        open_many(True)
-    finally:
-        resource.setrlimit(resource.RLIMIT_NOFILE, limit)
 
-def test_on_conflict_rollback_executemany(con):
-    major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
-    if (int(major), int(minor), int(micro)) < (3, 2, 2):
-        pytest.skip("requires sqlite3 version >= 3.2.2")
-    con.execute("create table foo(x, unique(x) on conflict rollback)")
-    con.execute("insert into foo(x) values (1)")
-    try:
-        con.executemany("insert into foo(x) values (?)", [[1]])
-    except _sqlite3.DatabaseError:
-        pass
-    con.execute("insert into foo(x) values (2)")
-    try:
-        con.commit()
-    except _sqlite3.OperationalError:
-        pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
+class TestSQLitePyPy(BaseTestSQLite):
+    def setup_class(cls):
+        if sys.version_info < (2, 7):
+            pytest.skip("_sqlite3 requires Python 2.7")
 
-def test_statement_arg_checking(con):
-    with pytest.raises(_sqlite3.Warning) as e:
-        con(123)
-    assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.'
-    with pytest.raises(ValueError) as e:
-        con.execute(123)
-    assert str(e.value) == 'operation parameter must be str or unicode'
-    with pytest.raises(ValueError) as e:
-        con.executemany(123, 123)
-    assert str(e.value) == 'operation parameter must be str or unicode'
-    with pytest.raises(ValueError) as e:
-        con.executescript(123)
-    assert str(e.value) == 'script argument must be unicode or string.'
+        try:
+            import _cffi_backend
+        except ImportError:
+            # On CPython, "pip install cffi".  On old PyPy's, no chance
+            pytest.skip("_sqlite3 requires _cffi_backend to be installed")
 
-def test_statement_param_checking(con):
-    con.execute('create table foo(x)')
-    con.execute('insert into foo(x) values (?)', [2])
-    con.execute('insert into foo(x) values (?)', (2,))
-    class seq(object):
-        def __len__(self):
-            return 1
-        def __getitem__(self, key):
-            return 2
-    con.execute('insert into foo(x) values (?)', seq())
-    del seq.__len__
-    with pytest.raises(_sqlite3.ProgrammingError):
-        con.execute('insert into foo(x) values (?)', seq())
-    with pytest.raises(_sqlite3.ProgrammingError):
-        con.execute('insert into foo(x) values (?)', {2:2})
-    with pytest.raises(ValueError) as e:
-        con.execute('insert into foo(x) values (?)', 2)
-    assert str(e.value) == 'parameters are of unsupported type'
-
-def test_explicit_begin(con):
-    con.execute('BEGIN')
-    con.execute('BEGIN ')
-    con.execute('BEGIN')
-    con.commit()
-    con.execute('BEGIN')
-    con.commit()
-
-def test_row_factory_use(con):
-    con.row_factory = 42
-    con.execute('select 1')
-
-def test_returning_blob_must_own_memory(con):
-    import gc
-    con.create_function("returnblob", 0, lambda: buffer("blob"))
-    cur = con.execute("select returnblob()")
-    val = cur.fetchone()[0]
-    for i in range(5):
-        gc.collect()
-        got = (val[0], val[1], val[2], val[3])
-        assert got == ('b', 'l', 'o', 'b')
-    # in theory 'val' should be a read-write buffer
-    # but it's not right now
-    pytest.skip("in theory 'val' should be a read-write buffer")
-    val[1] = 'X'
-    got = (val[0], val[1], val[2], val[3])
-    assert got == ('b', 'X', 'o', 'b')
-
-def test_description_after_fetchall(con):
-    cur = con.cursor()
-    cur.execute("select 42").fetchall()
-    assert cur.description is not None
-
-def test_executemany_lastrowid(con):
-    cur = con.cursor()
-    cur.execute("create table test(a)")
-    cur.executemany("insert into test values (?)", [[1], [2], [3]])
-    assert cur.lastrowid is None
-
-
-def test_authorizer_bad_value(con):
-    def authorizer_cb(action, arg1, arg2, dbname, source):
-        return 42
-    con.set_authorizer(authorizer_cb)
-    with pytest.raises(_sqlite3.OperationalError) as e:
-        con.execute('select 123')
-    major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
-    if (int(major), int(minor), int(micro)) >= (3, 6, 14):
-        assert str(e.value) == 'authorizer malfunction'
-    else:
-        assert str(e.value) == \
-            ("illegal return value (1) from the authorization function - "
-             "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
-
-
-def test_issue1573(con):
-    cur = con.cursor()
-    cur.execute(u'SELECT 1 as m&#233;il')
-    assert cur.description[0][0] == u"m&#233;il".encode('utf-8')
-
-def test_adapter_exception(con):
-    def cast(obj):
-        raise ZeroDivisionError
-
-    _sqlite3.register_adapter(int, cast)
-    try:
-        cur = con.cursor()
-        cur.execute("select ?", (4,))
-        val = cur.fetchone()[0]
-        # Adapter error is ignored, and parameter is passed as is.
-        assert val == 4
-        assert type(val) is int
-    finally:
-        del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
+        global _sqlite3
+        from lib_pypy import _sqlite3

From noreply at buildbot.pypy.org  Fri Dec 12 18:10:40 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 12 Dec 2014 18:10:40 +0100 (CET)
Subject: [pypy-commit] pypy default: clarify names
Message-ID: <20141212171040.0247D1C0188@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: 
Changeset: r74906:29c3daa5d709
Date: 2014-12-12 12:10 -0500
http://bitbucket.org/pypy/pypy/changeset/29c3daa5d709/

Log:	clarify names

diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -3,6 +3,7 @@
 from __future__ import absolute_import
 import py
 
+
 class BaseTestDatetime:
     def test_repr(self):
         print datetime
@@ -210,11 +211,13 @@
             naive == aware
         assert str(e.value) == "can't compare offset-naive and offset-aware times"
 
-class TestDatetimeCPython(BaseTestDatetime):
+
+class TestDatetimeHost(BaseTestDatetime):
     def setup_class(cls):
         global datetime
         import datetime
 
+
 class TestDatetimePyPy(BaseTestDatetime):
     def setup_class(cls):
         global datetime
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 """Tests for _sqlite3.py"""
 
+from __future__ import absolute_import
 import pytest
 import sys
 
@@ -263,7 +264,7 @@
             del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
 
 
-class TestSQLiteCPython(BaseTestSQLite):
+class TestSQLiteHost(BaseTestSQLite):
     def setup_class(cls):
         global _sqlite3
         import _sqlite3

From noreply at buildbot.pypy.org  Fri Dec 12 18:48:31 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Fri, 12 Dec 2014 18:48:31 +0100 (CET)
Subject: [pypy-commit] benchmarks min_5_secs: Merge default.
Message-ID: <20141212174832.01D0D1C0188@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: min_5_secs
Changeset: r299:e3aea38bffa2
Date: 2014-12-12 17:47 +0000
http://bitbucket.org/pypy/benchmarks/changeset/e3aea38bffa2/

Log:	Merge default.

diff too long, truncating to 2000 out of 119576 lines

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -82,7 +82,8 @@
              'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast',
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
              'json_bench', 'pidigits', 'hexiom2', 'eparse', 'deltablue',
-             'bm_dulwich_log']:
+             'bm_dulwich_log', 'bm_krakatau', 'bm_mdp', 'pypy_interp',
+             'bm_icbd']:
     _register_new_bm(name, name, globals(), **opts.get(name, {}))
 
 for name in ['names', 'iteration', 'tcp', 'pb', ]:#'web']:#, 'accepts']:
diff --git a/lib/Krakatau/Documentation/assembler.txt b/lib/Krakatau/Documentation/assembler.txt
deleted file mode 100644
--- a/lib/Krakatau/Documentation/assembler.txt
+++ /dev/null
@@ -1,88 +0,0 @@
-Krakatau Assembler Syntax
-
-This guide is intended to help write bytecode assembly files for use with the Krakatau assembler. It assumes that you are already familiar with the JVM classfile format and how to write bytecode. If not, you can find a simple tutorial to writing bytecode at https://greyhat.gatech.edu/wiki/index.php?title=Java_Bytecode_Tutorial. You can also find some examples of assembler files in the examples directory.
-
-Krakatau syntax is largely backwards compatible with the classic Jasmin assembler syntax. In a couple of places, backwards compatibility is broken either by the introduction of new keywords or to fix ambiguities in the Jasmin syntax. However, Krakatau is not necessarily compatible with the extensions introduced by JasminXT.
-
-The basic format for an assembler file consists of a list of classfile entries. Each entry will result in the generation of a seperate classfile, so a single assembly file can contain multiple classes where convienent. These entries are completely independent - mutiple classes never share constant pool entries, fields, methods, or directives, even the version directive. Each one has the format
-
-.bytecode major minor  (optional)
-class directives
-.class classref
-.super classref
-interface declarations
-class directives
-topitems
-.end class
-
-The .end class on the final entry may be ommitted. So the simplest possible assembler file to declare a class named Foo would be
-
-.class Foo
-.super java/lang/Object
-
-To declare three classes A, B, and C in the same file with B and C inheriting from A and different versions, you could do
-
-.class A
-.super java/lang/Object
-.end class
-.class B
-.super A
-.end class
-.class C
-.super A
-
-The classfile version is specified by the .bytecode directive. It is specified by major, minor, a pair of decimal integers. If ommitted, the default is version 49.0. So the following is equivalent to the earlier example
-
-.bytecode 49 0
-.class Foo
-.super java/lang/Object
-
-Other class directives include .runtimevisible, .runtimeinvisible, .signature, .attribute, .source, .inner, .innerlength, and .enclosing. These are used to control the attributes of the class and will be covered later.
-
-Topitems are the actual meat of the class. There are three types: fields, methods, and constant definitions. The last is unique to Krakatau and is closely related to the rest of the syntax. In Krakatau, there are multiple ways to specify a constant pool entry. The most common are via WORD tokens, symbolic references and numerical references. The later constist of square brackets with lowercase alphanumerics and underscores inside.
-
-When you specify .class Foo, the string Foo isn't directly included in the output. The classfile format says that the class field is actually a two byte index into the constant pool of the classfile. This points to a Class_info which points to a Utf8_info which holds the actual name of the class. Therefore, Krakatau implicitly creates constant pool entries and inserts the appropriate references. But this process can be controlled more directly.
-
-Instead of writing
-.class Foo
-.super java/lang/Object
-
-You could explicitly write out all the classfile references as follows
-
-.class [foocls]
-.super [objcls]
-
-.const [foocls] = Class [fooutf]
-.const [fooutf] = Utf8 Foo
-.const [objcls] = Class [objutf]
-.const [objutf] = Utf8 java/lang/Object
-
-There are two types of references. If the contents are a decimal int, then it is a direct numerical reference to a particular slot in the constant pool. You are responsible for making sure that everything is consistent and that the contents of that slot are valid. This option is most useful for specifiying the null entry [0]. For example, to express the Object class itself, one would do
-
-.class java/lang/Object
-.super [0]
-
-If the contents are any other nonempty lowercase alphanumeric + underscores string, it is interperted as a symbolic reference. This is a reference to some slot but you don't care which one. Krakatau will pick an available slot and fill it in automatically. Symbolic references may be ommitted from the generated classfile if unused or merged with identical entries, including automatically generated entries.
-
-With that out of the way, the basic form of a constant definition is
-
-.const ref = entrytype arguments
-
-You can also just define one reference in terms of another. You are responsible for making sure there are no circular references or duplicated definitions. If not, unpredictable results may occur.
-
-Examples include
-
-.const [1] = Class Foo
-.const [1] = Class [fooutf]
-.const [89] = Long 1234567L
-.const [mynat] = NameAndType main ([Ljava/lang/String;)V
-.const [myref] = [myotherref]
-.const [really42] = [42]
-
-
-
-
-
-
-
-
diff --git a/lib/Krakatau/Krakatau/__init__.py b/lib/Krakatau/Krakatau/__init__.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-'''Monkeypatch a fix for bugs.python.org/issue9825 in case users are running an old version.'''
-import collections
-try:
-    del collections.OrderedDict.__del__
-except AttributeError:
-    pass
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/__init__.py b/lib/Krakatau/Krakatau/assembler/__init__.py
deleted file mode 100644
diff --git a/lib/Krakatau/Krakatau/assembler/assembler.py b/lib/Krakatau/Krakatau/assembler/assembler.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/assembler.py
+++ /dev/null
@@ -1,580 +0,0 @@
-import collections
-import struct, operator
-
-from . import instructions, codes
-from .. import constant_pool
-from ..classfile import ClassFile
-from ..method import Method
-from ..field import Field
-
-class AssemblerError(Exception):
-    def __init__(self, message, data=None):
-        super(AssemblerError, self).__init__(message)
-        self.data = data
-
-def error(msg):
-    raise AssemblerError(msg)
-
-class PoolRef(object):
-    def __init__(self, *args, **kwargs):
-        self.index = kwargs.get('index')
-        self.lbl = kwargs.get('lbl')
-        self.args = args
-
-    def toIndex(self, pool, forbidden=(), **kwargs):
-        if self.index is not None:
-            return self.index
-        if self.lbl:
-            self.index = pool.getLabel(self.lbl, forbidden, **kwargs)
-        else:
-            self.args = [(x.toIndex(pool) if isinstance(x, PoolRef) else x) for x in self.args]
-            self.index = pool.getItem(*self.args, **kwargs)
-        return self.index
-
-class PoolInfo(object):
-    def __init__(self):
-        self.pool = constant_pool.ConstPool()
-        self.lbls = {}
-        self.fixed = {} # constant pool entries in a specific slot
-        self.bootstrap = [] #entries for the BootstrapMethods attribute if any
-
-    def getLabel(self, lbl, forbidden=(), **kwargs):
-        if lbl in forbidden:
-            error('Circular constant pool reference: ' + ', '.join(forbidden))
-        forbidden = forbidden + (lbl,)
-        return self.lbls[lbl].toIndex(self, forbidden, **kwargs)
-
-    def getItem(self, type_, *args, **kwargs):
-        if type_ == 'InvokeDynamic':
-            self.bootstrap.append(args[:-1])
-            args = len(self.bootstrap)-1, args[-1]    
-        return self.pool.addItem((type_, tuple(args)), **kwargs)
-
-    def Utf8(self, s):
-        return self.getItem('Utf8', s)
-
-    def assignFixedSlots(self):
-        self.pool.reserved.update(self.fixed)
-        for i,v in self.fixed.items():
-            if v.args and v.args[0] in ('Double','Long'):
-                self.pool.reserved.add(i+1)
-                
-        #TODO - order these in terms of dependencies?
-        for index, value in self.fixed.items():
-            used = value.toIndex(self, index=index)
-            if used != index: #we need to copy an existing item
-                self.pool.copyItem(used, index)
-
-_format_ops = collections.defaultdict(tuple)
-_format_ops[''] = instructions.instrs_noarg
-_format_ops['>B'] = 'iload', 'lload', 'fload', 'dload', 'aload', 'istore', 'lstore', 'fstore', 'dstore', 'astore', 'ret'
-_format_ops['>h'] = 'ifeq', 'ifne', 'iflt', 'ifge', 'ifgt', 'ifle', 'if_icmpeq', 'if_icmpne', 'if_icmplt', 'if_icmpge', 'if_icmpgt', 'if_icmple', 'if_acmpeq', 'if_acmpne', 'goto', 'jsr', 'ifnull', 'ifnonnull'
-_format_ops['>H'] = 'ldc_w', 'ldc2_w', 'getstatic', 'putstatic', 'getfield', 'putfield', 'invokevirtual', 'invokespecial', 'invokestatic', 'new', 'anewarray', 'checkcast', 'instanceof'
-
-_format_ops['>b'] += 'bipush', 
-_format_ops['>Bb'] += 'iinc', 
-_format_ops['>h'] += 'sipush', 
-_format_ops['>HB'] += 'multianewarray',
-_format_ops['>HBB'] += 'invokeinterface',
-_format_ops['>HH'] += 'invokedynamic',
-_format_ops['>B'] += 'ldc', 'newarray'
-_format_ops['>i'] += 'goto_w', 'jsr_w'
-
-op_structs = {}
-for fmt, ops in _format_ops.items():
-    _s = struct.Struct(fmt)
-    for _op in ops:
-        op_structs[_op] = _s
-
-def getPadding(pos):
-    return (3-pos) % 4
-
-def getInstrLen(instr, pos):
-    op = instr[0]
-    if op in op_structs:
-        return 1 + op_structs[op].size
-    elif op == 'wide':
-        return 2 + 2 * len(instr[1][1])
-    else:
-        padding = getPadding(pos)
-        count = len(instr[1][1])
-        if op == 'tableswitch':
-            return 13 + padding + 4*count
-        else:
-            return 9 + padding + 8*count 
-
-def assembleInstruction(instr, labels, pos, pool):
-    def lbl2Off(lbl):
-        if lbl not in labels:
-            del labels[None]
-            error('Undefined label: {}\nDefined labels for current method are: {}'.format(lbl, ', '.join(sorted(labels))))
-        return labels[lbl] - pos
-
-    op = instr[0]
-    first = chr(instructions.allinstructions.index(op))
-
-    instr = [(x.toIndex(pool) if isinstance(x, PoolRef) else x) for x in instr[1:]]
-    if op in instructions.instrs_lbl:
-        instr[0] = lbl2Off(instr[0])
-    if op in op_structs:
-        rest = op_structs[op].pack(*instr)
-        return first+rest
-    elif op == 'wide':
-        subop, args = instr[0]
-        prefix = chr(instructions.allinstructions.index(subop))
-        fmt = '>Hh' if len(args) > 1 else '>H'
-        rest = struct.pack(fmt, *args)
-        return first + prefix + rest
-    else:
-        padding = getPadding(pos)
-        param, jumps, default = instr[0]
-        default = lbl2Off(default)
-
-        if op == 'tableswitch':
-            jumps = map(lbl2Off, jumps)
-            low, high = param, param + len(jumps)-1
-            temp = struct.Struct('>i')
-            part1 = first + '\0'*padding + struct.pack('>iii', default, low, high)
-            return part1 + ''.join(map(temp.pack, jumps))
-        elif op == 'lookupswitch':
-            jumps = {k:lbl2Off(lbl) for k,lbl in jumps}
-            jumps = sorted(jumps.items())
-            temp = struct.Struct('>ii')
-            part1 = first + '\0'*padding + struct.pack('>ii', default, len(jumps))
-            part2 = ''.join(map(temp.pack, *zip(*jumps))) if jumps else ''
-            return part1 + part2
-
-def groupList(pairs):
-    d = collections.defaultdict(list)
-    for k,v in pairs:
-        d[k].append(v)
-    return d
-
-def splitList(pairs):
-    d = groupList(pairs)
-    return d[False], d[True]
-       
-def assembleCodeAttr(statements, pool, version, addLineNumbers, jasmode):
-    directives, lines = splitList(statements)
-    dir_offsets = collections.defaultdict(list)
-
-    offsets = []
-    labels = {}
-    pos = 0
-    #first run through to calculate bytecode offsets
-    #this is greatly complicated due to the need to
-    #handle Jasmine line number directives
-    for t, statement in statements:
-        if t:
-            lbl, instr = statement
-            labels[lbl] = pos
-            if instr is not None:
-                offsets.append(pos)
-                pos += getInstrLen(instr, pos)
-        #some directives require us to keep track of the corresponding bytecode offset
-        elif statement[0] in ('.line','.stackmap'):
-            dir_offsets[statement[0]].append(pos)
-    code_len = pos
-
-    code_bytes = ''
-    for lbl, instr in lines:
-        if instr is not None:
-            code_bytes += assembleInstruction(instr, labels, len(code_bytes), pool)
-    assert(len(code_bytes) == code_len)
-
-    directive_dict = groupList(directives)
-    limits = groupList(directive_dict['.limit'])
-
-    stack = min(limits['stack'] + [65535]) 
-    locals_ = min(limits['locals'] + [65535]) 
-
-    excepts = []
-    for name, start, end, target in directive_dict['.catch']:
-        #Hack for compatibility with Jasmin
-        if jasmode and name.args and (name.args[1].args == ('Utf8','all')):
-            name.index = 0
-        vals = labels[start], labels[end], labels[target], name.toIndex(pool)
-        excepts.append(struct.pack('>HHHH',*vals))
-    
-    attributes = []
-
-    #StackMapTable
-    def pack_vt(vt):
-        s = chr(codes.vt_codes[vt[0]])
-        if vt[0] == 'Object':
-            s += struct.pack('>H', vt[1].toIndex(pool))        
-        elif vt[0] == 'Uninitialized':
-            s += struct.pack('>H', labels[vt[1]])
-        return s
-
-    if directive_dict['.stackmap']:
-        frames = []
-        last_pos = -1
-
-        for pos, info in zip(dir_offsets['.stackmap'], directive_dict['.stackmap']):
-            offset = pos - last_pos - 1
-            last_pos = pos
-            assert(offset >= 0)
-
-            tag = info[0]
-            if tag == 'same':
-                if offset >= 64:
-                    error('Max offset on a same frame is 63.')
-                frames.append(chr(offset))            
-            elif tag == 'same_locals_1_stack_item':
-                if offset >= 64:
-                    error('Max offset on a same_locals_1_stack_item frame is 63.')
-                frames.append(chr(64 + offset) + pack_vt(info[2][0]))            
-            elif tag == 'same_locals_1_stack_item_extended':
-                frames.append(struct.pack('>BH', 247, offset) + pack_vt(info[2][0]))            
-            elif tag == 'chop':
-                if not (1 <= info[1] <= 3):
-                    error('Chop frame can only remove 1-3 locals')
-                frames.append(struct.pack('>BH', 251-info[1], offset))
-            elif tag == 'same_extended':
-                frames.append(struct.pack('>BH', 251, offset))
-            elif tag == 'append':
-                local_vts = map(pack_vt, info[2])
-                if not (1 <= len(local_vts) <= 3):
-                    error('Append frame can only add 1-3 locals')
-                frames.append(struct.pack('>BH', 251+len(local_vts), offset) + ''.join(local_vts))
-            elif tag == 'full':
-                local_vts = map(pack_vt, info[2])
-                stack_vts = map(pack_vt, info[3])
-                frame = struct.pack('>BH', 255, offset)
-                frame += struct.pack('>H', len(local_vts)) + ''.join(local_vts)
-                frame += struct.pack('>H', len(stack_vts)) + ''.join(stack_vts)
-                frames.append(frame)
-
-        sm_body = ''.join(frames)
-        sm_attr = struct.pack('>HIH', pool.Utf8("StackMapTable"), len(sm_body)+2, len(frames)) + sm_body
-        attributes.append(sm_attr)
-
-    #line number attribute
-    if addLineNumbers and not directive_dict['line']:
-        dir_offsets['line'] = directive_dict['line'] = offsets
-    if directive_dict['line']:
-        lntable = [struct.pack('>HH',x,y) for x,y in zip(dir_offsets['line'], directive_dict['line'])]
-        ln_attr = struct.pack('>HIH', pool.Utf8("LineNumberTable"), 2+4*len(lntable), len(lntable)) + ''.join(lntable)        
-        attributes.append(ln_attr)
-
-    if directive_dict['.var']:
-        sfunc = struct.Struct('>HHHHH').pack
-        vartable = []
-        for index, name, desc, start, end in directive_dict['.var']:
-            start, end = labels[start], labels[end]
-            name, desc = name.toIndex(pool), desc.toIndex(pool)
-            vartable.append(sfunc(start, end-start, name, desc, index))
-        var_attr = struct.pack('>HIH', pool.Utf8("LocalVariableTable"), 2+10*len(vartable), len(vartable)) + ''.join(vartable)        
-        attributes.append(var_attr)
-
-    if not code_len:
-        return None
-
-    for attrname, data in directive_dict['.codeattribute']:
-        attr = struct.pack('>HI', attrname.toIndex(pool), len(data)) + data
-        attributes.append(attr)        
-
-
-    #Old versions use shorter fields for stack, locals, and code length
-    header_fmt = '>HHI' if version > (45,2) else '>BBH'
-
-    name_ind = pool.Utf8("Code")
-    attr_len = struct.calcsize(header_fmt) + 4 + len(code_bytes) + 8*len(excepts) + sum(map(len, attributes))
-    
-    assembled_bytes = struct.pack('>HI', name_ind, attr_len)
-    assembled_bytes += struct.pack(header_fmt, stack, locals_, len(code_bytes))
-    assembled_bytes += code_bytes
-    assembled_bytes += struct.pack('>H', len(excepts)) + ''.join(excepts)
-    assembled_bytes += struct.pack('>H', len(attributes)) + ''.join(attributes)
-    return assembled_bytes
-
-def _assembleEVorAnnotationSub(pool, init_args, isAnnot):
-    #call types
-    C_ANNOT, C_ANNOT2, C_EV = range(3)
-    init_callt = C_ANNOT if isAnnot else C_EV
-
-    stack = [(init_callt, init_args)]
-    parts = []
-    add = parts.append
-
-    while stack:
-        callt, args = stack.pop()
-
-        if callt == C_ANNOT:
-            typeref, keylines = args
-            add(struct.pack('>HH', typeref.toIndex(pool), len(keylines)))
-            for pair in reversed(keylines):
-                stack.append((C_ANNOT2, pair))
-
-        elif callt == C_ANNOT2:
-            name, val = args
-            add(struct.pack('>H', name.toIndex(pool)))
-            stack.append((C_EV, val))
-
-        elif callt == C_EV:
-            tag, data = args
-            assert(tag in codes.et_rtags)
-            add(tag)
-
-            if tag in 'BCDFIJSZsc':
-                add(struct.pack('>H', data[0].toIndex(pool)))
-            elif tag == 'e':
-                add(struct.pack('>HH', data[0].toIndex(pool), data[1].toIndex(pool)))
-            elif tag == '@':
-                stack.append((C_ANNOT, data[0]))
-            elif tag == '[':
-                add(struct.pack('>H', len(data[1])))
-                for arrval in reversed(data[1]):
-                    stack.append((C_EV, arrval))
-    return ''.join(parts)
-
-def assembleElementValue(val, pool):
-    return  _assembleEVorAnnotationSub(pool, val, False)
-
-def assembleAnnotation(annotation, pool):
-    return  _assembleEVorAnnotationSub(pool, annotation, True)
-
-def assembleMethod(header, statements, pool, version, addLineNumbers, jasmode):
-    mflags, (name, desc) = header
-    name = name.toIndex(pool)
-    desc = desc.toIndex(pool)
-
-    flagbits = map(Method.flagVals.get, mflags)
-    flagbits = reduce(operator.__or__, flagbits, 0)
-
-    meth_statements, code_statements = splitList(statements)
-
-    method_attributes = []
-    code_attr = assembleCodeAttr(code_statements, pool, version, addLineNumbers, jasmode)
-    if code_attr is not None:
-        method_attributes.append(code_attr)
-
-    directive_dict = groupList(meth_statements)
-    if directive_dict['.throws']:
-        t_inds = [struct.pack('>H', x.toIndex(pool)) for x in directive_dict['.throws']]
-        throw_attr = struct.pack('>HIH', pool.Utf8("Exceptions"), 2+2*len(t_inds), len(t_inds)) + ''.join(t_inds)        
-        method_attributes.append(throw_attr)
-
-    #Runtime annotations
-    for vis in ('Invisible','Visible'):
-        paramd = groupList(directive_dict['.runtime'+vis.lower()])
-
-        if None in paramd:
-            del paramd[None]
-
-        if paramd:
-            parts = []
-            for i in range(max(paramd)):
-                annotations = [assembleAnnotation(a, pool) for a in paramd[i]]
-                part = struct.pack('>H', len(annotations)) + ''.join(annotations)
-                parts.append(part)
-            attrlen = 1+sum(map(len, parts))
-            attr = struct.pack('>HIB', pool.Utf8("Runtime{}ParameterAnnotations".format(vis)), attrlen, len(parts)) + ''.join(parts)
-            method_attributes.append(attr)
-
-    if '.annotationdefault' in directive_dict:
-        val = directive_dict['.annotationdefault'][0]
-        data = assembleElementValue(val, pool)
-        attr = struct.pack('>HI', pool.Utf8("AnnotationDefault"), len(data)) + data        
-        method_attributes.append(attr)
-
-    assembleClassFieldMethodAttributes(method_attributes.append, directive_dict, pool)
-    return struct.pack('>HHHH', flagbits, name, desc, len(method_attributes)) + ''.join(method_attributes)
-
-def getLdcRefs(statements):
-    lines = [x[1][1] for x in statements if x[0] and x[1][0]]
-    instructions = [x[1] for x in lines if x[1] is not None]
-
-    for instr in instructions:
-        op = instr[0]
-        if op == 'ldc':
-            yield instr[1]
- 
-def addLdcRefs(methods, pool):
-    def getRealRef(ref, forbidden=()):
-        '''Get the root PoolRef associated with a given PoolRef, following labels'''
-        if ref.index is None and ref.lbl:
-            if ref.lbl in forbidden:
-                error('Circular constant pool reference: ' + ', '.join(forbidden))
-            forbidden = forbidden + (ref.lbl,)
-            return getRealRef(pool.lbls[ref.lbl], forbidden) #recursive call
-        return ref
-
-    #We attempt to estimate how many slots are needed after merging identical entries
-    #So we can reserve the correct number of slots without leaving unused gaps
-    #However, in complex cases, such as string/class/mt referring to an explicit
-    #reference, we may overestimate
-    ldc_refs = collections.defaultdict(set)
-
-    for header, statements in methods:
-        for ref in getLdcRefs(statements):
-            ref = getRealRef(ref)
-            if ref.index is not None:
-                continue
-
-            type_ = ref.args[0]
-            if type_ in ('Int','Float'):
-                key = ref.args[1]
-            elif type_ in ('String','Class','MethodType'): 
-                uref = getRealRef(ref.args[1])
-                key = uref.index, uref.args[1:]
-            else: #for MethodHandles, don't even bother trying to estimate merging
-                key = ref.args[1:] 
-            ldc_refs[type_].add(key)    
-
-    #TODO - make this a little cleaner so we don't have to mess with the ConstantPool internals
-    num = sum(map(len, ldc_refs.values()))
-    slots = [pool.pool.getAvailableIndex() for _ in range(num)]
-    pool.pool.reserved.update(slots)
-
-    for type_ in ('Int','Float'):
-        for arg in ldc_refs[type_]:
-            pool.getItem(type_, arg, index=slots.pop())
-    for type_ in ('String','Class','MethodType'):
-        for ind,args in ldc_refs[type_]:
-            arg = ind if ind is not None else pool.Utf8(*args)
-            pool.getItem(type_, arg, index=slots.pop())
-    for type_ in ('MethodHandle',):
-        for code, ref in ldc_refs[type_]:
-            pool.getItem(type_, code, ref.toIndex(pool), index=slots.pop())
-    assert(not slots)
-    assert(not pool.pool.reserved)
-
-def assembleClassFieldMethodAttributes(addcb, directive_dict, pool):
-    for vis in ('Invisible','Visible'):
-        paramd = groupList(directive_dict['.runtime'+vis.lower()])
-        if None in paramd:
-            annotations = [assembleAnnotation(a, pool) for a in paramd[None]]
-            attrlen = 2+sum(map(len, annotations))
-            attr = struct.pack('>HIH', pool.Utf8("Runtime{}Annotations".format(vis)), attrlen, len(annotations)) + ''.join(annotations)
-            addcb(attr)
-
-    for name in directive_dict['.signature']:
-        attr = struct.pack('>HIH', pool.Utf8("Signature"), 2, name.toIndex(pool))
-        addcb(attr)
-
-    #.innerlength directive overrides the normal attribute length calculation
-    hasoverride = len(directive_dict['.innerlength']) > 0
-
-    for name, data in directive_dict['.attribute']:    
-        name_ind = name.toIndex(pool)
-
-        if hasoverride and pool.pool.getArgsCheck('Utf8', name_ind) == 'InnerClasses':
-            attrlen = directive_dict['.innerlength'][0]
-        else:
-            attrlen = len(data)
-
-        attr = struct.pack('>HI', name_ind, attrlen) + data
-        addcb(attr)
-
-def assembleClassAttributes(addcb, directive_dict, pool, addLineNumbers, jasmode, filename):
-
-    sourcefile = directive_dict.get('.source',[None])[0] #PoolRef or None
-    if jasmode and not sourcefile:
-        sourcefile = pool.Utf8(filename)
-    elif addLineNumbers and not sourcefile:
-        sourcefile = pool.Utf8("SourceFile")
-    if sourcefile:
-        attr = struct.pack('>HIH', pool.Utf8("SourceFile"), 2, sourcefile.toIndex(pool))
-        addcb(attr)
-
-    if '.inner' in directive_dict:
-        parts = []
-        for inner, outer, name, flags in directive_dict['.inner']:
-            flagbits = map(ClassFile.flagVals.get, flags)
-            flagbits = reduce(operator.__or__, flagbits, 0)
-            part = struct.pack('>HHHH', inner.toIndex(pool), outer.toIndex(pool), name.toIndex(pool), flagbits)
-            parts.append(part)
-
-        #.innerlength directive overrides the normal attribute length calculation
-        innerlen = 2+8*len(parts) if '.innerlength' not in directive_dict else directive_dict['.innerlength'][0]
-        attr = struct.pack('>HIH', pool.Utf8("InnerClasses"), innerlen, len(parts)) + ''.join(parts)
-        addcb(attr)
-
-    if '.enclosing' in directive_dict:
-        class_, nat = directive_dict['.enclosing'][0]
-        attr = struct.pack('>HIHH', pool.Utf8("EnclosingMethod"), 4, class_.toIndex(pool), nat.toIndex(pool))
-        addcb(attr)
-
-    assembleClassFieldMethodAttributes(addcb, directive_dict, pool)
-
-
-def assemble(tree, addLineNumbers, jasmode, filename):
-    pool = PoolInfo()
-    version, cattrs1, classdec, superdec, interface_decs, cattrs2, topitems = tree
-    if not version: #default to version 49.0 except in Jasmin compatibility mode
-        version = (45,3) if jasmode else (49,0)
-
-    #scan topitems, plus statements in each method to get cpool directives
-    interfaces = []
-    fields = []
-    methods = []
-    attributes = []
-
-    directive_dict = groupList(cattrs1 + cattrs2)
-    top_d = groupList(topitems)
-
-    for slot, value in top_d['const']:
-        if slot.index is not None:
-            pool.fixed[slot.index] = value
-        else:
-            pool.lbls[slot.lbl] = value
-    pool.assignFixedSlots()
-
-    #Now find all cp references used in an ldc instruction
-    #Since they must be <=255, we give them priority in assigning slots
-    #to maximize the chance of a successful assembly
-    addLdcRefs(top_d['method'], pool)
-
-    for flags, name, desc, const, field_directives in top_d['field']:
-        flagbits = map(Field.flagVals.get, flags)
-        flagbits = reduce(operator.__or__, flagbits, 0)
-        name = name.toIndex(pool)
-        desc = desc.toIndex(pool)
-
-        fattrs = []
-        if const is not None:
-            attr = struct.pack('>HIH', pool.Utf8("ConstantValue"), 2, const.toIndex(pool))
-            fattrs.append(attr)
-
-        assembleClassFieldMethodAttributes(fattrs.append, groupList(field_directives), pool)
-
-        field_code = struct.pack('>HHHH', flagbits, name, desc, len(fattrs)) + ''.join(fattrs)
-        fields.append(field_code)
-
-    for header, statements in top_d['method']:
-        methods.append(assembleMethod(header, statements, pool, version, addLineNumbers, jasmode))
-
-    if pool.bootstrap:
-        entries = [struct.pack('>H' + 'H'*len(bsargs), bsargs[0], len(bsargs)-1, *bsargs[1:]) for bsargs in pool.bootstrap]   
-        attrbody = ''.join(entries)
-        attrhead = struct.pack('>HIH', pool.Utf8("BootstrapMethods"), 2+len(attrbody), len(entries))
-        attributes.append(attrhead + attrbody)
-
-    #Explicit class attributes
-    assembleClassAttributes(attributes.append, directive_dict, pool, addLineNumbers, jasmode, filename)
-
-    interfaces = [struct.pack('>H', x.toIndex(pool)) for x in interface_decs]
-    intf, cflags, this = classdec
-    cflags = set(cflags)
-    if intf:
-        cflags.add('INTERFACE')
-    if jasmode:
-        cflags.add('SUPER')
-
-    flagbits = map(ClassFile.flagVals.get, cflags)
-    flagbits = reduce(operator.__or__, flagbits, 0)
-    this = this.toIndex(pool)
-    super_ = superdec.toIndex(pool)
-
-    major, minor = version
-    class_code = '\xCA\xFE\xBA\xBE' + struct.pack('>HH', minor, major)
-    class_code += pool.pool.bytes()
-    class_code += struct.pack('>HHH', flagbits, this, super_)
-    for stuff in (interfaces, fields, methods, attributes):
-        bytes_ = struct.pack('>H', len(stuff)) + ''.join(stuff)
-        class_code += bytes_
-
-    name = pool.pool.getArgs(this)[0]
-    return name, class_code
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/codes.py b/lib/Krakatau/Krakatau/assembler/codes.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/codes.py
+++ /dev/null
@@ -1,11 +0,0 @@
-_handle_types = 'getField getStatic putField putStatic invokeVirtual invokeStatic invokeSpecial newInvokeSpecial invokeInterface'.split()
-handle_codes = dict(zip(_handle_types, range(1,10)))
-
-newarr_codes = dict(zip('boolean char float double byte short int long'.split(), range(4,12)))
-
-vt_keywords = ['Top','Integer','Float','Double','Long','Null','UninitializedThis','Object','Uninitialized']
-vt_codes = {k:i for i,k in enumerate(vt_keywords)}
-
-
-et_rtags = dict(zip('BCDFIJSZsce@[', 'byte char double int float long short boolean string class enum annotation array'.split()))
-et_tags = {v:k for k,v in et_rtags.items()}
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/disassembler.py b/lib/Krakatau/Krakatau/assembler/disassembler.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/disassembler.py
+++ /dev/null
@@ -1,539 +0,0 @@
-import collections
-import re
-
-from . import instructions, tokenize, parse, assembler, codes
-from ..binUnpacker import binUnpacker
-from ..classfile import ClassFile
-
-MAX_INLINE_LENGTH = 50
-
-rhandle_codes = {v:k for k,v in codes.handle_codes.items()}
-rnewarr_codes = {v:k for k,v in codes.newarr_codes.items()}
-
-not_word_regex = '(?:{}|{}|{}|;)'.format(tokenize.int_base, tokenize.float_base, tokenize.t_CPINDEX)
-not_word_regex = re.compile(not_word_regex, re.VERBOSE)
-is_word_regex = re.compile(tokenize.t_WORD.__doc__+'$')
-assert(is_word_regex.match("''") is None)
-
-def isWord(s):
-    '''Determine if s can be used as an inline word'''
-    if s in parse.badwords or (not_word_regex.match(s) is not None):
-        return False
-    #eliminate unprintable characters below 32
-    #also, don't allow characters above 127 to keep things simpler
-    return (is_word_regex.match(s) is not None) and min(s) > ' ' and max(s) <= '\x7f'
-
-def rstring(s, allowWord=True):
-    '''Returns a representation of the string. If allowWord is true, it will be unquoted if possible'''
-    if allowWord and isWord(s):
-        return s
-    try:
-        if s.encode('ascii') == s:
-            return repr(str(s))
-    except (UnicodeEncodeError, UnicodeDecodeError):
-        pass
-    return repr(s)
-
-class PoolManager(object):
-    def __init__(self, pool):
-        self.const_pool = pool #keep this around for the float conversion function
-        self.pool = pool.pool
-        self.bootstrap_methods = [] #filled in externally
-        self.used = set() #which cp entries are used non inline and so must be printed
-
-        #For each type, store the function needed to generate the rhs of a constant pool specifier
-        temp1 = lambda ind: rstring(self.cparg1(ind))
-        temp2 = lambda ind: self.utfref(self.cparg1(ind))
-
-        self.cpref_table = {
-            "Utf8": temp1,
-
-            "Class": temp2,
-            "String": temp2,
-            "MethodType": temp2,
-
-            "NameAndType": self.multiref,
-            "Field": self.multiref,
-            "Method": self.multiref,
-            "InterfaceMethod": self.multiref,
-
-            "Int": self.ldc,
-            "Long": self.ldc,
-            "Float": self.ldc,
-            "Double": self.ldc,
-
-            "MethodHandle": self.methodhandle_notref,
-            "InvokeDynamic": self.invokedynamic_notref,
-            }
-
-    def cparg1(self, ind):
-        return self.pool[ind][1][0]
-
-    def inlineutf(self, ind, allowWord=True):
-        '''Returns the word if it's short enough to inline, else None'''
-        arg = self.cparg1(ind)
-        rstr = rstring(arg, allowWord=allowWord)
-        if len(rstr) <= MAX_INLINE_LENGTH:
-            return rstr
-        return None
-
-    def ref(self, ind):
-        self.used.add(ind)
-        return '[_{}]'.format(ind)
-
-    def utfref(self, ind):
-        if ind == 0:
-            return '[0]'
-        inline = self.inlineutf(ind)
-        return inline if inline is not None else self.ref(ind)
-
-    #Also works for Strings and MethodTypes
-    def classref(self, ind):
-        if ind == 0:
-            return '[0]'
-        inline = self.inlineutf(self.cparg1(ind))
-        return inline if inline is not None else self.ref(ind)
-
-    #For Field, Method, IMethod, and NameAndType. Effectively notref
-    def multiref(self, ind):
-        if ind == 0:
-            return '[0]'
-        typen, args = self.pool[ind]
-        if typen == "Utf8":
-            return self.utfref(ind)
-        elif typen == "Class":
-            return self.classref(ind)
-        return ' '.join(map(self.multiref, args))
-
-    #Special case for instruction fieldrefs as a workaround for Jasmin's awful syntax
-    def notjasref(self, ind):
-        typen, args = self.pool[ind]
-        cind = self.cparg1(ind)
-        inline = self.inlineutf(self.cparg1(cind))
-        if inline is None:
-            return self.ref(ind)
-        return inline + ' ' + self.multiref(args[1])
-
-    def ldc(self, ind):
-        typen, args = self.pool[ind]
-        arg = args[0]
-
-        if typen == 'String':
-            inline = self.inlineutf(arg, allowWord=False)
-            return inline if inline is not None else self.ref(ind)
-        elif typen in ('Int','Long','Float','Double'):
-            if typen == "Float" or typen == "Double":
-                arg = self.const_pool.getArgs(ind)[0]
-
-            rstr = repr(arg).rstrip("Ll")
-            if typen == "Float" or typen == "Long":
-                rstr += typen[0]
-            return rstr
-        else:
-            return self.ref(ind)
-
-    def methodhandle_notref(self, ind):
-        typen, args = self.pool[ind]
-        code = rhandle_codes[args[0]]
-        return code + ' ' + self.ref(args[1])
-
-    def invokedynamic_notref(self, ind):
-        typen, args = self.pool[ind]
-        bs_args = self.bootstrap_methods[args[0]]
-
-        parts = [self.methodhandle_notref(bs_args[0])]
-        parts += map(self.ref, bs_args[1:])
-        parts += [':', self.multiref(args[1])]
-        return ' '.join(parts)
-
-    def printConstDefs(self, add):
-        defs = {}
-
-        while self.used:
-            temp, self.used = self.used, set()
-            for ind in temp:
-                if ind in defs:
-                    continue
-                typen = self.pool[ind][0]
-                defs[ind] = self.cpref_table[typen](ind)
-
-        for ind in sorted(defs):
-            add('.const [_{}] = {} {}'.format(ind, self.pool[ind][0], defs[ind]))
-
-def getAttributeTriples(obj): #name_ind, name, data
-    return [(name_ind, name, data1) for (name_ind, data1), (name, data2) in zip(obj.attributes_raw, obj.attributes)]
-
-def getAttributesDict(obj):
-    d = collections.defaultdict(list)
-    for ind, name, attr in getAttributeTriples(obj):
-        d[name].append((ind, attr))
-    return d
-
-fmt_lookup = {k:v.format for k,v in assembler.op_structs.items()}
-def getInstruction(b, getlbl, poolm):
-    pos = b.off
-    op = b.get('B')
-
-    name = instructions.allinstructions[op]
-    if name == 'wide':
-        name2 = instructions.allinstructions[b.get('B')]
-        if name2 == 'iinc':
-            args = list(b.get('>Hh'))
-        else:
-            args = [b.get('>H')]
-
-        parts = [name, name2] + map(str, args)
-        return '\t' + ' '.join(parts)
-    elif name == 'tableswitch' or name == 'lookupswitch':
-        padding = assembler.getPadding(pos)
-        b.getRaw(padding)
-
-        default = getlbl(b.get('>i')+pos)
-
-        if name == 'lookupswitch':
-            num = b.get('>I')
-            entries = ['\t'+name]
-            entries += ['\t\t{} : {}'.format(b.get('>i'), getlbl(b.get('>i')+pos)) for _ in range(num)]
-        else:
-            low, high = b.get('>ii')
-            num = high-low+1
-            entries = ['\t{} {}'.format(name, low)]
-            entries += ['\t\t{}'.format(getlbl(b.get('>i')+pos)) for _ in range(num)]
-        entries += ['\t\tdefault : {}'.format(default)]
-        return '\n'.join(entries)
-    else:
-        args = list(b.get(fmt_lookup[name], forceTuple=True))
-        #remove extra padding 0
-        if name in ('invokeinterface','invokedynamic'):
-            args = args[:-1]
-
-        funcs = {
-                'OP_CLASS': poolm.classref,
-                'OP_CLASS_INT': poolm.classref,
-                'OP_FIELD': poolm.notjasref, #this is a special case due to the jasmin thing
-                'OP_METHOD': poolm.multiref,
-                'OP_METHOD_INT': poolm.multiref,
-                'OP_DYNAMIC': poolm.ref,
-                'OP_LDC1': poolm.ldc,
-                'OP_LDC2': poolm.ldc,
-                'OP_NEWARR': rnewarr_codes.get,
-            }
-
-        token_t = tokenize.wordget[name]
-        if token_t == 'OP_LBL':
-            assert(len(args) == 1)
-            args[0] = getlbl(args[0]+pos)
-        elif token_t in funcs:
-            args[0] = funcs[token_t](args[0])
-
-        parts = [name] + map(str, args)
-        return '\t' + ' '.join(parts)
-
-def disMethodCode(code, add, poolm):
-    if code is None:
-        return
-    add('\t; method code size: {} bytes'.format(code.codelen))
-    add('\t.limit stack {}'.format(code.stack))
-    add('\t.limit locals {}'.format(code.locals))
-
-    lbls = set()
-    def getlbl(x):
-        lbls.add(x)
-        return 'L'+str(x)
-
-    for e in code.except_raw:
-        parts = poolm.classref(e.type_ind), getlbl(e.start), getlbl(e.end), getlbl(e.handler)
-        add('\t.catch {} from {} to {} using {}'.format(*parts))
-
-    code_attributes = getAttributesDict(code)
-    frames = getStackMapTable(code_attributes, poolm, getlbl)
-
-    instrs = []
-    b = binUnpacker(code.bytecode_raw)
-    while b.size():
-        instrs.append((b.off, getInstruction(b, getlbl, poolm)))
-    instrs.append((b.off, None))
-
-    for off, instr in instrs:
-        if off in lbls:
-            add('L{}:'.format(off))
-        if off in frames:
-            add(frames[off])
-        if instr:
-            add(instr)
-
-    #Generic code attributes
-    for name in code_attributes:
-        #We can't disassemble these because Jasmin's format for these attributes
-        #is overly cumbersome and not easy to disassemble into, but we can't just
-        #leave them as binary blobs either as they are verified by the JVM and the
-        #later two contain constant pool references which won't be preserved even
-        #if the bytecode isn't changed. For now, we just ommit them entirely.
-        #TODO - find a better solution
-        if name in ("LineNumberTable","LocalVariableTable","LocalVariableTypeTable"):
-            continue
-
-        for name_ind, attr in code_attributes[name]:
-            add('.codeattribute {} {!r}'.format(poolm.utfref(name_ind), attr))
-
-def getVerificationType(bytes_, poolm, getLbl):
-    s = codes.vt_keywords[bytes_.get('>B')]
-    if s == 'Object':
-        s += ' ' + poolm.classref(bytes_.get('>H'))
-    elif s == 'Uninitialized':
-        s += ' ' + getLbl(bytes_.get('>H'))
-    return s
-
-def getStackMapTable(code_attributes, poolm, getLbl):
-    smt_attrs = code_attributes['StackMapTable']
-
-    frames = {}
-    offset = 0
-
-    if smt_attrs:
-        assert(len(smt_attrs) == 1)
-        bytes_ = binUnpacker(smt_attrs.pop()[1])
-        count = bytes_.get('>H')
-        getVT = lambda: getVerificationType(bytes_, poolm, getLbl)
-
-        for _ in range(count):
-            tag = bytes_.get('>B')
-            header, contents = None, []
-
-            if 0 <= tag <= 63:
-                offset += tag
-                header = 'same'
-            elif 64 <= tag <= 127:
-                offset += tag - 64
-                header = 'same_locals_1_stack_item'
-                contents.append('\tstack ' + getVT())
-            elif tag == 247:
-                offset += bytes_.get('>H')
-                header = 'same_locals_1_stack_item_extended'
-                contents.append('\tstack ' + getVT())
-            elif 248 <= tag <= 250:
-                offset += bytes_.get('>H')
-                header = 'chop ' + str(251-tag)
-            elif tag == 251:
-                offset += bytes_.get('>H')
-                header = 'same_extended'
-            elif 252 <= tag <= 254:
-                offset += bytes_.get('>H')
-                header = 'append'
-                contents.append('\tlocals ' + ' '.join(getVT() for _ in range(tag-251)))
-            elif tag == 255:
-                offset += bytes_.get('>H')
-                header = 'full'
-                local_count = bytes_.get('>H')
-                contents.append('\tlocals ' + ' '.join(getVT() for _ in range(local_count)))
-                stack_count = bytes_.get('>H')
-                contents.append('\tstack ' + ' '.join(getVT() for _ in range(stack_count)))
-
-            if contents:
-                contents.append('.end stack')
-            contents = ['.stack ' + header] + contents
-            frame = '\n'.join(contents)
-            frames[offset] = frame
-            offset += 1 #frames after the first have an offset one larger than the listed offset
-    return frames
-
-def disCFMAttribute(name_ind, name, bytes_, add, poolm):
-    for vis in ('Visible', 'Invisible'):
-        if name == 'Runtime{}Annotations'.format(vis):
-            count = bytes_.get('>H')
-            for _ in range(count):
-                disAnnotation(bytes_, '.runtime{} '.format(vis.lower()), add, poolm, '')
-            if count: #otherwise we'll create an empty generic attribute
-                return
-
-    if name == "Signature":
-        add('.signature {}'.format(poolm.utfref(bytes_.get('>H'))))
-        return
-    #Create generic attribute if it can't be represented by a standard directive
-    add('.attribute {} {!r}'.format(poolm.utfref(name_ind), bytes_.getRaw(bytes_.size())))
-
-def disMethodAttribute(name_ind, name, bytes_, add, poolm):
-    if name == 'Code':
-        return
-    elif name == 'AnnotationDefault':
-        disElementValue(bytes_, '.annotationdefault ', add, poolm, '')
-        return
-    elif name == 'Exceptions':
-        count = bytes_.get('>H')
-        for _ in range(count):
-            add('.throws ' + poolm.classref(bytes_.get('>H')))
-        if count: #otherwise we'll create an empty generic attribute
-            return
-
-    for vis in ('Visible', 'Invisible'):
-        if name == 'Runtime{}ParameterAnnotations'.format(vis):
-            for i in range(bytes_.get('>B')):
-                for _ in range(bytes_.get('>H')):
-                    disAnnotation(bytes_, '.runtime{} parameter {} '.format(vis.lower(), i), add, poolm, '')
-            return #generic fallback on empty list not yet supported
-
-    disCFMAttribute(name_ind, name, bytes_, add, poolm)
-
-def disMethod(method, add, poolm):
-    mflags = ' '.join(map(str.lower, method.flags))
-    add('.method {} {} : {}'.format(mflags, poolm.utfref(method.name_id), poolm.utfref(method.desc_id)))
-
-    for name_ind, name, attr in getAttributeTriples(method):
-        disMethodAttribute(name_ind, name, binUnpacker(attr), add, poolm)
-
-    disMethodCode(method.code, add, poolm)
-    add('.end method')
-
-def _disEVorAnnotationSub(bytes_, add, poolm, isAnnot, init_prefix, init_indent):
-    C_ANNOT, C_ANNOT2, C_ANNOT3, C_EV, C_EV2 = range(5)
-    init_callt = C_ANNOT if isAnnot else C_EV
-
-    stack = [(init_callt, init_prefix, init_indent)]
-    while stack:
-        callt, prefix, indent = stack.pop()
-
-        if callt == C_ANNOT:
-            add(indent + prefix + 'annotation ' + poolm.utfref(bytes_.get('>H')))
-            #ones we want to happen last should be first on the stack. Annot3 is the final call which ends the annotation
-            stack.append((C_ANNOT3, None, indent))
-            stack.extend([(C_ANNOT2, None, indent)] * bytes_.get('>H'))
-
-        elif callt == C_ANNOT2:
-            key = poolm.utfref(bytes_.get('>H'))
-            stack.append((C_EV, key + ' = ', indent+'\t'))
-
-        elif callt == C_ANNOT3:
-            add(indent + '.end annotation')
-
-        elif callt == C_EV:
-            tag = codes.et_rtags[bytes_.getRaw(1)]
-            if tag == 'annotation':
-                stack.append((C_ANNOT, prefix, indent + '\t'))
-            else:
-                if tag in ('byte','char','double','int','float','long','short','boolean','string'):
-                    val = poolm.ldc(bytes_.get('>H'))
-                elif tag == 'class':
-                    val = poolm.utfref(bytes_.get('>H'))
-                elif tag == 'enum':
-                    val = poolm.utfref(bytes_.get('>H')) + ' ' + poolm.utfref(bytes_.get('>H'))
-                elif tag == 'array':
-                    val = ''
-
-                add(indent + '{} {} {}'.format(prefix, tag, val))
-                if tag == 'array':
-                    for _ in range(bytes_.get('>H')):
-                        stack.append((C_EV, '', indent+'\t'))
-                    stack.append((C_EV2, None, indent))
-
-        elif callt == C_EV2:
-            add(indent + '.end array')
-
-def disElementValue(bytes_, prefix, add, poolm, indent):
-    _disEVorAnnotationSub(bytes_, add, poolm, False, prefix, indent)
-
-def disAnnotation(bytes_, prefix, add, poolm, indent):
-    _disEVorAnnotationSub(bytes_, add, poolm, True, prefix, indent)
-
-#Todo - make fields automatically unpack this themselves
-def getConstValue(field):
-    if not field.static:
-        return None
-    const_attrs = [attr for attr in field.attributes if attr[0] == 'ConstantValue']
-    if const_attrs:
-        assert(len(const_attrs) == 1)
-        bytes_ = binUnpacker(const_attrs[0][1])
-        return bytes_.get('>H')
-
-_classflags = [(v,k.lower()) for k,v in ClassFile.flagVals.items()]
-def disInnerClassesAttribute(name_ind, length, bytes_, add, poolm):
-    count = bytes_.get('>H')
-
-    if length != 2+8*count:
-        add('.innerlength {}'.format(length))
-
-    for _ in range(count):
-        inner, outer, innername, flagbits = bytes_.get('>HHHH')
-
-        flags = [v for k,v in _classflags if k&flagbits]
-        inner = poolm.classref(inner)
-        outer = poolm.classref(outer)
-        innername = poolm.utfref(innername)
-
-        add('.inner {} {} {} {}'.format(' '.join(flags), innername, inner, outer))
-
-    if not count:
-        add('.attribute InnerClasses "\\0\\0"')
-
-def disOtherClassAttribute(name_ind, name, bytes_, add, poolm):
-    assert(name != 'InnerClasses')
-    if name == 'EnclosingMethod':
-        cls, nat = bytes_.get('>HH')
-        add('.enclosing method {} {}'.format(poolm.classref(cls), poolm.multiref(nat)))
-        return
-    disCFMAttribute(name_ind, name, bytes_, add, poolm)
-
-def disassemble(cls):
-    lines = []
-    add = lines.append
-    poolm = PoolManager(cls.cpool)
-
-    # def add(s): print s
-    add('.version {0[0]} {0[1]}'.format(cls.version))
-
-    class_attributes = getAttributesDict(cls)
-    if 'SourceFile' in class_attributes:
-        bytes_ = binUnpacker(class_attributes['SourceFile'].pop()[1])
-        val_ind = bytes_.get('>H')
-        add('.source {}'.format(poolm.utfref(val_ind)))
-
-    if 'BootstrapMethods' in class_attributes:
-        bytes_ = binUnpacker(class_attributes['BootstrapMethods'].pop()[1])
-        count = bytes_.get('>H')
-        for _ in range(count):
-            arg1, argc = bytes_.get('>HH')
-            args = (arg1,) + bytes_.get('>'+'H'*argc, forceTuple=True)
-            poolm.bootstrap_methods.append(args)
-
-    cflags = ' '.join(map(str.lower, cls.flags))
-    add('.class {} {}'.format(cflags, poolm.classref(cls.this)))
-    add('.super {}'.format(poolm.classref(cls.super)))
-    for ii in cls.interfaces_raw:
-        add('.implements {}'.format(poolm.classref(ii)))
-
-    for name in class_attributes:
-        if name == "InnerClasses":
-            assert(len(class_attributes[name]) == 1)
-            for name_ind, (length, attr) in class_attributes[name]:
-                disInnerClassesAttribute(name_ind, length, binUnpacker(attr), add, poolm)
-        else:
-            for name_ind, attr in class_attributes[name]:
-                disOtherClassAttribute(name_ind, name, binUnpacker(attr), add, poolm)
-
-    add('')
-    for field in cls.fields:
-        fflags = ' '.join(map(str.lower, field.flags))
-        const = getConstValue(field)
-
-        if const is not None:
-            add('.field {} {} {} = {}'.format(fflags, poolm.utfref(field.name_id), poolm.utfref(field.desc_id), poolm.ldc(const)))
-        else:
-            add('.field {} {} {}'.format(fflags, poolm.utfref(field.name_id), poolm.utfref(field.desc_id)))
-
-        facount = 0
-        for name_ind, name, attr in getAttributeTriples(field):
-            if name == 'ConstantValue' and field.static:
-                continue
-            disMethodAttribute(name_ind, name, binUnpacker(attr), add, poolm)
-            facount += 1
-        if facount > 0:
-            add('.end field')
-            add('')
-
-    add('')
-
-    for method in cls.methods:
-        disMethod(method, add, poolm)
-        add('')
-
-    poolm.printConstDefs(add)
-    return '\n'.join(lines)
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/instructions.py b/lib/Krakatau/Krakatau/assembler/instructions.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/instructions.py
+++ /dev/null
@@ -1,11 +0,0 @@
-instrs_noarg = ('nop', 'aconst_null', 'iconst_m1', 'iconst_0', 'iconst_1', 'iconst_2', 'iconst_3', 'iconst_4', 'iconst_5', 'lconst_0', 'lconst_1', 'fconst_0', 'fconst_1', 'fconst_2', 'dconst_0', 'dconst_1', 'iload_0', 'iload_1', 'iload_2', 'iload_3', 'lload_0', 'lload_1', 'lload_2', 'lload_3', 'fload_0', 'fload_1', 'fload_2', 'fload_3', 'dload_0', 'dload_1', 'dload_2', 'dload_3', 'aload_0', 'aload_1', 'aload_2', 'aload_3', 'iaload', 'laload', 'faload', 'daload', 'aaload', 'baload', 'caload', 'saload', 'istore_0', 'istore_1', 'istore_2', 'istore_3', 'lstore_0', 'lstore_1', 'lstore_2', 'lstore_3', 'fstore_0', 'fstore_1', 'fstore_2', 'fstore_3', 'dstore_0', 'dstore_1', 'dstore_2', 'dstore_3', 'astore_0', 'astore_1', 'astore_2', 'astore_3', 'iastore', 'lastore', 'fastore', 'dastore', 'aastore', 'bastore', 'castore', 'sastore', 'pop', 'pop2', 'dup', 'dup_x1', 'dup_x2', 'dup2', 'dup2_x1', 'dup2_x2', 'swap', 'iadd', 'ladd', 'fadd', 'dadd', 'isub', 'lsub', 'fsub', 'dsub', 'imul', 'lmul', 'fmul', 'dmul', 'idiv', 'ldiv', 'fdiv', 'ddiv', 'irem', 'lrem', 'frem', 'drem', 'ineg', 'lneg', 'fneg', 'dneg', 'ishl', 'lshl', 'ishr', 'lshr', 'iushr', 'lushr', 'iand', 'land', 'ior', 'lor', 'ixor', 'lxor', 'i2l', 'i2f', 'i2d', 'l2i', 'l2f', 'l2d', 'f2i', 'f2l', 'f2d', 'd2i', 'd2l', 'd2f', 'i2b', 'i2c', 'i2s', 'lcmp', 'fcmpl', 'fcmpg', 'dcmpl', 'dcmpg', 'ireturn', 'lreturn', 'freturn', 'dreturn', 'areturn', 'return', 'arraylength', 'athrow', 'monitorenter', 'monitorexit')
-
-instrs_int = ('bipush', 'sipush', 'iload', 'lload', 'fload', 'dload', 'aload', 'istore', 'lstore', 'fstore', 'dstore', 'astore', 'ret')
-
-instrs_lbl = ('ifeq', 'ifne', 'iflt', 'ifge', 'ifgt', 'ifle', 'if_icmpeq', 'if_icmpne', 'if_icmplt', 'if_icmpge', 'if_icmpgt', 'if_icmple', 'if_acmpeq', 'if_acmpne', 'goto', 'jsr', 'ifnull', 'ifnonnull', 'goto_w', 'jsr_w')
-
-instrs_cp = ('ldc', 'ldc_w', 'ldc2_w', 'getstatic', 'putstatic', 'getfield', 'putfield', 'invokevirtual', 'invokespecial', 'invokestatic', 'invokedynamic', 'new', 'anewarray', 'checkcast', 'instanceof')
-
-instrs_other = ('iinc', 'tableswitch', 'lookupswitch', 'invokeinterface', 'newarray', 'wide', 'multianewarray')
-
-allinstructions = ('nop', 'aconst_null', 'iconst_m1', 'iconst_0', 'iconst_1', 'iconst_2', 'iconst_3', 'iconst_4', 'iconst_5', 'lconst_0', 'lconst_1', 'fconst_0', 'fconst_1', 'fconst_2', 'dconst_0', 'dconst_1', 'bipush', 'sipush', 'ldc', 'ldc_w', 'ldc2_w', 'iload', 'lload', 'fload', 'dload', 'aload', 'iload_0', 'iload_1', 'iload_2', 'iload_3', 'lload_0', 'lload_1', 'lload_2', 'lload_3', 'fload_0', 'fload_1', 'fload_2', 'fload_3', 'dload_0', 'dload_1', 'dload_2', 'dload_3', 'aload_0', 'aload_1', 'aload_2', 'aload_3', 'iaload', 'laload', 'faload', 'daload', 'aaload', 'baload', 'caload', 'saload', 'istore', 'lstore', 'fstore', 'dstore', 'astore', 'istore_0', 'istore_1', 'istore_2', 'istore_3', 'lstore_0', 'lstore_1', 'lstore_2', 'lstore_3', 'fstore_0', 'fstore_1', 'fstore_2', 'fstore_3', 'dstore_0', 'dstore_1', 'dstore_2', 'dstore_3', 'astore_0', 'astore_1', 'astore_2', 'astore_3', 'iastore', 'lastore', 'fastore', 'dastore', 'aastore', 'bastore', 'castore', 'sastore', 'pop', 'pop2', 'dup', 'dup_x1', 'dup_x2', 'dup2', 'dup2_x1', 'dup2_x2', 'swap', 'iadd', 'ladd', 'fadd', 'dadd', 'isub', 'lsub', 'fsub', 'dsub', 'imul', 'lmul', 'fmul', 'dmul', 'idiv', 'ldiv', 'fdiv', 'ddiv', 'irem', 'lrem', 'frem', 'drem', 'ineg', 'lneg', 'fneg', 'dneg', 'ishl', 'lshl', 'ishr', 'lshr', 'iushr', 'lushr', 'iand', 'land', 'ior', 'lor', 'ixor', 'lxor', 'iinc', 'i2l', 'i2f', 'i2d', 'l2i', 'l2f', 'l2d', 'f2i', 'f2l', 'f2d', 'd2i', 'd2l', 'd2f', 'i2b', 'i2c', 'i2s', 'lcmp', 'fcmpl', 'fcmpg', 'dcmpl', 'dcmpg', 'ifeq', 'ifne', 'iflt', 'ifge', 'ifgt', 'ifle', 'if_icmpeq', 'if_icmpne', 'if_icmplt', 'if_icmpge', 'if_icmpgt', 'if_icmple', 'if_acmpeq', 'if_acmpne', 'goto', 'jsr', 'ret', 'tableswitch', 'lookupswitch', 'ireturn', 'lreturn', 'freturn', 'dreturn', 'areturn', 'return', 'getstatic', 'putstatic', 'getfield', 'putfield', 'invokevirtual', 'invokespecial','invokestatic', 'invokeinterface', 'invokedynamic', 'new', 'newarray', 'anewarray', 'arraylength', 'athrow', 'checkcast', 'instanceof', 'monitorenter', 'monitorexit', 'wide', 'multianewarray', 'ifnull', 'ifnonnull', 'goto_w', 'jsr_w')
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/parse.py b/lib/Krakatau/Krakatau/assembler/parse.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/parse.py
+++ /dev/null
@@ -1,554 +0,0 @@
-import ast, struct
-import itertools
-
-from ..classfile import ClassFile
-from ..method import Method
-from ..field import Field
-
-#Important to import tokens here even though it appears unused, as ply uses it
-from .tokenize import tokens, wordget, flags
-from .assembler import PoolRef
-
-#Specify the starting symbol
-start = 'top'
-
-###############################################################################
-name_counter = itertools.count()
-def addRule(func, name, *rhs_rules):
-    def _inner(p):
-        func(p)
-    _inner.__doc__ = name + ' : ' + '\n| '.join(rhs_rules)
-    fname = 'p_{}'.format(next(name_counter))
-    globals()[fname] = _inner
-
-def list_sub(p):p[0] = p[1] + p[2:]
-def listRule(name): #returns a list
-    name2 = name + 's'
-    addRule(list_sub, name2, '{} {}'.format(name2, name), 'empty')    
-
-def nothing(p):pass
-def assign1(p):p[0] = p[1]
-def assign2(p):p[0] = p[2]
-def upper1(p): p[0] = p[1].upper()
-
-# Common Rules ################################################################
-addRule(nothing, 'sep', 'sep NEWLINE', 'NEWLINE')
-
-def p_empty(p):
-    'empty :'
-    p[0] = []
-
-def p_intl(p):
-    '''intl : INT_LITERAL'''
-    p[0] = ast.literal_eval(p[1])
-
-def p_longl(p):
-    '''longl : LONG_LITERAL'''
-    p[0] = ast.literal_eval(p[1][:-1])
-
-#Todo - find a better way of handling floats
-def parseFloat(s):
-    s = s[:-1]
-    if s.strip('-')[:2].lower() == '0x':
-        f = float.fromhex(s)
-    else:
-        f = float(s)
-    return struct.unpack('>i', struct.pack('>f', f))[0]
-
-def parseDouble(s):
-    if s.strip('-')[:2].lower() == '0x':
-        f = float.fromhex(s)
-    else:
-        f = float(s)
-    return struct.unpack('>q', struct.pack('>d', f))[0]
-
-def p_floatl(p):
-    '''floatl : FLOAT_LITERAL'''
-    p[0] = parseFloat(p[1])
-def p_doublel(p):
-    '''doublel : DOUBLE_LITERAL'''
-    p[0] = parseDouble(p[1])
-
-#We can allow keywords as inline classnames as long as they aren't flag names
-#which would be ambiguous. We don't allow directives to simplfy the grammar
-#rules, since they wouldn't be valid identifiers anyway.
-badwords = frozenset(map(str.lower, flags))
-badwords |= frozenset(k for k in wordget if '.' in k) 
-oktokens = frozenset(v for k,v in wordget.items() if k not in badwords)
-addRule(assign1, 'notflag', 'WORD', 'STRING_LITERAL', *oktokens)
-
-def p_ref(p):
-    '''ref : CPINDEX'''
-    s = p[1][1:-1]
-    try:
-        i = int(s)
-        if 0 <= i <= 0xFFFF:
-            p[0] = PoolRef(index=i)
-        else:
-            p[0] = PoolRef(lbl=s)    
-    except ValueError:
-        p[0] = PoolRef(lbl=s)
-
-def p_utf8_notref(p):
-    '''utf8_notref : notflag'''
-    p[0] = PoolRef('Utf8', p[1])
-
-def p_class_notref(p):
-    '''class_notref : utf8_notref'''
-    p[0] = PoolRef('Class', p[1])
-
-def p_string_notref(p):
-    '''string_notref : utf8_notref'''
-    p[0] = PoolRef('String', p[1])
-
-def p_nat_notref(p):
-    '''nameandtype_notref : utf8ref utf8ref'''
-    p[0] = PoolRef('NameAndType', p[1], p[2])
-
-def p_field_notref(p):
-    '''field_notref : classref nameandtyperef'''
-    p[0] = PoolRef('Field', p[1], p[2])
-
-def p_method_notref(p):
-    '''method_notref : classref nameandtyperef'''
-    p[0] = PoolRef('Method', p[1], p[2])
-
-def p_imethod_notref(p):
-    '''interfacemethod_notref : classref nameandtyperef'''
-    p[0] = PoolRef('InterfaceMethod', p[1], p[2])
-
-#constant pool types related to InvokeDynamic handled later
-
-for _name in ('utf8','class', 'nameandtype', 'method', 'interfacemethod', 'methodhandle'):
-    addRule(assign1, '{}ref'.format(_name), '{}_notref'.format(_name), 'ref')
-
-###############################################################################
-def p_classnoend(p):
-    '''classnoend : version_opt class_directive_lines classdec superdec interfacedecs class_directive_lines topitems'''
-    p[0] = tuple(p[1:])
-
-addRule(assign1, 'classwithend', 'classnoend D_END CLASS sep')
-listRule('classwithend')
-
-def p_top(p):
-    '''top : sep classwithends classnoend'''
-    p[0] = p[2] + [p[3]]
-#case where all classes have an end
-addRule(assign2, 'top', 'sep classwithends')
-
-def p_version(p):
-    '''version_opt : D_VERSION intl intl sep'''
-    p[0] = p[2], p[3]
-addRule(assign1, 'version_opt', 'empty')
-
-###############################################################################
-for c, type_ in zip('cmf', (ClassFile, Method, Field)):
-    _name = "{}flag".format(c)
-    addRule(upper1, _name, *list(type_.flagVals))
-    listRule(_name)
-
-def p_classdec(p):
-    '''classdec : D_CLASS cflags classref sep 
-                | D_INTERFACE cflags classref sep'''
-    #if interface, add interface to flags
-    p[0] = (p[1] == '.interface'), p[2], p[3]
-
-addRule(assign2, 'superdec', 'D_SUPER classref sep')
-addRule(assign2, 'interfacedec', 'D_IMPLEMENTS classref sep')
-listRule('interfacedec')
-
-addRule(assign1, 'class_directive', 'classattribute', 'innerlength_dir')
-addRule(assign1, 'class_directive_line', 'class_directive sep')
-listRule('class_directive_line')
-
-def p_topitem_c(p):
-    '''topitem : const_spec'''
-    p[0] = 'const', p[1]
-def p_topitem_f(p):
-    '''topitem : field_spec'''
-    p[0] = 'field', p[1]
-def p_topitem_m(p):
-    '''topitem : method_spec'''
-    p[0] = 'method', p[1]
-listRule('topitem')
-
-###############################################################################
-#invoke dynamic stuff
-from .codes import handle_codes
-_handle_token_types = set(wordget.get(x, 'WORD') for x in handle_codes)
-def p_handle(p):
-    p[0] = handle_codes[p[1]]
-p_handle.__doc__ = "handlecode : " + '\n| '.join(_handle_token_types)
-
-#The second argument's type depends on the code, so we require an explicit reference for simplicity
-def p_methodhandle_notref(p):
-    '''methodhandle_notref : handlecode ref'''
-    p[0] = PoolRef('MethodHandle', p[1], p[2])
-
-def p_methodtype_notref(p):
-    '''methodtype_notref : utf8_notref'''
-    p[0] = PoolRef('Methodtype', p[1])
-
-addRule(assign1, 'bootstrap_arg', 'ref') #TODO - allow inline constants and strings?
-listRule('bootstrap_arg')
-
-def p_invokedynamic_notref(p):
-    '''invokedynamic_notref : methodhandleref bootstrap_args COLON nameandtyperef'''
-    args = [p[1]] + p[2] + [p[4]]
-    p[0] = PoolRef('InvokeDynamic', *args)
-
-###############################################################################
-def p_const_spec(p):
-    '''const_spec : D_CONST ref EQUALS const_rhs sep'''
-    p[0] = p[2], p[4]
-
-def assignPoolSingle(typen):
-    def inner(p):
-        p[0] = PoolRef(typen, p[2])
-    return inner
-
-addRule(assign1, 'const_rhs', 'ref')
-for tt in ['UTF8', 'CLASS','STRING','NAMEANDTYPE','FIELD','METHOD','INTERFACEMETHOD',
-            'METHODHANDLE','METHODTYPE','INVOKEDYNAMIC']:
-    addRule(assign2, 'const_rhs', '{} {}_notref'.format(tt, tt.lower()))
-
-#these are special cases, since they take a single argument
-#and the notref version can't have a ref as its argument due to ambiguity
-for ptype in ('Class','String','MethodType'):
-    addRule(assignPoolSingle(ptype), 'const_rhs', ptype.upper() + ' ref')
-
-for ptype in ('Int','Float','Long','Double'):
-    addRule(assignPoolSingle(ptype), 'const_rhs', '{} {}l'.format(ptype.upper(), ptype.lower()))
-###############################################################################
-
-
-def p_field_spec(p):
-    '''field_spec : D_FIELD fflags utf8ref utf8ref field_constval fieldattribute_list'''
-    p[0] = p[2:7]
-
-addRule(nothing, 'field_constval', 'empty')
-addRule(assign2, 'field_constval', 'EQUALS ref', 
-                                    'EQUALS ldc1_notref', 
-                                    'EQUALS ldc2_notref')
-
-#Sadly, we must only allow .end field when at least one attribute is specified
-#in order to avoid grammatical ambiguity. JasminXT does not share this problem
-#because it lacks the .end class syntax which causes the conflict
-def p_field_attrlist1(p):
-    '''field_al_nonempty : fieldattribute sep field_al_nonempty'''
-    p[0] = [p[1]]+ p[3]
-def p_field_attrlist2(p):
-    '''field_al_nonempty : fieldattribute sep D_END FIELD sep'''
-    p[0] = [p[1]]
-
-addRule(assign2, 'fieldattribute_list', 'sep field_al_nonempty', 'sep empty')
-
-
-def p_method_spec(p):
-    '''method_spec : defmethod statements endmethod'''
-    p[0] = p[1],p[2]
-
-def p_defmethod_0(p):
-    '''defmethod : D_METHOD mflags jas_meth_namedesc sep'''
-    p[0] = p[2],p[3] 
-def p_defmethod_1(p):
-    '''defmethod : D_METHOD mflags utf8ref COLON utf8ref sep'''
-    p[0] = p[2],(p[3], p[5]) 
-
-def p_jas_meth_namedesc(p):
-    '''jas_meth_namedesc : WORD'''
-    name, paren, desc = p[1].rpartition('(')
-    name = PoolRef('Utf8', name)
-    desc = PoolRef('Utf8', paren+desc)
-    p[0] = name, desc
-addRule(nothing, 'endmethod', 'D_END METHOD sep')
-
-def p_statement_0(p):
-    '''statement : method_directive sep'''
-    p[0] = False, p[1]
-def p_statement_1(p):
-    '''statement : code_directive sep'''
-    p[0] = True, (False, p[1])
-def p_statement_2(p):
-    '''statement : empty instruction sep 
-                | lbldec instruction sep
-                | lbldec sep'''
-    p[0] = True, (True, ((p[1] or None), p[2]))
-listRule('statement')
-
-addRule(assign1, 'lbldec', 'lbl COLON')
-addRule(assign1, 'method_directive', 'methodattribute')
-addRule(assign1, 'code_directive', 'limit_dir', 'except_dir','localvar_dir','linenumber_dir','stack_dir', 'generic_codeattribute_dir')
-
-def p_limit_dir(p):
-    '''limit_dir : D_LIMIT LOCALS intl 
-                | D_LIMIT STACK intl'''
-    p[0] = p[1], (p[2], p[3])
-
-def p_except_dir(p):
-    '''except_dir : D_CATCH classref FROM lbl TO lbl USING lbl'''
-    p[0] = p[1], (p[2], p[4], p[6], p[8])
-
-def p_linenumber_dir(p):
-    '''linenumber_dir : D_LINE intl'''
-    p[0] = p[1], p[2]
-
-def p_localvar_dir(p):
-    '''localvar_dir : D_VAR intl IS utf8ref utf8ref FROM lbl TO lbl'''
-    p[0] = p[1], (p[2], p[4], p[5], p[7], p[9])
-
-def p_instruction(p):
-    '''instruction : OP_NONE
-                    | OP_INT intl
-                    | OP_INT_INT intl intl
-                    | OP_LBL lbl
-                    | OP_FIELD fieldref_or_jas
-                    | OP_METHOD methodref_or_jas
-                    | OP_METHOD_INT imethodref_or_jas intl
-                    | OP_DYNAMIC ref
-                    | OP_CLASS classref
-                    | OP_CLASS_INT classref intl
-                    | OP_LDC1 ldc1_ref
-                    | OP_LDC2 ldc2_ref
-                    | OP_NEWARR nacode
-                    | OP_LOOKUPSWITCH luswitch
-                    | OP_TABLESWITCH tblswitch
-                    | OP_WIDE wide_instr
-                    '''
-    if p[1] == 'invokenonvirtual':
-        p[1] = 'invokespecial'
-    p[0] = tuple(p[1:])
-    #these instructions have 0 padding at the end
-    #this is kind of an ungly hack, but the best way I could think of
-    if p[1] in ('invokeinterface','invokedynamic'):
-        p[0] += (0,)
-
-addRule(assign1, 'lbl', 'WORD')
-addRule(assign1, 'fieldref_or_jas', 'jas_fieldref', 'ref', 'inline_fieldref')
-def p_jas_fieldref(p):
-    '''jas_fieldref : WORD WORD'''
-    class_, sep, name = p[1].replace('.','/').rpartition('/')
-
-    desc = PoolRef('Utf8', p[2])
-    class_ = PoolRef('Class', PoolRef('Utf8', class_))
-    name = PoolRef('Utf8', name)
-    nt = PoolRef('NameAndType', name, desc)
-    p[0] = PoolRef('Field', class_, nt)
-
-#This is an ugly hack to work around the fact that Jasmin syntax would otherwise be impossible to 
-#handle with a LALR(1) parser
-def p_inline_fieldref_1(p):
-    '''inline_fieldref : WORD nameandtyperef
-                        | STRING_LITERAL nameandtyperef'''
-    class_ = PoolRef('Class', PoolRef('Utf8', p[1]))
-    p[0] = PoolRef('Field', class_, p[2])
-def p_inline_fieldref_2(p):
-    '''inline_fieldref : ref nameandtyperef'''
-    p[0] = PoolRef('Field', p[1], p[2])
-
-
-def p_jas_meth_classnamedesc(p):
-    '''jas_methodref : WORD'''
-    name, paren, desc = p[1].rpartition('(')
-    class_, sep, name = name.replace('.','/').rpartition('/')
-    desc = paren + desc
-
-    class_ = PoolRef('Class', PoolRef('Utf8', class_))
-    nt = PoolRef('NameAndType', PoolRef('Utf8', name), PoolRef('Utf8', desc))
-    p[0] = class_, nt
-
-addRule(assign1, 'methodref_or_jas', 'methodref')
-def p_methodref_or_jas(p):
-    '''methodref_or_jas : jas_methodref'''
-    p[0] = PoolRef('Method', *p[1])
-
-addRule(assign1, 'imethodref_or_jas', 'interfacemethodref')
-def p_imethodref_or_jas(p):
-    '''imethodref_or_jas : jas_methodref'''
-    p[0] = PoolRef('InterfaceMethod', *p[1])
-
-
-from .codes import newarr_codes
-_newarr_token_types = set(wordget.get(x, 'WORD') for x in newarr_codes)
-def p_nacode(p):
-    p[0] = newarr_codes[p[1]]
-p_nacode.__doc__ = "nacode : " + '\n| '.join(_newarr_token_types)
-
-addRule(assign1, 'ldc1_ref', 'ldc1_notref', 'ref')
-def p_ldc1_notref_string(p):
-    '''ldc1_notref : STRING_LITERAL'''
-    p[0] = PoolRef('String', PoolRef('Utf8', p[1]))
-def p_ldc1_notref_int(p):
-    '''ldc1_notref : intl'''
-    p[0] = PoolRef('Int', p[1])
-def p_ldc1_notref_float(p):
-    '''ldc1_notref : floatl'''
-    p[0] = PoolRef('Float', p[1])
-
-addRule(assign1, 'ldc2_ref', 'ldc2_notref', 'ref')
-def p_ldc2_notref_long(p):
-    '''ldc2_notref : longl'''
-    p[0] = PoolRef('Long', p[1])
-def p_ldc2_notref_double(p):
-    '''ldc2_notref : doublel'''
-    p[0] = PoolRef('Double', p[1])
-
-def p_defaultentry(p):
-    '''defaultentry : DEFAULT COLON lbl'''
-    p[0] = p[3]
-
-def p_luentry(p):
-    '''luentry : intl COLON lbl sep'''
-    p[0] = p[1], p[3]
-listRule('luentry')
-
-addRule(assign1, 'tblentry', 'lbl sep')
-listRule('tblentry')
-
-def p_lookupswitch(p):
-    '''luswitch : empty sep luentrys defaultentry'''
-    p[0] = p[1], p[3], p[4]
-
-def p_tableswitch(p):
-    '''tblswitch : intl sep tblentrys defaultentry'''
-    p[0] = p[1], p[3], p[4]
-
-def p_wide_instr(p):
-    '''wide_instr : OP_INT intl
-                | OP_INT_INT intl intl'''
-    p[0] = p[1], tuple(p[2:])
-
-#######################################################################
-# Explicit Attributes
-addRule(assign1, 'cfmattribute', 'annotation_dir', 'signature_dir', 'generic_attribute_dir')
-addRule(assign1, 'classattribute', 'cfmattribute', 'sourcefile_dir', 'inner_dir', 'enclosing_dir')
-addRule(assign1, 'fieldattribute', 'cfmattribute')
-addRule(assign1, 'methodattribute', 'cfmattribute', 'throws_dir', 'annotation_param_dir', 'annotation_def_dir')
-
-#Class, field, method
-def p_annotation_dir(p):
-    '''annotation_dir : D_RUNTIMEVISIBLE annotation
-                    | D_RUNTIMEINVISIBLE annotation'''
-    p[0] = p[1], (None, p[2])
-
-def p_signature_dir(p):
-    '''signature_dir : D_SIGNATURE utf8ref'''
-    p[0] = p[1], p[2]
-
-#Class only
-def p_sourcefile_dir(p):
-    '''sourcefile_dir : D_SOURCE utf8ref'''
-    p[0] = p[1], p[2]
-
-def p_inner_dir(p): 
-    '''inner_dir : D_INNER cflags utf8ref classref classref'''
-    p[0] = p[1], (p[4],p[5],p[3],p[2]) #use JasminXT's (flags, name, inner, outer) order but switch internally to correct order
-
-def p_enclosing_dir(p): 
-    '''enclosing_dir : D_ENCLOSING METHOD classref nameandtyperef'''
-    p[0] = p[1], (p[3], p[4])
-
-#This is included here even though strictly speaking, it's not an attribute. Rather it's a directive that affects the assembly
-#of the InnerClasses attribute
-def p_innerlength_dir(p): 
-    '''innerlength_dir : D_INNERLENGTH intl'''
-    p[0] = p[1], p[2]
-
-
-#Method only
-def p_throws_dir(p):
-    '''throws_dir : D_THROWS classref'''
-    p[0] = p[1], p[2]
-
-def p_annotation_param_dir(p):
-    '''annotation_param_dir : D_RUNTIMEVISIBLE PARAMETER intl annotation
-                           | D_RUNTIMEINVISIBLE PARAMETER intl annotation'''
-    p[0] = p[1], (p[3], p[4])
-def p_annotation_def_dir(p):
-    '''annotation_def_dir : D_ANNOTATIONDEFAULT element_value'''
-    p[0] = p[1], p[2]
-
-#Generic
-def p_generic_attribute_dir(p): 
-    '''generic_attribute_dir : D_ATTRIBUTE utf8ref STRING_LITERAL'''
-    p[0] = p[1], (p[2], p[3])
-
-def p_generic_codeattribute_dir(p): 
-    '''generic_codeattribute_dir : D_CODEATTRIBUTE utf8ref STRING_LITERAL'''
-    p[0] = p[1], (p[2], p[3])
-
-#######################################################################
-#Stack map stuff
-addRule(nothing, 'endstack', 'D_END STACK') #directives are not expected to end with a sep
-
-def assign1All(p):p[0] = tuple(p[1:])
-addRule(assign1All, 'verification_type', 'TOP', 'INTEGER', 'FLOAT', 'DOUBLE', 'LONG', 'NULL', 'UNINITIALIZEDTHIS',
-                                        'OBJECT classref', 'UNINITIALIZED lbl')
-listRule('verification_type')
-addRule(assign2, 'locals_vtlist', 'LOCALS verification_types sep')
-addRule(assign2, 'stack_vtlist', 'STACK verification_types sep')
-
-def p_stack_dir(p):
-    '''stack_dir_rest : SAME 
-                    | SAME_EXTENDED
-                    | CHOP intl 
-                    | SAME_LOCALS_1_STACK_ITEM sep stack_vtlist endstack
-                    | SAME_LOCALS_1_STACK_ITEM_EXTENDED sep stack_vtlist endstack
-                    | APPEND sep locals_vtlist endstack
-                    | FULL sep locals_vtlist stack_vtlist endstack'''
-    p[0] = '.stackmap', tuple(p[1:])
-addRule(assign2, 'stack_dir', 'D_STACK stack_dir_rest')
-#######################################################################
-#Annotation stuff
-from .codes import et_tags
-primtags = set(wordget.get(x, 'WORD') for x in 'byte char double int float long short boolean string'.split())
-addRule(assign1, 'primtag', *primtags)
-addRule(assign1, 'ldc_any', 'ldc1_notref', 'ldc2_notref', 'ref')
-
-def p_element_value_0(p):
-    '''element_value : primtag ldc_any
-                    | CLASS utf8ref
-                    | ENUM utf8ref utf8ref
-                    | ARRAY sep element_array'''
-    p[0] = et_tags[p[1]], tuple(p[2:])
-def p_element_value_1(p):
-    '''element_value : annotation'''
-    p[0] = '@', (p[1],)
-
-addRule(assign1, 'element_value_line', 'element_value sep')
-listRule('element_value_line')
-addRule(assign1, 'element_array', 'element_value_lines D_END ARRAY')
-
-def p_key_ev_line(p):
-    '''key_ev_line : utf8ref EQUALS element_value_line'''
-    p[0] = p[1], p[3]
-listRule('key_ev_line')
-
-def p_annotation(p):
-    '''annotation : ANNOTATION utf8ref sep key_ev_lines D_END ANNOTATION'''
-    p[0] = p[2], p[4]
-#######################################################################
-
-def p_error(p):
-    if p is None:
-        print "Syntax error: unexpected EOF"
-    else: #remember to subtract 1 from line number since we had a newline at the start of the file
-        print "Syntax error at line {}: unexpected token {!r}".format(p.lineno-1, p.value)
-    
-    #Ugly hack since Ply doesn't provide any useful error information
-    import inspect
-    frame = inspect.currentframe()
-    cvars = frame.f_back.f_locals
-    print 'Expected:', ', '.join(cvars['actions'][cvars['state']].keys())
-    print 'Found:', cvars['ltype']
-    print 'Current stack:', cvars['symstack']
-
-    #Discard the rest of the input so that Ply doesn't attempt error recovery
-    from ply import yacc
-    tok = yacc.token()
-    while tok is not None:
-        tok = yacc.token()
-
-def makeParser(**kwargs):
-    from ply import yacc
-    return yacc.yacc(**kwargs)
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/assembler/tokenize.py b/lib/Krakatau/Krakatau/assembler/tokenize.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/assembler/tokenize.py
+++ /dev/null
@@ -1,140 +0,0 @@
-import ast
-
-from ..classfile import ClassFile
-from ..method import Method
-from ..field import Field
-from .. import constant_pool
-from . import instructions as ins
-from . import codes
-
-directives = 'CLASS','INTERFACE','SUPER','IMPLEMENTS','CONST','FIELD','METHOD','END','LIMIT','CATCH','SOURCE','LINE','VAR','THROWS',
-directives += 'VERSION', 'STACK', 'RUNTIMEVISIBLE', 'RUNTIMEINVISIBLE', 'ANNOTATIONDEFAULT', 'INNER', 'ENCLOSING', 'SIGNATURE', 
-directives += 'ATTRIBUTE', 'CODEATTRIBUTE', 'INNERLENGTH'
-keywords = ['CLASS','METHOD','FIELD','LOCALS','STACK','FROM','TO','USING','DEFAULT','IS']
-keywords += ['SAME','SAME_LOCALS_1_STACK_ITEM','SAME_LOCALS_1_STACK_ITEM_EXTENDED','CHOP','SAME_EXTENDED','APPEND','FULL']
-keywords += ['ANNOTATION','ARRAY','PARAMETER']
-flags = ClassFile.flagVals.keys() + Method.flagVals.keys() + Field.flagVals.keys()
-
-lowwords = set().union(keywords, flags)
-casewords = set().union(codes.vt_keywords, constant_pool.name2Type.keys())
-
-wordget = {}
-wordget.update({w.lower():w.upper() for w in lowwords})
-wordget.update({w:w.upper() for w in casewords})
-wordget.update({'.'+w.lower():'D_'+w for w in directives})
-
-assert(set(wordget).isdisjoint(ins.allinstructions))
-for op in ins.instrs_noarg:
-    wordget[op] = 'OP_NONE'
-for op in ins.instrs_int:
-    wordget[op] = 'OP_INT'
-for op in ins.instrs_lbl:
-    wordget[op] = 'OP_LBL'
-for op in ('getstatic', 'putstatic', 'getfield', 'putfield'):
-    wordget[op] = 'OP_FIELD'
-#support invokenonvirtual for backwards compatibility with Jasmin
-for op in ('invokevirtual', 'invokespecial', 'invokestatic', 'invokenonvirtual'): 
-    wordget[op] = 'OP_METHOD'
-for op in ('new', 'anewarray', 'checkcast', 'instanceof'):
-    wordget[op] = 'OP_CLASS'
-for op in ('wide','lookupswitch','tableswitch'):
-    wordget[op] = 'OP_' + op.upper()
-
-wordget['ldc'] = 'OP_LDC1'
-wordget['ldc_w'] = 'OP_LDC1'
-wordget['ldc2_w'] = 'OP_LDC2'
-wordget['iinc'] = 'OP_INT_INT'
-wordget['newarray'] = 'OP_NEWARR'
-wordget['multianewarray'] = 'OP_CLASS_INT'
-wordget['invokeinterface'] = 'OP_METHOD_INT'
-wordget['invokedynamic'] = 'OP_DYNAMIC'
-
-for op in ins.allinstructions:
-    wordget.setdefault(op,op.upper())
-
-#special PLY value
-tokens = ('NEWLINE', 'COLON', 'EQUALS', 'WORD', 'CPINDEX', 
-    'STRING_LITERAL', 'INT_LITERAL', 'LONG_LITERAL', 'FLOAT_LITERAL', 'DOUBLE_LITERAL') + tuple(set(wordget.values()))
-
-def t_ignore_COMMENT(t):
-    r';.*'
-
-# Define a rule so we can track line numbers
-def t_NEWLINE(t):
-    r'\n+'
-    t.lexer.lineno += len(t.value)
-    return t
-
-def t_STRING_LITERAL(t):
-    # See http://stackoverflow.com/questions/430759/regex-for-managing-escaped-characters-for-items-like-string-literals/5455705#5455705
-    r'''[uUbB]?[rR]?(?:
-        """[^"\\]*              # any number of unescaped characters
-            (?:\\.[^"\\]*       # escaped followed by 0 or more unescaped
-                |"[^"\\]+       # single quote followed by at least one unescaped
-                |""[^"\\]+      # two quotes followed by at least one unescaped
-            )*"""
-        |"[^"\n\\]*              # any number of unescaped characters
-            (?:\\.[^"\n\\]*      # escaped followed by 0 or more unescaped
-            )*"
-    '''r"""                     # concatenated string literals
-        |'''[^'\\]*              # any number of unescaped characters
-            (?:\\.[^'\\]*       # escaped followed by 0 or more unescaped
-                |'[^'\\]+       # single quote followed by at least one unescaped
-                |''[^'\\]+      # two quotes followed by at least one unescaped
-            )*'''
-        |'[^'\n\\]*              # any number of unescaped characters
-            (?:\\.[^'\n\\]*      # escaped followed by 0 or more unescaped
-            )*'
-        )"""
-
-    t.value = ast.literal_eval(t.value)
-    return t
-
-#careful here: | is not greedy so hex must come first
-int_base = r'[+-]?(?:0[xX][0-9a-fA-F]+|[0-9]+)'
-float_base = r'''(?:
-    [Nn][Aa][Nn]|                                       #Nan
-    [-+]?(?:                                            #Inf and normal both use sign
-        [Ii][Nn][Ff]|                                   #Inf
-        \d+\.\d*(?:[eE][+-]?\d+)?|                         #decimal float
-        \d+[eE][+-]?\d+|                                   #decimal float with no fraction (exponent mandatory)
-        0[xX][0-9a-fA-F]*\.[0-9a-fA-F]+[pP][+-]?\d+        #hexidecimal float
-        )
-    )
-'''
-
-#Sadly there's no nice way to define these even with reflection hacks
-#Hopefully we can get Ply patched some day or fork it or something so 
-#it's not so much of a pain
-
-#These are matched in order of appearence (specifically, f.func_code.co_firstlineno)
-#So anything that can be a prefix of another must go last
-def t_FLOAT_LITERAL(t): return t
-t_FLOAT_LITERAL.__doc__ = float_base + r'[fF]'
-def t_DOUBLE_LITERAL(t): return t
-t_DOUBLE_LITERAL.__doc__ = float_base
-def t_LONG_LITERAL(t): return t
-t_LONG_LITERAL.__doc__ = int_base + r'[lL]'
-def t_INT_LITERAL(t): return t
-t_INT_LITERAL.__doc__ = int_base
-
-def t_CPINDEX(t): return t
-t_CPINDEX.__doc__ = r'\[[0-9a-z_]+\]'
-
-
-def t_WORD(t):
-    r'''[^\s:="']+'''
-    t.type = wordget.get(t.value, 'WORD')
-    return t
-
-t_COLON = r':'
-t_EQUALS = r'='
-t_ignore = ' \t\r'
-
-def t_error(t):
-    print 'Parser error on line {} at {}'.format(t.lexer.lineno, t.lexer.lexpos)
-    print t.value[:79]
-
-def makeLexer(**kwargs):
-    from ply import lex
-    return lex.lex(**kwargs)
\ No newline at end of file
diff --git a/lib/Krakatau/Krakatau/attributes_raw.py b/lib/Krakatau/Krakatau/attributes_raw.py
deleted file mode 100644
--- a/lib/Krakatau/Krakatau/attributes_raw.py
+++ /dev/null
@@ -1,21 +0,0 @@
-def get_attribute_raw(bytestream, ic_indices):
-    name_ind, length = bytestream.get('>HL')
-
-    #Hotspot does not actually check the attribute length of InnerClasses prior to 49.0

From noreply at buildbot.pypy.org  Fri Dec 12 22:37:22 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 12 Dec 2014 22:37:22 +0100 (CET)
Subject: [pypy-commit] creflect default: Trying out the overloaded function
 declarations. Right now only
Message-ID: <20141212213722.EAFC61D24AD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r199:a34448506d46
Date: 2014-12-12 20:50 +0000
http://bitbucket.org/cffi/creflect/changeset/a34448506d46/

Log:	Trying out the overloaded function declarations. Right now only
	supports overloaded by number of arguments.

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -828,10 +828,10 @@
 
     PyObject *l_dict = ((zeffir_builder_t *)cb)->l_dict;
     PyObject *l_libname_obj = ((zeffir_builder_t *)cb)->lib->l_libname_obj;
+    PyObject *x = PyDict_GetItemString(l_dict, name);
 
     assert(trampl != NULL);
-    PyObject *x = make_builtin_func(l_libname_obj, name, ret,
-                                    args, nargs, trampl);
+    x = make_builtin_func(l_libname_obj, name, ret, args, nargs, trampl, x);
     if (x == NULL)
         return;
 
diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -1,5 +1,5 @@
 
-typedef struct {
+typedef struct ZefFuncSupportObject_s {
     PyObject_HEAD
 
     /* note that 'zfs_trampl' is either a '_crx_trampoline0_fn' or a
@@ -9,6 +9,7 @@
     int                 zfs_nargs;
     int                 zfs_dotdotdot;
     void               *zfs_trampl;
+    struct ZefFuncSupportObject_s *zfs_overloaded_next;
     PyMethodDef         zfs_md;
     size_t              zfs_size_args;
     CTypeDescrObject   *zfs_ret;
@@ -25,6 +26,14 @@
         Py_DECREF(zfs->zfs_args[i]);
     Py_DECREF(zfs->zfs_ret);
 
+    /* avoid long chain of recursive calls to zef_func_support_dealloc */
+    while (zfs->zfs_overloaded_next != NULL) {
+        ZefFuncSupportObject *zfs2 = zfs->zfs_overloaded_next;
+        zfs->zfs_overloaded_next = zfs2->zfs_overloaded_next;
+        zfs2->zfs_overloaded_next = NULL;
+        Py_DECREF(zfs2);
+    }
+
     PyObject_Del(zfs);
 }
 
@@ -117,21 +126,41 @@
 static PyObject *common_call(ZefFuncSupportObject *zfs, PyObject *args,
                              void *func, const char *funcname)
 {
-    int i, nargs = zfs->zfs_nargs;
-    Py_ssize_t actualnargs;
+    int nargs;
+    Py_ssize_t actualnargs = PyTuple_Size(args);
+
+    nargs = zfs->zfs_nargs;
+    if (actualnargs != nargs) {
+        if (PyErr_Occurred())
+            return NULL;
+
+        if (zfs->zfs_overloaded_next == NULL) {
+            PyErr_Format(PyExc_TypeError,
+                         "'%s' expected %d arguments, but got %zd",
+                         funcname, nargs, actualnargs);
+            return NULL;
+        }
+
+        /* XXX right now, pick the first overloaded function with the
+           correct number of arguments.  Later we'll need to pick the
+           right one based on the provided argument types. */
+        zfs = zfs->zfs_overloaded_next;
+        while (zfs->zfs_nargs != actualnargs) {
+            zfs = zfs->zfs_overloaded_next;
+            if (zfs == NULL) {
+                PyErr_Format(PyExc_TypeError,
+                             "'%s': no version with %zd arguments found",
+                             funcname, actualnargs);
+                return NULL;
+            }
+        }
+        nargs = zfs->zfs_nargs;
+    }
+
     void *llargs[nargs];
     char *llbuffer;
     size_t lloffset;
-    int returns_void;
-
-    actualnargs = PyTuple_Size(args);
-    if (actualnargs != nargs) {
-        if (!PyErr_Occurred())
-            PyErr_Format(PyExc_TypeError,
-                         "'%s' expected %d arguments, but got %zd",
-                         funcname, nargs, actualnargs);
-        return NULL;
-    }
+    int i, returns_void;
 
     llbuffer = alloca(zfs->zfs_size_args);
     returns_void = (zfs->zfs_ret->ct_flags & CT_VOID);
@@ -241,6 +270,7 @@
 
     memset(&zfs->zfs_md, 0, sizeof(PyMethodDef));
     zfs->zfs_trampl = NULL;
+    zfs->zfs_overloaded_next = NULL;
     zfs->zfs_nargs = nargs;
     zfs->zfs_dotdotdot = dotdotdot;
     zfs->zfs_ret = ret;
@@ -273,7 +303,8 @@
 static PyObject *make_builtin_func(PyObject *libname_obj,
                                    const char *funcname, _crx_type_t *ret,
                                    _crx_qual_type args[], int nargs,
-                                   _crx_trampoline0_fn trampl)
+                                   _crx_trampoline0_fn trampl,
+                                   PyObject *overloaded)
 {
     char *p;
     ZefFuncSupportObject *zfs;
@@ -290,10 +321,27 @@
     zfs->zfs_md.ml_flags = METH_VARARGS;
     /*zfs->zfs_md.ml_doc = ... */
 
-    PyObject *res = PyCFunction_NewEx(&zfs->zfs_md, (PyObject *)zfs,
-                                      libname_obj);
-    Py_DECREF(zfs);
-    return res;
+    if (overloaded != NULL && PyCFunction_Check(overloaded) &&
+            PyCFunction_GET_FUNCTION(overloaded) == &zfs_call) {
+
+        /* attach 'zfs' to the end of the chain of zfs_overloaded_next */
+        PyObject *oself = PyCFunction_GET_SELF(overloaded);
+        ZefFuncSupportObject *zfs1 = (ZefFuncSupportObject *)oself;
+
+        while (zfs1->zfs_overloaded_next != NULL) {
+            zfs1 = zfs1->zfs_overloaded_next;
+        }
+        zfs1->zfs_overloaded_next = zfs;
+        Py_INCREF(overloaded);
+        return overloaded;
+    }
+    else {
+        /* make a new PyCFunction */
+        PyObject *res = PyCFunction_NewEx(&zfs->zfs_md, (PyObject *)zfs,
+                                          libname_obj);
+        Py_DECREF(zfs);
+        return res;
+    }
 }
 
 static PyObject *cfunctype_getargs(CTypeDescrObject *ct)
diff --git a/zeffir/test/function.crx b/zeffir/test/function.crx
--- a/zeffir/test/function.crx
+++ b/zeffir/test/function.crx
@@ -1,3 +1,5 @@
+#include <stdarg.h>
+
 int simple_function(int x)
 {
     return x + 1;
@@ -22,6 +24,18 @@
 {
 }
 
+int sum_ints(int count, ...)
+{
+    int i, result = 0;
+    va_list ap;
+    va_start(ap, count);
+    for (i = 0; i < count; i++) {
+        result += va_arg(ap, int);
+    }
+    va_end(ap);
+    return result;
+}
+
 
 // CREFLECT: start
 
@@ -29,5 +43,10 @@
 int add_from_array(int array[], int count);
 void returning_nothing(char *);
 void cant_call_dotdotdot(int x, ...);
+int sum_ints(int);
+int sum_ints(int, int);
+int sum_ints(int, int, int);
+int sum_ints(int, int, int, int);
+int sum_ints(int, int, int, int, int);
 
 // CREFLECT: end
diff --git a/zeffir/test/test_function.py b/zeffir/test/test_function.py
--- a/zeffir/test/test_function.py
+++ b/zeffir/test/test_function.py
@@ -49,3 +49,11 @@
     assert str(e.value) == ("cdata 'void(int, ...)' cannot be called, because "
                             "no wrapper was generated for functions of this "
                             "type (because signature ends in '...')")
+
+def test_overloaded_function():
+    ffi, lib = support.compile_and_open('function')
+    assert lib.sum_ints(0) == 0
+    assert lib.sum_ints(1, 10) == 10
+    assert lib.sum_ints(2, 20, 30) == 50
+    assert lib.sum_ints(3, 40, 50, 60) == 150
+    assert lib.sum_ints(4, 70, 80, 90, 100) == 340

From noreply at buildbot.pypy.org  Sat Dec 13 11:34:01 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 13 Dec 2014 11:34:01 +0100 (CET)
Subject: [pypy-commit] pypy more-rposix: hg merge default
Message-ID: <20141213103401.394731D2761@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: more-rposix
Changeset: r74907:f566a4468a10
Date: 2014-12-13 11:33 +0100
http://bitbucket.org/pypy/pypy/changeset/f566a4468a10/

Log:	hg merge default

diff too long, truncating to 2000 out of 2781 lines

diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -478,6 +478,29 @@
         except TypeError:
             pass
 
+    def CheckCurDescription(self):
+        self.cu.execute("select * from test")
+
+        actual = self.cu.description
+        expected = [
+            ('id', None, None, None, None, None, None),
+            ('name', None, None, None, None, None, None),
+            ('income', None, None, None, None, None, None),
+        ]
+        self.assertEqual(expected, actual)
+
+    def CheckCurDescriptionVoidStatement(self):
+        self.cu.execute("insert into test(name) values (?)", ("foo",))
+        self.assertIsNone(self.cu.description)
+
+    def CheckCurDescriptionWithoutStatement(self):
+        cu = self.cx.cursor()
+        try:
+            self.assertIsNone(cu.description)
+        finally:
+            cu.close()
+
+
 @unittest.skipUnless(threading, 'This test requires threading.')
 class ThreadTests(unittest.TestCase):
     def setUp(self):
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1175,8 +1175,9 @@
         try:
             return self.__description
         except AttributeError:
-            self.__description = self.__statement._get_description()
-            return self.__description
+            if self.__statement:
+                self.__description = self.__statement._get_description()
+                return self.__description
     description = property(__get_description)
 
     def __get_lastrowid(self):
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,6 +6,10 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
+**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
+on linux, linux64 and windows. We will make it the default on all platforms
+by the time of the next release.
+
 The first thing that you need is to compile PyPy yourself with the option
 ``--shared``. We plan to make ``--shared`` the default in the future. Consult
 the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -35,6 +35,13 @@
 PyPy's bytearray type is very inefficient. It would be an interesting
 task to look into possible optimizations on this.
 
+Implement AF_XXX packet types for PyPy
+--------------------------------------
+
+PyPy is missing AF_XXX types of sockets. Implementing it is easy-to-medium
+task. `bug report`_
+
+.. _`bug report`: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets#more
 
 Implement copy-on-write list slicing
 ------------------------------------
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -125,6 +125,8 @@
         else:
             return self.space.builtin
 
+    _NO_CELLS = []
+
     @jit.unroll_safe
     def initialize_frame_scopes(self, outer_func, code):
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
@@ -143,7 +145,7 @@
         nfreevars = len(code.co_freevars)
         if not nfreevars:
             if not ncellvars:
-                self.cells = []
+                self.cells = self._NO_CELLS
                 return            # no self.cells needed - fast path
         elif outer_func is None:
             space = self.space
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -14,6 +14,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.module._rawffi.alt.type_converter import FromAppLevelConverter, ToAppLevelConverter
 from pypy.module._rawffi.interp_rawffi import got_libffi_error, wrap_dlopenerror
+from pypy.module._rawffi import lasterror
 
 import os
 if os.name == 'nt':
@@ -201,11 +202,23 @@
         self.func = func
         self.argchain = argchain
 
+    def before(self):
+        lasterror.restore_last_error(self.space)
+
+    def after(self):
+        lasterror.save_last_error(self.space)
+
     def get_longlong(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.LONGLONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.LONGLONG)
+        self.after()
+        return x
 
     def get_ulonglong(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.ULONGLONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.ULONGLONG)
+        self.after()
+        return x
 
     def get_signed(self, w_ffitype):
         # if the declared return type of the function is smaller than LONG,
@@ -216,64 +229,94 @@
         # to space.wrap in order to get a nice applevel <int>.
         #
         restype = w_ffitype.get_ffitype()
+        self.before()
         call = self.func.call
         if restype is libffi.types.slong:
-            return call(self.argchain, rffi.LONG)
+            x = call(self.argchain, rffi.LONG)
         elif restype is libffi.types.sint:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.INT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.INT))
         elif restype is libffi.types.sshort:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT))
         elif restype is libffi.types.schar:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR))
         else:
-            self.error(w_ffitype)
+            raise self.error(w_ffitype)
+        self.after()
+        return x
 
     def get_unsigned(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.ULONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.ULONG)
+        self.after()
+        return x
 
     def get_unsigned_which_fits_into_a_signed(self, w_ffitype):
         # the same comment as get_signed apply
         restype = w_ffitype.get_ffitype()
+        self.before()
         call = self.func.call
         if restype is libffi.types.uint:
             assert not libffi.IS_32_BIT
             # on 32bit machines, we should never get here, because it's a case
             # which has already been handled by get_unsigned above.
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT))
         elif restype is libffi.types.ushort:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT))
         elif restype is libffi.types.uchar:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR))
         else:
-            self.error(w_ffitype)
+            raise self.error(w_ffitype)
+        self.after()
+        return x
 
 
     def get_pointer(self, w_ffitype):
+        self.before()
         ptrres = self.func.call(self.argchain, rffi.VOIDP)
+        self.after()
         return rffi.cast(rffi.ULONG, ptrres)
 
     def get_char(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.UCHAR)
+        self.before()
+        x = self.func.call(self.argchain, rffi.UCHAR)
+        self.after()
+        return x
 
     def get_unichar(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.WCHAR_T)
+        self.before()
+        x = self.func.call(self.argchain, rffi.WCHAR_T)
+        self.after()
+        return x
 
     def get_float(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.DOUBLE)
+        self.before()
+        x = self.func.call(self.argchain, rffi.DOUBLE)
+        self.after()
+        return x
 
     def get_singlefloat(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.FLOAT)
+        self.before()
+        x = self.func.call(self.argchain, rffi.FLOAT)
+        self.after()
+        return x
 
     def get_struct(self, w_ffitype, w_structdescr):
+        self.before()
         addr = self.func.call(self.argchain, rffi.LONG, is_struct=True)
+        self.after()
         return w_structdescr.fromaddress(self.space, addr)
 
     def get_struct_rawffi(self, w_ffitype, w_structdescr):
+        self.before()
         uintval = self.func.call(self.argchain, rffi.ULONG, is_struct=True)
+        self.after()
         return w_structdescr.fromaddress(self.space, uintval)
 
     def get_void(self, w_ffitype):
-        return self.func.call(self.argchain, lltype.Void)
+        self.before()
+        x = self.func.call(self.argchain, lltype.Void)
+        self.after()
+        return x
 
 
 def unpack_argtypes(space, w_argtypes, w_restype):
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -18,6 +18,7 @@
 from rpython.rlib.rarithmetic import intmask, r_uint
 from pypy.module._rawffi.buffer import RawFFIBuffer
 from pypy.module._rawffi.tracker import tracker
+from pypy.module._rawffi import lasterror
 
 TYPEMAP = {
     # XXX A mess with unsigned/signed/normal chars :-/
@@ -495,10 +496,14 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                lasterror.restore_last_error(space)
                 self.ptr.call(args_ll, result.ll_buffer)
+                lasterror.save_last_error(space)
                 return space.wrap(result)
             else:
+                lasterror.restore_last_error(space)
                 self.ptr.call(args_ll, lltype.nullptr(rffi.VOIDP.TO))
+                lasterror.save_last_error(space)
                 return space.w_None
         except StackCheckError, e:
             raise OperationError(space.w_ValueError, space.wrap(e.message))
@@ -615,12 +620,10 @@
 
 if sys.platform == 'win32':
     def get_last_error(space):
-        from rpython.rlib.rwin32 import GetLastError
-        return space.wrap(GetLastError())
+        return space.wrap(lasterror.fetch_last_error(space))
     @unwrap_spec(error=int)
     def set_last_error(space, error):
-        from rpython.rlib.rwin32 import SetLastError
-        SetLastError(error)
+        lasterror.store_last_error(space, error)
 else:
     # always have at least a dummy version of these functions
     # (https://bugs.pypy.org/issue1242)
diff --git a/pypy/module/_rawffi/lasterror.py b/pypy/module/_rawffi/lasterror.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_rawffi/lasterror.py
@@ -0,0 +1,40 @@
+# For Windows only.
+# https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
+
+import os
+
+_MS_WINDOWS = os.name == "nt"
+
+
+if _MS_WINDOWS:
+    from rpython.rlib import rwin32
+    from pypy.interpreter.executioncontext import ExecutionContext
+
+
+    ExecutionContext._rawffi_last_error = 0
+
+    def fetch_last_error(space):
+        ec = space.getexecutioncontext()
+        return ec._rawffi_last_error
+
+    def store_last_error(space, last_error):
+        ec = space.getexecutioncontext()
+        ec._rawffi_last_error = last_error
+
+    def restore_last_error(space):
+        ec = space.getexecutioncontext()
+        lasterror = ec._rawffi_last_error
+        rwin32.SetLastError(lasterror)
+
+    def save_last_error(space):
+        lasterror = rwin32.GetLastError()
+        ec = space.getexecutioncontext()
+        ec._rawffi_last_error = lasterror
+
+else:
+
+    def restore_last_error(space):
+        pass
+
+    def save_last_error(space):
+        pass
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -16,6 +16,7 @@
         #include "src/precommondefs.h"
         #include <stdlib.h>
         #include <stdio.h>
+        #include <errno.h>
 
         struct x
         {
@@ -204,6 +205,24 @@
             return inp;
         }
 
+        RPY_EXPORTED
+        int check_errno(int incoming)
+        {
+            int old_errno = errno;
+            errno = incoming;
+            return old_errno;
+        }
+
+        #ifdef _WIN32
+        #include <Windows.h>
+        RPY_EXPORTED
+        int check_last_error(int incoming)
+        {
+            int old_errno = GetLastError();
+            SetLastError(incoming);
+            return old_errno;
+        }
+        #endif
         '''))
         eci = ExternalCompilationInfo(include_dirs=[cdir])
         return str(platform.compile([c_file], eci, 'x', standalone=False))
@@ -1150,6 +1169,37 @@
         raises(OverflowError, "arg1[0] = 10**900")
         arg1.free()
 
+    def test_errno(self):
+        import _rawffi
+        lib = _rawffi.CDLL(self.lib_name)
+        A = _rawffi.Array('i')
+        f = lib.ptr('check_errno', ['i'], 'i')
+        _rawffi.set_errno(42)
+        arg = A(1)
+        arg[0] = 43
+        res = f(arg)
+        assert res[0] == 42
+        z = _rawffi.get_errno()
+        assert z == 43
+        arg.free()
+
+    def test_last_error(self):
+        import sys
+        if sys.platform != 'win32':
+            skip("Windows test")
+        import _rawffi
+        lib = _rawffi.CDLL(self.lib_name)
+        A = _rawffi.Array('i')
+        f = lib.ptr('check_last_error', ['i'], 'i')
+        _rawffi.set_last_error(42)
+        arg = A(1)
+        arg[0] = 43
+        res = f(arg)
+        assert res[0] == 42
+        z = _rawffi.get_last_error()
+        assert z == 43
+        arg.free()
+
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -52,7 +52,8 @@
 
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
-constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
+if not OPENSSL_NO_SSL3:
+    constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 
@@ -656,7 +657,7 @@
     # set up context
     if protocol == PY_SSL_VERSION_TLS1:
         method = libssl_TLSv1_method()
-    elif protocol == PY_SSL_VERSION_SSL3:
+    elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
         method = libssl_SSLv3_method()
     elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
         method = libssl_SSLv2_method()
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -58,7 +58,7 @@
     w_globals = from_ref(space, py_frame.c_f_globals)
 
     frame = space.FrameClass(space, code, w_globals, outer_func=None)
-    frame.f_lineno = py_frame.c_f_lineno
+    frame.f_lineno = rffi.getintfield(py_frame, 'c_f_lineno')
     w_obj = space.wrap(frame)
     track_reference(space, py_obj, w_obj)
     return w_obj
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -34,8 +34,8 @@
 
 
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
-                        "unegative", "flat", "tostring","count_nonzero",
-                        "argsort"]
+                        "unegative", "flat", "tostring", "count_nonzero",
+                        "argsort", "cumsum", "logical_xor_reduce"]
 TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted']
 TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype']
 THREE_ARG_FUNCTIONS = ['where']
@@ -559,6 +559,11 @@
                 w_res = arr.descr_any(interp.space)
             elif self.name == "all":
                 w_res = arr.descr_all(interp.space)
+            elif self.name == "cumsum":
+                w_res = arr.descr_cumsum(interp.space)
+            elif self.name == "logical_xor_reduce":
+                logical_xor = ufuncs.get(interp.space).logical_xor
+                w_res = logical_xor.reduce(interp.space, arr, None)
             elif self.name == "unegative":
                 neg = ufuncs.get(interp.space).negative
                 w_res = neg.call(interp.space, [arr])
diff --git a/pypy/module/micronumpy/flagsobj.py b/pypy/module/micronumpy/flagsobj.py
--- a/pypy/module/micronumpy/flagsobj.py
+++ b/pypy/module/micronumpy/flagsobj.py
@@ -5,46 +5,26 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import constants as NPY
-
+from pypy.module.micronumpy.strides import is_c_contiguous, is_f_contiguous
 
 def enable_flags(arr, flags):
     arr.flags |= flags
 
-
 def clear_flags(arr, flags):
     arr.flags &= ~flags
 
-
- at jit.unroll_safe
 def _update_contiguous_flags(arr):
-    shape = arr.shape
-    strides = arr.strides
-
-    is_c_contig = True
-    sd = arr.dtype.elsize
-    for i in range(len(shape) - 1, -1, -1):
-        dim = shape[i]
-        if strides[i] != sd:
-            is_c_contig = False
-            break
-        if dim == 0:
-            break
-        sd *= dim
+    is_c_contig = is_c_contiguous(arr)
     if is_c_contig:
         enable_flags(arr, NPY.ARRAY_C_CONTIGUOUS)
     else:
         clear_flags(arr, NPY.ARRAY_C_CONTIGUOUS)
 
-    sd = arr.dtype.elsize
-    for i in range(len(shape)):
-        dim = shape[i]
-        if strides[i] != sd:
-            clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
-            return
-        if dim == 0:
-            break
-        sd *= dim
-    enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
+    is_f_contig = is_f_contiguous(arr)
+    if is_f_contig:
+        enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
+    else:
+        clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
 
 
 class W_FlagsObject(W_Root):
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -166,6 +166,7 @@
 def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity):
     obj_iter, obj_state = obj.create_iter()
     out_iter, out_state = out.create_iter()
+    out_iter.track_index = False
     if identity is None:
         cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
         out_iter.setitem(out_state, cur_value)
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -19,7 +19,7 @@
     order_converter, shape_converter, searchside_converter
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.strides import get_shape_from_iterable, \
-    shape_agreement, shape_agreement_multiple
+    shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
 
 
 def _match_dot_shapes(space, left, right):
@@ -837,7 +837,15 @@
                 raise OperationError(space.w_ValueError, space.wrap(
                     "new type not compatible with array."))
         else:
-            if dims == 1 or impl.get_strides()[0] < impl.get_strides()[-1]:
+            if not is_c_contiguous(impl) and not is_f_contiguous(impl):
+                if old_itemsize != new_itemsize:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array."))
+                # Strides, shape does not change
+                v = impl.astype(space, dtype)
+                return wrap_impl(space, w_type, self, v) 
+            strides = impl.get_strides()
+            if dims == 1 or strides[0] <strides[-1]:
                 # Column-major, resize first dimension
                 if new_shape[0] * old_itemsize % new_itemsize != 0:
                     raise OperationError(space.w_ValueError, space.wrap(
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -429,3 +429,35 @@
                     n_old_elems_to_use *= old_shape[oldI]
     assert len(new_strides) == len(new_shape)
     return new_strides[:]
+
+ at jit.unroll_safe
+def is_c_contiguous(arr):
+    shape = arr.get_shape()
+    strides = arr.get_strides()
+    ret = True
+    sd = arr.dtype.elsize
+    for i in range(len(shape) - 1, -1, -1):
+        dim = shape[i]
+        if strides[i] != sd:
+            ret = False
+            break
+        if dim == 0:
+            break
+        sd *= dim
+    return ret
+
+ at jit.unroll_safe
+def is_f_contiguous(arr):
+    shape = arr.get_shape()
+    strides = arr.get_strides()
+    ret = True
+    sd = arr.dtype.elsize
+    for i in range(len(shape)):
+        dim = shape[i]
+        if strides[i] != sd:
+            ret = False
+            break
+        if dim == 0:
+            break
+        sd *= dim
+    return ret
diff --git a/pypy/module/micronumpy/test/test_iterators.py b/pypy/module/micronumpy/test/test_iterators.py
--- a/pypy/module/micronumpy/test/test_iterators.py
+++ b/pypy/module/micronumpy/test/test_iterators.py
@@ -13,6 +13,11 @@
         self.strides = strides
         self.start = start
 
+    def get_shape(self):
+        return self.shape
+
+    def get_strides(self):
+        return self.strides
 
 class TestIterDirect(object):
     def test_iterator_basic(self):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1807,6 +1807,17 @@
         x = array([], dtype=[('a', 'int8'), ('b', 'int8')])
         y = x.view(dtype='int16')
 
+    def test_view_of_slice(self):
+        from numpy import empty
+        x = empty([6], 'uint32')
+        x.fill(0xdeadbeef)
+        s = x[::3]
+        exc = raises(ValueError, s.view, 'uint8')
+        assert exc.value[0] == 'new type not compatible with array.'
+        s[...] = 2
+        v = s.view(x.__class__)
+        assert (v == 2).all()
+    
     def test_tolist_scalar(self):
         from numpy import dtype
         int32 = dtype('int32').type
@@ -2912,11 +2923,7 @@
         a = empty(10, dtype=[(_, int) for _ in 'abcde'])
         a.fill(123)
         for i in a:
-            import sys
-            if '__pypy__' in sys.builtin_module_names:
-                assert tuple(i) == (123,) + (0,) * 4
-            else:
-                assert tuple(i) == (123,) * 5
+            assert tuple(i) == (123,) * 5
 
         a = zeros(3, dtype=dtype(complex).newbyteorder())
         a.fill(1.5+2.5j)
@@ -3846,7 +3853,7 @@
         a = np.array([b, b, b], dtype=dt)
         assert a.shape == (3, 2)
         for i in a.flat:
-            assert tuple(i) == (True, False)
+            assert tuple(i) == (True, True)
 
         dt = np.dtype([('A', '<i8'), ('B', '<f8'), ('C', '<c16')])
         b = np.array((999999, 1e+20, 1e+20+0j), dtype=dt)
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -183,6 +183,28 @@
             'raw_load': 1,
         })
 
+    def define_cumsum():
+        return """
+        a = |30|
+        b = cumsum(a)
+        b -> 5
+        """
+
+    def test_cumsum(self):
+        result = self.run("cumsum")
+        assert result == 15
+        self.check_trace_count(1)
+        self.check_simple_loop({
+            'float_add': 1,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'int_add': 3,
+            'int_ge': 1,
+            'jump': 1,
+            'raw_load': 1,
+            'raw_store': 1,
+        })
+
     def define_axissum():
         return """
         a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
@@ -218,7 +240,7 @@
             'getfield_gc': 5,
             'getfield_gc_pure': 51,
             'guard_class': 3,
-            'guard_false': 12,
+            'guard_false': 13,
             'guard_nonnull': 11,
             'guard_nonnull_class': 3,
             'guard_not_invalidated': 2,
@@ -378,6 +400,7 @@
     def test_all(self):
         result = self.run("all")
         assert result == 1
+        self.check_trace_count(1)
         self.check_simple_loop({
             'cast_float_to_int': 1,
             'guard_false': 1,
@@ -390,6 +413,35 @@
             'raw_load': 1,
         })
 
+    def define_logical_xor_reduce():
+        return """
+        a = [1,1,1,1,1,1,1,1]
+        logical_xor_reduce(a)
+        """
+
+    def test_logical_xor_reduce(self):
+        result = self.run("logical_xor_reduce")
+        assert result == 0
+        self.check_trace_count(2)
+        # XXX fix this
+        self.check_simple_loop({
+            'cast_float_to_int': 1,
+            'getfield_gc': 2,
+            'getfield_gc_pure': 11,
+            'guard_class': 1,
+            'guard_false': 1,
+            'guard_not_invalidated': 1,
+            'guard_true': 5,
+            'int_add': 2,
+            'int_and': 1,
+            'int_ge': 1,
+            'int_is_true': 2,
+            'jump': 1,
+            'new_with_vtable': 1,
+            'raw_load': 1,
+            'setfield_gc': 4,
+        })
+
     def define_already_forced():
         return """
         a = |30|
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -285,7 +285,9 @@
 
     @raw_binary_op
     def logical_xor(self, v1, v2):
-        return bool(v1) ^ bool(v2)
+        a = bool(v1)
+        b = bool(v2)
+        return (not b and a) or (not a and b)
 
     @raw_unary_op
     def bool(self, v):
@@ -309,26 +311,26 @@
     BoxType = boxes.W_BoolBox
     format_code = "?"
 
-    True = BoxType(True)
-    False = BoxType(False)
+    _True = BoxType(True)
+    _False = BoxType(False)
 
     @specialize.argtype(1)
     def box(self, value):
         box = Primitive.box(self, value)
         if box.value:
-            return self.True
+            return self._True
         else:
-            return self.False
+            return self._False
 
     @specialize.argtype(1, 2)
     def box_complex(self, real, imag):
         box = Primitive.box(self, real)
         if box.value:
-            return self.True
+            return self._True
         box = Primitive.box(self, imag)
         if box.value:
-            return self.True
-        return self.False
+            return self._True
+        return self._False
 
     def coerce_subtype(self, space, w_subtype, w_item):
         # Doesn't return subclasses so it can return the constants.
@@ -1258,7 +1260,9 @@
 
     @raw_binary_op
     def logical_xor(self, v1, v2):
-        return self._bool(v1) ^ self._bool(v2)
+        a = self._bool(v1)
+        b = self._bool(v2)
+        return (not b and a) or (not a and b)
 
     def min(self, v1, v2):
         if self.le(v1, v2) or self.isnan(v1):
@@ -1733,7 +1737,9 @@
 
     @str_binary_op
     def logical_xor(self, v1, v2):
-        return bool(v1) ^ bool(v2)
+        a = bool(v1)
+        b = bool(v2)
+        return (not b and a) or (not a and b)
 
     def bool(self, v):
         return bool(self.to_str(v))
@@ -1863,7 +1869,7 @@
                 items_w = space.fixedview(w_item.get_scalar_value())
             else:
                 # XXX support initializing from readable buffers
-                items_w = [w_item]
+                items_w = [w_item] * len(dtype.fields)
         else:
             items_w = [None] * len(dtype.fields)
         arr = VoidBoxStorage(dtype.elsize, dtype)
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -415,9 +415,12 @@
         self.match_descr(op.descr, exp_descr)
 
 
-    def _next_op(self, iter_ops, assert_raises=False):
+    def _next_op(self, iter_ops, assert_raises=False, ignore_ops=set()):
         try:
-            op = iter_ops.next()
+            while True:
+                op = iter_ops.next()
+                if op.name not in ignore_ops:
+                    break
         except StopIteration:
             self._assert(assert_raises, "not enough operations")
             return
@@ -454,9 +457,7 @@
         else:
             assert 0, "'{{{' not followed by '}}}'"
         while exp_ops:
-            op = self._next_op(iter_ops)
-            if op.name in ignore_ops:
-                continue
+            op = self._next_op(iter_ops, ignore_ops=ignore_ops)
             # match 'op' against any of the exp_ops; the first successful
             # match is kept, and the exp_op gets removed from the list
             for i, exp_op in enumerate(exp_ops):
@@ -491,10 +492,7 @@
                     self.match_any_order(iter_exp_ops, iter_ops, ignore_ops)
                     continue
                 else:
-                    while True:
-                        op = self._next_op(iter_ops)
-                        if op.name not in ignore_ops:
-                            break
+                    op = self._next_op(iter_ops, ignore_ops=ignore_ops)
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
                 if type(exp_op) is not str and exp_op[4] is False:    # optional operation
@@ -504,7 +502,7 @@
                 raise
         #
         # make sure we exhausted iter_ops
-        self._next_op(iter_ops, assert_raises=True)
+        self._next_op(iter_ops, assert_raises=True, ignore_ops=ignore_ops)
 
     def match(self, expected_src, ignore_ops=[]):
         def format(src, opindex=None):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -318,6 +318,16 @@
             jump(i4, descr=...)
         """
         assert self.match(loop, expected, ignore_ops=['force_token'])
+        #
+        loop = """
+            [i0]
+            i1 = int_add(i0, 1)
+            i4 = force_token()
+        """
+        expected = """
+            i1 = int_add(i0, 1)
+        """
+        assert self.match(loop, expected, ignore_ops=['force_token'])
 
     def test_match_dots_in_arguments(self):
         loop = """
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -342,9 +342,9 @@
         guard_false(i114, descr=...)
         --TICK--
         i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=4 OS=110>)
-        raw_store(i119, 0, i112, descr=<ArrayS 2>)
-        raw_store(i119, 2, i112, descr=<ArrayS 2>)
-        raw_store(i119, 4, i112, descr=<ArrayS 2>)
+        raw_store(i119, 0, i160, descr=<ArrayS 2>)
+        raw_store(i119, 2, i160, descr=<ArrayS 2>)
+        raw_store(i119, 4, i160, descr=<ArrayS 2>)
         setfield_gc(p167, i119, descr=<FieldU pypy.module._cffi_backend.cdataobj.W_CData.inst__cdata .+>)
         i123 = arraylen_gc(p67, descr=<ArrayP .>)
         jump(..., descr=...)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -30,9 +30,7 @@
             guard_true(i14, descr=...)
             i15 = getfield_gc_pure(p1, descr=<FieldU pypy.module.micronumpy.boxes.W_BoolBox.inst_value \d+>)
             i16 = int_is_true(i15)
-            i18 = int_xor(i16, 1)
-            i19 = int_is_true(i18)
-            guard_true(i19, descr=...)
+            guard_false(i16, descr=...)
             i20 = getfield_gc(p2, descr=<FieldS pypy.module.micronumpy.iterators.IterState.inst_index \d+>)
             i21 = getfield_gc_pure(p0, descr=<FieldU pypy.module.micronumpy.iterators.ArrayIter.inst_track_index \d+>)
             guard_true(i21, descr=...)
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -3,6 +3,7 @@
 from __future__ import absolute_import
 import py
 
+
 class BaseTestDatetime:
     def test_repr(self):
         print datetime
@@ -210,11 +211,13 @@
             naive == aware
         assert str(e.value) == "can't compare offset-naive and offset-aware times"
 
-class TestDatetimeCPython(BaseTestDatetime):
+
+class TestDatetimeHost(BaseTestDatetime):
     def setup_class(cls):
         global datetime
         import datetime
 
+
 class TestDatetimePyPy(BaseTestDatetime):
     def setup_class(cls):
         global datetime
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -1,267 +1,285 @@
 # -*- coding: utf-8 -*-
 """Tests for _sqlite3.py"""
 
-import pytest, sys
+from __future__ import absolute_import
+import pytest
+import sys
 
-if sys.version_info < (2, 7):
-    pytest.skip("_sqlite3 requires Python 2.7")
-try:
-    import _cffi_backend
-except ImportError:
-    # On CPython, "pip install cffi".  On old PyPy's, no chance
-    pytest.skip("_sqlite3 requires _cffi_backend to be installed")
-
-from lib_pypy import _sqlite3
 
 def pytest_funcarg__con(request):
     con = _sqlite3.connect(':memory:')
     request.addfinalizer(lambda: con.close())
     return con
 
-def test_list_ddl(con):
-    """From issue996.  Mostly just looking for lack of exceptions."""
-    cursor = con.cursor()
-    cursor.execute('CREATE TABLE foo (bar INTEGER)')
-    result = list(cursor)
-    assert result == []
-    cursor.execute('INSERT INTO foo (bar) VALUES (42)')
-    result = list(cursor)
-    assert result == []
-    cursor.execute('SELECT * FROM foo')
-    result = list(cursor)
-    assert result == [(42,)]
 
-def test_connect_takes_same_positional_args_as_Connection(con):
-    from inspect import getargspec
-    clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
-    conargs = getargspec(_sqlite3.connect).args
-    assert clsargs == conargs
+class BaseTestSQLite:
+    def test_list_ddl(self, con):
+        """From issue996.  Mostly just looking for lack of exceptions."""
+        cursor = con.cursor()
+        cursor.execute('CREATE TABLE foo (bar INTEGER)')
+        result = list(cursor)
+        assert result == []
+        cursor.execute('INSERT INTO foo (bar) VALUES (42)')
+        result = list(cursor)
+        assert result == []
+        cursor.execute('SELECT * FROM foo')
+        result = list(cursor)
+        assert result == [(42,)]
 
-def test_total_changes_after_close(con):
-    con.close()
-    pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
+    def test_connect_takes_same_positional_args_as_Connection(self, con):
+        if not hasattr(_sqlite3, '_ffi'):
+            pytest.skip("only works for lib_pypy _sqlite3")
+        from inspect import getargspec
+        clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
+        conargs = getargspec(_sqlite3.connect).args
+        assert clsargs == conargs
 
-def test_connection_check_init():
-    class Connection(_sqlite3.Connection):
-        def __init__(self, name):
+    def test_total_changes_after_close(self, con):
+        con.close()
+        pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
+
+    def test_connection_check_init(self):
+        class Connection(_sqlite3.Connection):
+            def __init__(self, name):
+                pass
+
+        con = Connection(":memory:")
+        e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()")
+        assert '__init__' in e.value.message
+
+    def test_cursor_check_init(self, con):
+        class Cursor(_sqlite3.Cursor):
+            def __init__(self, name):
+                pass
+
+        cur = Cursor(con)
+        e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')")
+        assert '__init__' in e.value.message
+
+    def test_connection_after_close(self, con):
+        pytest.raises(TypeError, "con()")
+        con.close()
+        # raises ProgrammingError because should check closed before check args
+        pytest.raises(_sqlite3.ProgrammingError, "con()")
+
+    def test_cursor_iter(self, con):
+        cur = con.cursor()
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        next(cur)
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        con.commit()
+        next(cur)
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        with pytest.raises(_sqlite3.ProgrammingError):
+            cur.executemany('select 1', [])
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        cur.execute('create table test(ing)')
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        cur.execute('insert into test values(1)')
+        con.commit()
+        with pytest.raises(StopIteration):
+            next(cur)
+
+    def test_cursor_after_close(self, con):
+        cur = con.execute('select 1')
+        cur.close()
+        con.close()
+        pytest.raises(_sqlite3.ProgrammingError, "cur.close()")
+        # raises ProgrammingError because should check closed before check args
+        pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)")
+        pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)")
+
+    @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')")
+    def test_connection_del(self, tmpdir):
+        """For issue1325."""
+        import os
+        import gc
+        try:
+            import resource
+        except ImportError:
+            pytest.skip("needs resource module")
+
+        limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+        try:
+            fds = 0
+            while True:
+                fds += 1
+                resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
+                try:
+                    for p in os.pipe(): os.close(p)
+                except OSError:
+                    assert fds < 100
+                else:
+                    break
+
+            def open_many(cleanup):
+                con = []
+                for i in range(3):
+                    con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
+                    if cleanup:
+                        con[i] = None
+                        gc.collect(); gc.collect()
+
+            pytest.raises(_sqlite3.OperationalError, open_many, False)
+            gc.collect(); gc.collect()
+            open_many(True)
+        finally:
+            resource.setrlimit(resource.RLIMIT_NOFILE, limit)
+
+    def test_on_conflict_rollback_executemany(self, con):
+        major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+        if (int(major), int(minor), int(micro)) < (3, 2, 2):
+            pytest.skip("requires sqlite3 version >= 3.2.2")
+        con.execute("create table foo(x, unique(x) on conflict rollback)")
+        con.execute("insert into foo(x) values (1)")
+        try:
+            con.executemany("insert into foo(x) values (?)", [[1]])
+        except _sqlite3.DatabaseError:
             pass
+        con.execute("insert into foo(x) values (2)")
+        try:
+            con.commit()
+        except _sqlite3.OperationalError:
+            pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
 
-    con = Connection(":memory:")
-    e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()")
-    assert '__init__' in e.value.message
+    def test_statement_arg_checking(self, con):
+        with pytest.raises(_sqlite3.Warning) as e:
+            con(123)
+        assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.'
+        with pytest.raises(ValueError) as e:
+            con.execute(123)
+        assert str(e.value) == 'operation parameter must be str or unicode'
+        with pytest.raises(ValueError) as e:
+            con.executemany(123, 123)
+        assert str(e.value) == 'operation parameter must be str or unicode'
+        with pytest.raises(ValueError) as e:
+            con.executescript(123)
+        assert str(e.value) == 'script argument must be unicode or string.'
 
-def test_cursor_check_init(con):
-    class Cursor(_sqlite3.Cursor):
-        def __init__(self, name):
-            pass
+    def test_statement_param_checking(self, con):
+        con.execute('create table foo(x)')
+        con.execute('insert into foo(x) values (?)', [2])
+        con.execute('insert into foo(x) values (?)', (2,))
+        class seq(object):
+            def __len__(self):
+                return 1
+            def __getitem__(self, key):
+                return 2
+        con.execute('insert into foo(x) values (?)', seq())
+        del seq.__len__
+        with pytest.raises(_sqlite3.ProgrammingError):
+            con.execute('insert into foo(x) values (?)', seq())
+        with pytest.raises(_sqlite3.ProgrammingError):
+            con.execute('insert into foo(x) values (?)', {2:2})
+        with pytest.raises(ValueError) as e:
+            con.execute('insert into foo(x) values (?)', 2)
+        assert str(e.value) == 'parameters are of unsupported type'
 
-    cur = Cursor(con)
-    e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')")
-    assert '__init__' in e.value.message
+    def test_explicit_begin(self, con):
+        con.execute('BEGIN')
+        con.execute('BEGIN ')
+        con.execute('BEGIN')
+        con.commit()
+        con.execute('BEGIN')
+        con.commit()
 
-def test_connection_after_close(con):
-    pytest.raises(TypeError, "con()")
-    con.close()
-    # raises ProgrammingError because should check closed before check args
-    pytest.raises(_sqlite3.ProgrammingError, "con()")
+    def test_row_factory_use(self, con):
+        con.row_factory = 42
+        con.execute('select 1')
 
-def test_cursor_iter(con):
-    cur = con.cursor()
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_returning_blob_must_own_memory(self, con):
+        import gc
+        con.create_function("returnblob", 0, lambda: buffer("blob"))
+        cur = con.execute("select returnblob()")
+        val = cur.fetchone()[0]
+        for i in range(5):
+            gc.collect()
+            got = (val[0], val[1], val[2], val[3])
+            assert got == ('b', 'l', 'o', 'b')
+        # in theory 'val' should be a read-write buffer
+        # but it's not right now
+        if not hasattr(_sqlite3, '_ffi'):
+            val[1] = 'X'
+            got = (val[0], val[1], val[2], val[3])
+            assert got == ('b', 'X', 'o', 'b')
 
-    cur.execute('select 1')
-    next(cur)
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_description_after_fetchall(self, con):
+        cur = con.cursor()
+        assert cur.description is None
+        cur.execute("select 42").fetchall()
+        assert cur.description is not None
 
-    cur.execute('select 1')
-    con.commit()
-    next(cur)
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_executemany_lastrowid(self, con):
+        cur = con.cursor()
+        cur.execute("create table test(a)")
+        cur.executemany("insert into test values (?)", [[1], [2], [3]])
+        assert cur.lastrowid is None
 
-    with pytest.raises(_sqlite3.ProgrammingError):
-        cur.executemany('select 1', [])
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_authorizer_bad_value(self, con):
+        def authorizer_cb(action, arg1, arg2, dbname, source):
+            return 42
+        con.set_authorizer(authorizer_cb)
+        with pytest.raises(_sqlite3.OperationalError) as e:
+            con.execute('select 123')
+        major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+        if (int(major), int(minor), int(micro)) >= (3, 6, 14):
+            assert str(e.value) == 'authorizer malfunction'
+        else:
+            assert str(e.value) == \
+                ("illegal return value (1) from the authorization function - "
+                 "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
 
-    cur.execute('select 1')
-    cur.execute('create table test(ing)')
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_issue1573(self, con):
+        cur = con.cursor()
+        cur.execute(u'SELECT 1 as m&#233;il')
+        assert cur.description[0][0] == u"m&#233;il".encode('utf-8')
 
-    cur.execute('select 1')
-    cur.execute('insert into test values(1)')
-    con.commit()
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_adapter_exception(self, con):
+        def cast(obj):
+            raise ZeroDivisionError
 
-def test_cursor_after_close(con):
-    cur = con.execute('select 1')
-    cur.close()
-    con.close()
-    pytest.raises(_sqlite3.ProgrammingError, "cur.close()")
-    # raises ProgrammingError because should check closed before check args
-    pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)")
-    pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)")
+        _sqlite3.register_adapter(int, cast)
+        try:
+            cur = con.cursor()
+            cur.execute("select ?", (4,))
+            val = cur.fetchone()[0]
+            # Adapter error is ignored, and parameter is passed as is.
+            assert val == 4
+            assert type(val) is int
+        finally:
+            del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
 
- at pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')")
-def test_connection_del(tmpdir):
-    """For issue1325."""
-    import os
-    import gc
-    try:
-        import resource
-    except ImportError:
-        pytest.skip("needs resource module")
 
-    limit = resource.getrlimit(resource.RLIMIT_NOFILE)
-    try:
-        fds = 0
-        while True:
-            fds += 1
-            resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
-            try:
-                for p in os.pipe(): os.close(p)
-            except OSError:
-                assert fds < 100
-            else:
-                break
-        def open_many(cleanup):
-            con = []
-            for i in range(3):
-                con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
-                if cleanup:
-                    con[i] = None
-                    gc.collect(); gc.collect()
+class TestSQLiteHost(BaseTestSQLite):
+    def setup_class(cls):
+        global _sqlite3
+        import _sqlite3
 
-        pytest.raises(_sqlite3.OperationalError, open_many, False)
-        gc.collect(); gc.collect()
-        open_many(True)
-    finally:
-        resource.setrlimit(resource.RLIMIT_NOFILE, limit)
 
-def test_on_conflict_rollback_executemany(con):
-    major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
-    if (int(major), int(minor), int(micro)) < (3, 2, 2):
-        pytest.skip("requires sqlite3 version >= 3.2.2")
-    con.execute("create table foo(x, unique(x) on conflict rollback)")
-    con.execute("insert into foo(x) values (1)")
-    try:
-        con.executemany("insert into foo(x) values (?)", [[1]])
-    except _sqlite3.DatabaseError:
-        pass
-    con.execute("insert into foo(x) values (2)")
-    try:
-        con.commit()
-    except _sqlite3.OperationalError:
-        pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
+class TestSQLitePyPy(BaseTestSQLite):
+    def setup_class(cls):
+        if sys.version_info < (2, 7):
+            pytest.skip("_sqlite3 requires Python 2.7")
 
-def test_statement_arg_checking(con):
-    with pytest.raises(_sqlite3.Warning) as e:
-        con(123)
-    assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.'
-    with pytest.raises(ValueError) as e:
-        con.execute(123)
-    assert str(e.value) == 'operation parameter must be str or unicode'
-    with pytest.raises(ValueError) as e:
-        con.executemany(123, 123)
-    assert str(e.value) == 'operation parameter must be str or unicode'
-    with pytest.raises(ValueError) as e:
-        con.executescript(123)
-    assert str(e.value) == 'script argument must be unicode or string.'
+        try:
+            import _cffi_backend
+        except ImportError:
+            # On CPython, "pip install cffi".  On old PyPy's, no chance
+            pytest.skip("_sqlite3 requires _cffi_backend to be installed")
 
-def test_statement_param_checking(con):
-    con.execute('create table foo(x)')
-    con.execute('insert into foo(x) values (?)', [2])
-    con.execute('insert into foo(x) values (?)', (2,))
-    class seq(object):
-        def __len__(self):
-            return 1
-        def __getitem__(self, key):
-            return 2
-    con.execute('insert into foo(x) values (?)', seq())
-    del seq.__len__
-    with pytest.raises(_sqlite3.ProgrammingError):
-        con.execute('insert into foo(x) values (?)', seq())
-    with pytest.raises(_sqlite3.ProgrammingError):
-        con.execute('insert into foo(x) values (?)', {2:2})
-    with pytest.raises(ValueError) as e:
-        con.execute('insert into foo(x) values (?)', 2)
-    assert str(e.value) == 'parameters are of unsupported type'
-
-def test_explicit_begin(con):
-    con.execute('BEGIN')
-    con.execute('BEGIN ')
-    con.execute('BEGIN')
-    con.commit()
-    con.execute('BEGIN')
-    con.commit()
-
-def test_row_factory_use(con):
-    con.row_factory = 42
-    con.execute('select 1')
-
-def test_returning_blob_must_own_memory(con):
-    import gc
-    con.create_function("returnblob", 0, lambda: buffer("blob"))
-    cur = con.execute("select returnblob()")
-    val = cur.fetchone()[0]
-    for i in range(5):
-        gc.collect()
-        got = (val[0], val[1], val[2], val[3])
-        assert got == ('b', 'l', 'o', 'b')
-    # in theory 'val' should be a read-write buffer
-    # but it's not right now
-    pytest.skip("in theory 'val' should be a read-write buffer")
-    val[1] = 'X'
-    got = (val[0], val[1], val[2], val[3])
-    assert got == ('b', 'X', 'o', 'b')
-
-def test_description_after_fetchall(con):
-    cur = con.cursor()
-    cur.execute("select 42").fetchall()
-    assert cur.description is not None
-
-def test_executemany_lastrowid(con):
-    cur = con.cursor()
-    cur.execute("create table test(a)")
-    cur.executemany("insert into test values (?)", [[1], [2], [3]])
-    assert cur.lastrowid is None
-
-
-def test_authorizer_bad_value(con):
-    def authorizer_cb(action, arg1, arg2, dbname, source):
-        return 42
-    con.set_authorizer(authorizer_cb)
-    with pytest.raises(_sqlite3.OperationalError) as e:
-        con.execute('select 123')
-    major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
-    if (int(major), int(minor), int(micro)) >= (3, 6, 14):
-        assert str(e.value) == 'authorizer malfunction'
-    else:
-        assert str(e.value) == \
-            ("illegal return value (1) from the authorization function - "
-             "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
-
-
-def test_issue1573(con):
-    cur = con.cursor()
-    cur.execute(u'SELECT 1 as m&#233;il')
-    assert cur.description[0][0] == u"m&#233;il".encode('utf-8')
-
-def test_adapter_exception(con):
-    def cast(obj):
-        raise ZeroDivisionError
-
-    _sqlite3.register_adapter(int, cast)
-    try:
-        cur = con.cursor()
-        cur.execute("select ?", (4,))
-        val = cur.fetchone()[0]
-        # Adapter error is ignored, and parameter is passed as is.
-        assert val == 4
-        assert type(val) is int
-    finally:
-        del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
+        global _sqlite3
+        from lib_pypy import _sqlite3
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -366,8 +366,12 @@
             tup = w_self._lookup_where(name)
             return tup
         name = promote_string(name)
-        w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
-        return w_class, unwrap_cell(space, w_value)
+        tup_w = w_self._pure_lookup_where_with_method_cache(name, version_tag)
+        w_class, w_value = tup_w
+        if (space.config.objspace.std.withtypeversion and
+                isinstance(w_value, TypeCell)):
+            return w_class, w_value.w_value
+        return tup_w   # don't make a new tuple, reuse the old one
 
     def _pure_lookup_where_possibly_with_method_cache(w_self, name, version_tag):
         if w_self.space.config.objspace.std.withmethodcache:
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -19,7 +19,7 @@
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
 from rpython.annotator.argument import simple_args
-from rpython.rlib.objectmodel import r_dict, Symbolic
+from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper import extregistry
 
@@ -261,21 +261,23 @@
                     result.listdef.generalize(self.immutablevalue(e))
                 result.const_box = key
                 return result
-        elif tp is dict or tp is r_dict or tp is SomeOrderedDict.knowntype:
-            if tp is SomeOrderedDict.knowntype:
-                cls = SomeOrderedDict
-            else:
-                cls = SomeDict
+        elif (tp is dict or tp is r_dict or
+              tp is SomeOrderedDict.knowntype or tp is r_ordereddict):
             key = Constant(x)
             try:
                 return self.immutable_cache[key]
             except KeyError:
+                if tp is SomeOrderedDict.knowntype or tp is r_ordereddict:
+                    cls = SomeOrderedDict
+                else:
+                    cls = SomeDict
+                is_r_dict = issubclass(tp, r_dict)
                 result = cls(DictDef(self,
                                         s_ImpossibleValue,
                                         s_ImpossibleValue,
-                                        is_r_dict = tp is r_dict))
+                                        is_r_dict = is_r_dict))
                 self.immutable_cache[key] = result
-                if tp is r_dict:
+                if is_r_dict:
                     s_eqfn = self.immutablevalue(x.key_eq)
                     s_hashfn = self.immutablevalue(x.key_hash)
                     result.dictdef.dictkey.update_rdict_annotations(s_eqfn,
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -129,7 +129,8 @@
                default=False, cmdline=None),
 
     # misc
-    BoolOption("verbose", "Print extra information", default=False),
+    BoolOption("verbose", "Print extra information", default=False,
+               cmdline="--verbose"),
     StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"),
     StrOption("profopt", "Specify profile based optimization script",
               cmdline="--profopt"),
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -528,6 +528,10 @@
     pyfunc = staticmethod(getattr)
 
     def constfold(self):
+        from rpython.flowspace.flowcontext import FlowingError
+        if len(self.args) == 3:
+            raise FlowingError(
+                "getattr() with three arguments not supported: %s" % (self,))
         w_obj, w_name = self.args
         # handling special things like sys
         if (w_obj in NOT_REALLY_CONST and
@@ -538,7 +542,6 @@
             try:
                 result = getattr(obj, name)
             except Exception as e:
-                from rpython.flowspace.flowcontext import FlowingError
                 etype = e.__class__
                 msg = "getattr(%s, %s) always raises %s: %s" % (
                     obj, name, etype, e)
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -74,7 +74,6 @@
             # for a CALL_ASSEMBLER: record it as a potential jump.
             if descr is not original_jitcell_token:
                 original_jitcell_token.record_jump_to(descr)
-            descr.exported_state = None
             op.cleardescr()    # clear reference, mostly for tests
         elif isinstance(descr, TargetToken):
             # for a JUMP: record it as a potential jump.
@@ -84,10 +83,6 @@
             if descr.original_jitcell_token is not original_jitcell_token:
                 assert descr.original_jitcell_token is not None
                 original_jitcell_token.record_jump_to(descr.original_jitcell_token)
-            # exported_state is clear by optimizeopt when the short preamble is
-            # constrcucted. if that did not happen the label should not show up
-            # in a trace that will be used
-            assert descr.exported_state is None
             if not we_are_translated():
                 op._descr_wref = weakref.ref(op._descr)
             op.cleardescr()    # clear reference to prevent the history.Stats
@@ -133,7 +128,7 @@
                       [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
 
     try:
-        optimize_trace(metainterp_sd, part, enable_opts)
+        start_state = optimize_trace(metainterp_sd, part, enable_opts)
     except InvalidLoop:
         return None
     target_token = part.operations[0].getdescr()
@@ -146,7 +141,7 @@
     loop.quasi_immutable_deps = {}
     if part.quasi_immutable_deps:
         loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
-    while part.operations[-1].getopnum() == rop.LABEL:
+    if part.operations[-1].getopnum() == rop.LABEL:
         inliner = Inliner(inputargs, jumpargs)
         part.quasi_immutable_deps = None
         part.operations = [part.operations[-1]] + \
@@ -160,13 +155,15 @@
         jumpargs = part.operations[-1].getarglist()
 
         try:
-            optimize_trace(metainterp_sd, part, enable_opts)
+            optimize_trace(metainterp_sd, part, enable_opts,
+                           start_state=start_state)
         except InvalidLoop:
             return None
 
         loop.operations = loop.operations[:-1] + part.operations
         if part.quasi_immutable_deps:
             loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
+    assert part.operations[-1].getopnum() != rop.LABEL
 
     if not loop.quasi_immutable_deps:
         loop.quasi_immutable_deps = None
@@ -186,7 +183,7 @@
 
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,
-                    partial_trace, resumekey):
+                    partial_trace, resumekey, start_state):
     """Try to compile a new procedure by closing the current history back
     to the first operation.
     """
@@ -211,18 +208,19 @@
     orignial_label = label.clone()
     assert label.getopnum() == rop.LABEL
     try:
-        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts)
+        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
+                       start_state=start_state)
     except InvalidLoop:
         # Fall back on jumping to preamble
         target_token = label.getdescr()
         assert isinstance(target_token, TargetToken)
-        assert target_token.exported_state
         part.operations = [orignial_label] + \
                           [ResOperation(rop.JUMP, inputargs[:],
                                         None, descr=loop_jitcell_token)]
         try:
-            optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
-                           inline_short_preamble=False)
+            optimize_trace(metainterp_sd, part,
+                           jitdriver_sd.warmstate.enable_opts,
+                           inline_short_preamble=False, start_state=start_state)
         except InvalidLoop:
             return None
     assert part.operations[-1].getopnum() != rop.LABEL
@@ -791,7 +789,7 @@
     else:
         inline_short_preamble = True
     try:
-        optimize_trace(metainterp_sd, new_trace, state.enable_opts, inline_short_preamble)
+        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts, inline_short_preamble)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
@@ -807,7 +805,7 @@
         record_loop_or_bridge(metainterp_sd, new_trace)
         return target_token
     else:
-        metainterp.retrace_needed(new_trace)
+        metainterp.retrace_needed(new_trace, state)
         return None
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -617,7 +617,6 @@
         self.original_jitcell_token = None
 
         self.virtual_state = None
-        self.exported_state = None
         self.short_preamble = None
 
     def repr_of_descr(self):
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -178,13 +178,6 @@
             fail_args = ''
         return s_offset + res + op.getopname() + '(' + args + ')' + fail_args
 
-    def _log_inputarg_setup_ops(self, op):
-        target_token = op.getdescr()
-        if isinstance(target_token, TargetToken):
-            if target_token.exported_state:
-                for op in target_token.exported_state.inputarg_setup_ops:
-                    debug_print('    ' + self.repr_of_resop(op))
-
     def _log_operations(self, inputargs, operations, ops_offset):
         if not have_debug_prints():
             return
@@ -194,10 +187,10 @@
             args = ", ".join([self.repr_of_arg(arg) for arg in inputargs])
             debug_print('[' + args + ']')
         for i in range(len(operations)):
-            op = operations[i]
+            #op = operations[i]
             debug_print(self.repr_of_resop(operations[i], ops_offset))
-            if op.getopnum() == rop.LABEL:
-                self._log_inputarg_setup_ops(op)
+            #if op.getopnum() == rop.LABEL:
+            #    self._log_inputarg_setup_ops(op)
         if ops_offset and None in ops_offset:
             offset = ops_offset[None]
             debug_print("+%d: --end of the loop--" % offset)
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -47,7 +47,8 @@
 
     return optimizations, unroll
 
-def optimize_trace(metainterp_sd, loop, enable_opts, inline_short_preamble=True):
+def optimize_trace(metainterp_sd, loop, enable_opts,
+                   inline_short_preamble=True, start_state=None):
     """Optimize loop.operations to remove internal overheadish operations.
     """
 
@@ -57,7 +58,8 @@
                                                           loop.operations)
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
-            optimize_unroll(metainterp_sd, loop, optimizations, inline_short_preamble)
+            return optimize_unroll(metainterp_sd, loop, optimizations,
+                                   inline_short_preamble, start_state)
         else:
             optimizer = Optimizer(metainterp_sd, loop, optimizations)
             optimizer.propagate_all_forward()
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -42,8 +42,9 @@
             oldop = self.pure_operations.get(args, None)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 assert oldop.getopnum() == op.getopnum()
-                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
+                self.optimizer.make_equal_to(op.result,
+                                             self.getvalue(oldop.result),
+                                             True)
                 return
             else:
                 self.pure_operations[args] = op
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -32,6 +32,7 @@
                   if op.getopnum()==rop.LABEL]
         prv = 0
         last_label = []
+        state = None
         for nxt in labels + [len(loop.operations)]:
             assert prv != nxt
             operations = last_label + loop.operations[prv:nxt]
@@ -44,7 +45,7 @@
             part.operations = operations
 
             self.add_guard_future_condition(part)
-            self._do_optimize_loop(part, None)
+            state = self._do_optimize_loop(part, None, state)
             if part.operations[-1].getopnum() == rop.LABEL:
                 last_label = [part.operations.pop()]
             else:
@@ -496,7 +497,7 @@
 
 class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel):
 
-    def _do_optimize_loop(self, loop, call_pure_results):
+    def _do_optimize_loop(self, loop, call_pure_results, state):
         from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
         from rpython.jit.metainterp.optimizeopt.util import args_dict
         from rpython.jit.metainterp.optimizeopt.pure import OptPure
@@ -504,7 +505,7 @@
         self.loop = loop
         loop.call_pure_results = args_dict()
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True)
+        return optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True, state)
 
     def test_optimizer_renaming_boxes1(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -390,7 +390,7 @@
         assert equaloplists(optimized.operations,
                             expected.operations, False, remap, text_right)
 
-    def _do_optimize_loop(self, loop, call_pure_results):
+    def _do_optimize_loop(self, loop, call_pure_results, start_state=None):
         from rpython.jit.metainterp.optimizeopt import optimize_trace
         from rpython.jit.metainterp.optimizeopt.util import args_dict
 
@@ -405,7 +405,8 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        optimize_trace(metainterp_sd, loop, self.enable_opts)
+        return optimize_trace(metainterp_sd, loop, self.enable_opts,
+                              start_state=start_state)
 
     def unroll_and_optimize(self, loop, call_pure_results=None):
         self.add_guard_future_condition(loop)
@@ -425,7 +426,7 @@
         preamble.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(token))] + \
                               operations +  \
                               [ResOperation(rop.LABEL, jump_args, None, descr=token)]
-        self._do_optimize_loop(preamble, call_pure_results)
+        start_state = self._do_optimize_loop(preamble, call_pure_results)
 
         assert preamble.operations[-1].getopnum() == rop.LABEL
 
@@ -439,7 +440,7 @@
         assert loop.operations[0].getopnum() == rop.LABEL
         loop.inputargs = loop.operations[0].getarglist()
 
-        self._do_optimize_loop(loop, call_pure_results)
+        self._do_optimize_loop(loop, call_pure_results, start_state)
         extra_same_as = []
         while loop.operations[0].getopnum() != rop.LABEL:
             extra_same_as.append(loop.operations[0])
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -15,10 +15,11 @@
 
 # FIXME: Introduce some VirtualOptimizer super class instead
 
-def optimize_unroll(metainterp_sd, loop, optimizations, inline_short_preamble=True):
+def optimize_unroll(metainterp_sd, loop, optimizations,
+                    inline_short_preamble=True, start_state=None):
     opt = UnrollOptimizer(metainterp_sd, loop, optimizations)
     opt.inline_short_preamble = inline_short_preamble
-    opt.propagate_all_forward()
+    return opt.propagate_all_forward(start_state)
 
 
 class UnrollableOptimizer(Optimizer):
@@ -69,7 +70,7 @@
         prev = self.fix_snapshot(jump_args, snapshot.prev)
         return Snapshot(prev, new_snapshot_args)
 
-    def propagate_all_forward(self):
+    def propagate_all_forward(self, starting_state):
         loop = self.optimizer.loop
         self.optimizer.clear_newoperations()
 
@@ -94,7 +95,7 @@
         else:
             jumpop = None
 
-        self.import_state(start_label)
+        self.import_state(start_label, starting_state)
         self.optimizer.propagate_all_forward(clear=False)
 
         if not jumpop:
@@ -147,8 +148,9 @@
         KillHugeIntBounds(self.optimizer).apply()
 
         loop.operations = self.optimizer.get_newoperations()
-        self.export_state(stop_label)
+        final_state = self.export_state(stop_label)
         loop.operations.append(stop_label)
+        return final_state
 
     def jump_to_start_label(self, start_label, stop_label):
         if not start_label or not stop_label:
@@ -202,10 +204,9 @@
                 box = op.result
                 exported_values[box] = self.optimizer.getvalue(box)
 
-        target_token.exported_state = ExportedState(short_boxes, inputarg_setup_ops,
-                                                    exported_values)
+        return ExportedState(short_boxes, inputarg_setup_ops, exported_values)
 
-    def import_state(self, targetop):
+    def import_state(self, targetop, exported_state):
         if not targetop: # Trace did not start with a label
             self.inputargs = self.optimizer.loop.inputargs
             self.short = None
@@ -215,7 +216,6 @@
         self.inputargs = targetop.getarglist()
         target_token = targetop.getdescr()
         assert isinstance(target_token, TargetToken)
-        exported_state = target_token.exported_state
         if not exported_state:
             # No state exported, construct one without virtuals
             self.short = None
@@ -413,7 +413,6 @@
             if op.result:
                 op.result.forget_value()
         target_token.short_preamble = self.short
-        target_token.exported_state = None
 
     def ensure_short_op_emitted(self, op, optimizer, seen):
         if op is None:
@@ -561,7 +560,9 @@
 
             try:
                 # NB: the short_preamble ends with a jump
-                self._inline_short_preamble(target.short_preamble, inliner, patchguardop, target.assumed_classes)
+                self._inline_short_preamble(target.short_preamble, inliner,
+                                            patchguardop,
+                                            target.assumed_classes)
             except InvalidLoop:
                 #debug_print("Inlining failed unexpectedly",
                 #            "jumping to preamble instead")
@@ -572,7 +573,8 @@
         debug_stop('jit-log-virtualstate')
         return False
 
-    def _inline_short_preamble(self, short_preamble, inliner, patchguardop, assumed_classes):
+    def _inline_short_preamble(self, short_preamble, inliner, patchguardop,
+                               assumed_classes):
         i = 1
         # XXX this is intentiontal :-(. short_preamble can change during the
         # loop in some cases
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -585,13 +585,13 @@
 
 
 class ShortBoxes(object):
-    def __init__(self, optimizer, surviving_boxes, availible_boxes=None):
+    def __init__(self, optimizer, surviving_boxes, available_boxes=None):
         self.potential_ops = {}
         self.alternatives = {}
         self.synthetic = {}
         self.rename = {}
         self.optimizer = optimizer
-        self.availible_boxes = availible_boxes
+        self.available_boxes = available_boxes
         self.assumed_classes = {}
 
         if surviving_boxes is not None:
@@ -663,7 +663,7 @@
             return
         if box in self.short_boxes_in_production:
             raise BoxNotProducable
-        if self.availible_boxes is not None and box not in self.availible_boxes:
+        if self.available_boxes is not None and box not in self.available_boxes:
             raise BoxNotProducable
         self.short_boxes_in_production[box] = None
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -964,9 +964,40 @@
         assembler_call = False
         if warmrunnerstate.inlining:
             if warmrunnerstate.can_inline_callable(greenboxes):
+                # We've found a potentially inlinable function; now we need to
+                # see if it's already on the stack. In other words: are we about
+                # to enter recursion? If so, we don't want to inline the
+                # recursion, which would be equivalent to unrolling a while
+                # loop.
                 portal_code = targetjitdriver_sd.mainjitcode
-                return self.metainterp.perform_call(portal_code, allboxes,
-                                                    greenkey=greenboxes)
+                count = 0
+                for f in self.metainterp.framestack:
+                    if f.jitcode is not portal_code:
+                        continue
+                    gk = f.greenkey
+                    if gk is None:
+                        continue
+                    assert len(gk) == len(greenboxes)
+                    i = 0
+                    for i in range(len(gk)):
+                        if not gk[i].same_constant(greenboxes[i]):
+                            break
+                    else:
+                        count += 1
+                memmgr = self.metainterp.staticdata.warmrunnerdesc.memory_manager
+                if count >= memmgr.max_unroll_recursion:
+                    # This function is recursive and has exceeded the
+                    # maximum number of unrollings we allow. We want to stop
+                    # inlining it further and to make sure that, if it
+                    # hasn't happened already, the function is traced
+                    # separately as soon as possible.
+                    if have_debug_prints():
+                        loc = targetjitdriver_sd.warmstate.get_location_str(greenboxes)

From noreply at buildbot.pypy.org  Sat Dec 13 23:59:21 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 13 Dec 2014 23:59:21 +0100 (CET)
Subject: [pypy-commit] creflect default: test and fix for functions
	returning structs
Message-ID: <20141213225921.397DD1D22EA@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r200:5e92a110871a
Date: 2014-12-13 22:59 +0000
http://bitbucket.org/cffi/creflect/changeset/5e92a110871a/

Log:	test and fix for functions returning structs

diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -160,10 +160,11 @@
     void *llargs[nargs];
     char *llbuffer;
     size_t lloffset;
-    int i, returns_void;
+    int i, returns_void, returns_spec;
 
     llbuffer = alloca(zfs->zfs_size_args);
     returns_void = (zfs->zfs_ret->ct_flags & CT_VOID);
+    returns_spec = (zfs->zfs_ret->ct_flags & (CT_VOID | CT_STRUCT | CT_UNION));
     lloffset = returns_void ? 0 : zfs->zfs_ret->ct_size;
 
     for (i = 0; i < nargs; i++) {
@@ -208,9 +209,21 @@
         ((_crx_trampoline1_fn)zfs->zfs_trampl)(func, llargs, llbuffer);
     }
 
-    if (returns_void) {
-        Py_INCREF(Py_None);
-        return Py_None;
+    if (returns_spec) {
+        if (returns_void) {
+            Py_INCREF(Py_None);
+            return Py_None;
+        }
+        Py_ssize_t dataoffset = offsetof(CDataObject_own_nolength, alignment);
+        Py_ssize_t datasize = zfs->zfs_ret->ct_size;
+        CDataObject *cd;
+
+        cd = allocate_owning_object(dataoffset + datasize, zfs->zfs_ret);
+        if (cd == NULL)
+            return NULL;
+        cd->c_data = ((char *)cd) + dataoffset;
+        memcpy(cd->c_data, llbuffer, datasize);
+        return (PyObject *)cd;
     }
     return convert_to_object(llbuffer, zfs->zfs_ret);
 }
diff --git a/zeffir/test/struct.crx b/zeffir/test/struct.crx
--- a/zeffir/test/struct.crx
+++ b/zeffir/test/struct.crx
@@ -9,6 +9,13 @@
     int baz[];
 } myarrstr_t;
 
+mystruct_t swap(mystruct_t x) {
+    long v = x.b;
+    x.b = x.a;
+    x.a = v;
+    return x;
+}
+
 
 // CREFLECT: start
 
@@ -22,4 +29,6 @@
     int baz[];
 } myarrstr_t;
 
+mystruct_t swap(mystruct_t);
+
 // CREFLECT: end
diff --git a/zeffir/test/test_struct.py b/zeffir/test/test_struct.py
--- a/zeffir/test/test_struct.py
+++ b/zeffir/test/test_struct.py
@@ -54,3 +54,15 @@
     s.b += 1
     assert s.b == 0x600112234
     assert ffi.addressof(s)[0] == s
+    assert ffi.typeof(s) == ffi.typeof("mystruct_t")
+
+def test_func_with_struct_arg_and_result():
+    ffi, lib = support.compile_and_open('struct')
+    s1 = ffi.new("mystruct_t", [-5, 0x600112233])
+    s2 = lib.swap(s1)
+    assert ffi.typeof(s2) == ffi.typeof("mystruct_t")
+    assert (s1.a, s1.b) == (-5, 0x600112233)
+    assert (s2.a, s2.b) == (0x112233, -5)
+    s3 = lib.swap(s2)
+    assert (s2.a, s2.b) == (0x112233, -5)
+    assert (s3.a, s3.b) == (-5, 0x112233)

From noreply at buildbot.pypy.org  Sun Dec 14 11:42:58 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Dec 2014 11:42:58 +0100 (CET)
Subject: [pypy-commit] pypy object-dtype: start fighting with object dtype
Message-ID: <20141214104258.E61E51C050D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: object-dtype
Changeset: r74908:f72d18cb824c
Date: 2014-12-14 12:42 +0200
http://bitbucket.org/pypy/pypy/changeset/f72d18cb824c/

Log:	start fighting with object dtype

diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -188,7 +188,8 @@
                     "'%T' object is not iterable", self)
 
     def descr_str(self, space):
-        return space.wrap(self.get_dtype(space).itemtype.str_format(self))
+        return space.wrap(self.get_dtype(space).itemtype.str_format(space,
+                                                                    self))
 
     def descr_format(self, space, w_spec):
         return space.format(self.item(space), w_spec)
@@ -602,9 +603,14 @@
         return W_UnicodeBox(arr, 0, arr.dtype)
 
 class W_ObjectBox(W_GenericBox):
+    descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY.OBJECT)
+
     def __init__(self, w_obj):
         self.w_obj = w_obj
 
+    def convert_to(self, space, dtype):
+        return self # XXX
+
 
 W_GenericBox.typedef = TypeDef("numpy.generic",
     __new__ = interp2app(W_GenericBox.descr__new__.im_func),
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -110,6 +110,9 @@
     def is_str(self):
         return self.num == NPY.STRING
 
+    def is_object(self):
+        return self.num == NPY.OBJECT
+
     def is_str_or_unicode(self):
         return self.num == NPY.STRING or self.num == NPY.UNICODE
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -3,8 +3,9 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.objspace.std.floatobject import float2string
 from pypy.objspace.std.complexobject import str_format
+from pypy.interpreter.baseobjspace import W_Root
 from rpython.rlib import clibffi, jit, rfloat, rcomplex
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong, \
     most_neg_value_of, LONG_BIT
 from rpython.rlib.rawstorage import (alloc_raw_storage,
@@ -14,7 +15,9 @@
                                        pack_float80, unpack_float80)
 from rpython.rlib.rstruct.nativefmttable import native_is_bigendian
 from rpython.rlib.rstruct.runpack import runpack
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref,\
+     cast_gcref_to_instance
+from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
 from rpython.tool.sourcetools import func_with_new_name
 from pypy.module.micronumpy import boxes
 from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage
@@ -344,7 +347,7 @@
     def to_builtin_type(self, space, w_item):
         return space.wrap(self.unbox(w_item))
 
-    def str_format(self, box):
+    def str_format(self, space, box):
         return "True" if self.unbox(box) else "False"
 
     @staticmethod
@@ -409,7 +412,7 @@
     def _coerce(self, space, w_item):
         return self._base_coerce(space, w_item)
 
-    def str_format(self, box):
+    def str_format(self, space, box):
         return str(self.for_computation(self.unbox(box)))
 
     @staticmethod
@@ -647,7 +650,7 @@
             return self.box(rfloat.NAN)
         return self.box(space.float_w(space.call_function(space.w_float, w_item)))
 
-    def str_format(self, box):
+    def str_format(self, space, box):
         return float2string(self.for_computation(self.unbox(box)), "g",
                             rfloat.DTSF_STR_PRECISION)
 
@@ -1038,7 +1041,7 @@
         w_obj.__init__(w_tmpobj.real, w_tmpobj.imag)
         return w_obj
 
-    def str_format(self, box):
+    def str_format(self, space, box):
         real, imag = self.for_computation(self.unbox(box))
         imag_str = str_format(imag)
         if not rfloat.isfinite(imag):
@@ -1619,9 +1622,12 @@
         BoxType = boxes.W_ComplexLongBox
         ComponentBoxType = boxes.W_FloatLongBox
 
+_all_objs_for_tests = [] # for tests
+
 class ObjectType(BaseType):
     T = lltype.Signed
-    
+    BoxType = boxes.W_ObjectBox
+
     def get_element_size(self):
         return rffi.sizeof(lltype.Signed)
 
@@ -1633,6 +1639,36 @@
     def store(self, arr, i, offset, box):
         self._write(arr.storage, i, offset, self.unbox(box))
 
+    def read(self, arr, i, offset, dtype=None):
+        return self.box(self._read(arr.storage, i, offset))
+
+    def _write(self, storage, i, offset, w_obj):
+        if we_are_translated():
+            value = rffi.cast(lltype.Signed, cast_instance_to_gcref(w_obj))
+        else:
+            value = len(_all_objs_for_tests)
+            _all_objs_for_tests.append(w_obj)
+        raw_storage_setitem_unaligned(storage, i + offset, value)
+
+    def _read(self, storage, i, offset):
+        res = raw_storage_getitem_unaligned(self.T, storage, i + offset)
+        if we_are_translated():
+            gcref = rffi.cast(llmemory.GCREF, res)
+            w_obj = cast_gcref_to_instance(W_Root, gcref)
+        else:
+            w_obj = _all_objs_for_tests[res]
+        return w_obj
+
+    def unbox(self, box):
+        assert isinstance(box, self.BoxType)
+        return box.w_obj
+
+    def box(self, w_obj):
+        return self.BoxType(w_obj)
+
+    def str_format(self, space, box):
+        return space.str_w(space.repr(self.unbox(box)))
+
 class FlexibleType(BaseType):
     def get_element_size(self):
         return rffi.sizeof(self.T)
@@ -1702,7 +1738,7 @@
             dtype = arr.dtype
         return boxes.W_StringBox(arr, i + offset, dtype)
 
-    def str_format(self, item):
+    def str_format(self, space, item):
         builder = StringBuilder()
         builder.append("'")
         builder.append(self.to_str(item))
@@ -1832,7 +1868,7 @@
         return boxes.W_VoidBox(arr, i + offset, dtype)
 
     @jit.unroll_safe
-    def str_format(self, box):
+    def str_format(self, space, box):
         assert isinstance(box, boxes.W_VoidBox)
         arr = self.readarray(box.arr, box.ofs, 0, box.dtype)
         return arr.dump_data(prefix='', suffix='')
@@ -1932,7 +1968,7 @@
         return space.newtuple(items)
 
     @jit.unroll_safe
-    def str_format(self, box):
+    def str_format(self, space, box):
         assert isinstance(box, boxes.W_VoidBox)
         pieces = ["("]
         first = True
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -398,7 +398,9 @@
         w_rhs = numpify(space, w_rhs)
         w_ldtype = _get_dtype(space, w_lhs)
         w_rdtype = _get_dtype(space, w_rhs)
-        if w_ldtype.is_str() and w_rdtype.is_str() and \
+        if w_ldtype.is_object() or w_rdtype.is_object():
+            pass
+        elif w_ldtype.is_str() and w_rdtype.is_str() and \
                 self.comparison_func:
             pass
         elif (w_ldtype.is_str() or w_rdtype.is_str()) and \

From noreply at buildbot.pypy.org  Sun Dec 14 13:44:50 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Dec 2014 13:44:50 +0100 (CET)
Subject: [pypy-commit] pypy default: Import a recent tweak from rdict to
	rordereddict.
Message-ID: <20141214124450.216801D393A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74909:b25002d820a5
Date: 2014-12-14 13:44 +0100
http://bitbucket.org/pypy/pypy/changeset/b25002d820a5/

Log:	Import a recent tweak from rdict to rordereddict. Support
	rtype_newdict() making possibly-ordered dicts.

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -669,6 +669,7 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
+DictRepr.ll_newdict = staticmethod(ll_newdict)
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -839,6 +839,7 @@
     d.num_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
+OrderedDictRepr.ll_newdict = staticmethod(ll_newdict)
 
 def ll_newdict_size(DICT, orig_length_estimate):
     length_estimate = (orig_length_estimate // 2) * 3
@@ -1012,7 +1013,13 @@
     #      (d.resize_counter - 1) // 3 = room left in d
     #  so, if num_extra == 1, we need d.resize_counter > 3
     #      if num_extra == 2, we need d.resize_counter > 6  etc.
-    jit.conditional_call(d.resize_counter <= num_extra * 3,
+    # Note however a further hack: if num_extra <= d.num_items,
+    # we avoid calling _ll_dict_resize_to here.  This is to handle
+    # the case where dict.update() actually has a lot of collisions.
+    # If num_extra is much greater than d.num_items the conditional_call
+    # will trigger anyway, which is really the goal.
+    x = num_extra - d.num_items
+    jit.conditional_call(d.resize_counter <= x * 3,
                          _ll_dict_resize_to, d, num_extra)
 
 # this is an implementation of keys(), values() and items()
diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py
--- a/rpython/rtyper/rdict.py
+++ b/rpython/rtyper/rdict.py
@@ -57,11 +57,10 @@
 
 
 def rtype_newdict(hop):
-    from rpython.rtyper.lltypesystem.rdict import ll_newdict
     hop.inputargs()    # no arguments expected
     r_dict = hop.r_result
     cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    v_result = hop.gendirectcall(r_dict.ll_newdict, cDICT)
     return v_result
 
 

From noreply at buildbot.pypy.org  Sun Dec 14 14:23:43 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Dec 2014 14:23:43 +0100 (CET)
Subject: [pypy-commit] pypy default: Add stress tests for rordereddict,
 copied from rdict.  Kill an unused function.
Message-ID: <20141214132343.B1B271C1066@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74910:b30951625595
Date: 2014-12-14 13:23 +0000
http://bitbucket.org/pypy/pypy/changeset/b30951625595/

Log:	Add stress tests for rordereddict, copied from rdict. Kill an
	unused function.

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -865,23 +865,6 @@
     pass
 
 
-def rtype_r_dict(hop):
-    r_dict = hop.r_result
-    if not r_dict.custom_eq_hash:
-        raise TyperError("r_dict() call does not return an r_dict instance")
-    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
-    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
-    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    hop.exception_cannot_occur()
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
-    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
-        hop.genop('setfield', [v_result, cname, v_eqfn])
-    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
-        hop.genop('setfield', [v_result, cname, v_hashfn])
-    return v_result
-
 # ____________________________________________________________
 #
 #  Iteration.
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -287,3 +287,126 @@
 
     def test_memoryerror_should_not_insert(self):
         py.test.skip("I don't want to edit this file on two branches")
+
+
+class TestStress:
+
+    def test_stress(self):
+        from rpython.annotator.dictdef import DictKey, DictValue
+        from rpython.annotator import model as annmodel
+        from rpython.rtyper import rint
+        from rpython.rtyper.test.test_rdict import not_really_random
+        rodct = rordereddict
+        dictrepr = rodct.OrderedDictRepr(
+                                  None, rint.signed_repr, rint.signed_repr,
+                                  DictKey(None, annmodel.SomeInteger()),
+                                  DictValue(None, annmodel.SomeInteger()))
+        dictrepr.setup()
+        l_dict = rodct.ll_newdict(dictrepr.DICT)
+        referencetable = [None] * 400
+        referencelength = 0
+        value = 0
+
+        def complete_check():
+            for n, refvalue in zip(range(len(referencetable)), referencetable):
+                try:
+                    gotvalue = rodct.ll_dict_getitem(l_dict, n)
+                except KeyError:
+                    assert refvalue is None
+                else:
+                    assert gotvalue == refvalue
+
+        for x in not_really_random():
+            n = int(x*100.0)    # 0 <= x < 400
+            op = repr(x)[-1]
+            if op <= '2' and referencetable[n] is not None:
+                rodct.ll_dict_delitem(l_dict, n)
+                referencetable[n] = None
+                referencelength -= 1
+            elif op <= '6':
+                rodct.ll_dict_setitem(l_dict, n, value)
+                if referencetable[n] is None:
+                    referencelength += 1
+                referencetable[n] = value
+                value += 1
+            else:
+                try:
+                    gotvalue = rodct.ll_dict_getitem(l_dict, n)
+                except KeyError:
+                    assert referencetable[n] is None
+                else:
+                    assert gotvalue == referencetable[n]
+            if 1.38 <= x <= 1.39:
+                complete_check()
+                print 'current dict length:', referencelength
+            assert l_dict.num_items == referencelength
+        complete_check()
+
+    def test_stress_2(self):
+        yield self.stress_combination, True,  False
+        yield self.stress_combination, False, True
+        yield self.stress_combination, False, False
+        yield self.stress_combination, True,  True
+
+    def stress_combination(self, key_can_be_none, value_can_be_none):
+        from rpython.rtyper.lltypesystem.rstr import string_repr
+        from rpython.annotator.dictdef import DictKey, DictValue
+        from rpython.annotator import model as annmodel
+        from rpython.rtyper.test.test_rdict import not_really_random
+        rodct = rordereddict
+
+        print
+        print "Testing combination with can_be_None: keys %s, values %s" % (
+            key_can_be_none, value_can_be_none)
+
+        class PseudoRTyper:
+            cache_dummy_values = {}
+        dictrepr = rodct.OrderedDictRepr(
+                       PseudoRTyper(), string_repr, string_repr,
+                       DictKey(None, annmodel.SomeString(key_can_be_none)),
+                       DictValue(None, annmodel.SomeString(value_can_be_none)))
+        dictrepr.setup()
+        print dictrepr.lowleveltype
+        #for key, value in dictrepr.DICTENTRY._adtmeths.items():
+        #    print '    %s = %s' % (key, value)
+        l_dict = rodct.ll_newdict(dictrepr.DICT)
+        referencetable = [None] * 400
+        referencelength = 0
+        values = not_really_random()
+        keytable = [string_repr.convert_const("foo%d" % n)
+                    for n in range(len(referencetable))]
+
+        def complete_check():
+            for n, refvalue in zip(range(len(referencetable)), referencetable):
+                try:
+                    gotvalue = rodct.ll_dict_getitem(l_dict, keytable[n])
+                except KeyError:
+                    assert refvalue is None
+                else:
+                    assert gotvalue == refvalue
+
+        for x in not_really_random():
+            n = int(x*100.0)    # 0 <= x < 400
+            op = repr(x)[-1]
+            if op <= '2' and referencetable[n] is not None:
+                rodct.ll_dict_delitem(l_dict, keytable[n])
+                referencetable[n] = None
+                referencelength -= 1
+            elif op <= '6':
+                ll_value = string_repr.convert_const(str(values.next()))
+                rodct.ll_dict_setitem(l_dict, keytable[n], ll_value)
+                if referencetable[n] is None:
+                    referencelength += 1
+                referencetable[n] = ll_value
+            else:
+                try:
+                    gotvalue = rodct.ll_dict_getitem(l_dict, keytable[n])
+                except KeyError:
+                    assert referencetable[n] is None
+                else:
+                    assert gotvalue == referencetable[n]
+            if 1.38 <= x <= 1.39:
+                complete_check()
+                print 'current dict length:', referencelength
+            assert l_dict.num_items == referencelength
+        complete_check()

From noreply at buildbot.pypy.org  Sun Dec 14 14:39:20 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Dec 2014 14:39:20 +0100 (CET)
Subject: [pypy-commit] pypy default: more tweaks,
 with the goal that we can say 'SomeDict = SomeOrderedDict' in
 annotation/model.py
Message-ID: <20141214133920.D37621C1066@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74911:ae2c0310c9c9
Date: 2014-12-14 14:39 +0100
http://bitbucket.org/pypy/pypy/changeset/ae2c0310c9c9/

Log:	more tweaks, with the goal that we can say 'SomeDict =
	SomeOrderedDict' in annotation/model.py

diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -693,26 +693,6 @@
     pass
 
 
-def rtype_r_dict(hop, i_force_non_null=None):
-    r_dict = hop.r_result
-    if not r_dict.custom_eq_hash:
-        raise TyperError("r_dict() call does not return an r_dict instance")
-    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
-    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
-    if i_force_non_null is not None:
-        assert i_force_non_null == 2
-        hop.inputarg(lltype.Void, arg=2)
-    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    hop.exception_cannot_occur()
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
-    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
-        hop.genop('setfield', [v_result, cname, v_eqfn])
-    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
-        hop.genop('setfield', [v_result, cname, v_hashfn])
-    return v_result
-
 # ____________________________________________________________
 #
 #  Iteration.
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -156,7 +156,7 @@
 
     def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
                  custom_eq_hash=None, force_non_null=False):
-        assert not force_non_null
+        #assert not force_non_null
         self.rtyper = rtyper
         self.finalized = False
         self.DICT = lltype.GcForwardReference()
@@ -217,7 +217,7 @@
         #dictobj = getattr(dictobj, '__self__', dictobj)
         if dictobj is None:
             return lltype.nullptr(self.DICT)
-        if not isinstance(dictobj, (dict, objectmodel.r_ordereddict)):
+        if not isinstance(dictobj, (dict, objectmodel.r_dict)):
             raise TypeError("expected a dict: %r" % (dictobj,))
         try:
             key = Constant(dictobj)
@@ -231,8 +231,8 @@
             if r_key.lowleveltype == llmemory.Address:
                 raise TypeError("No prebuilt dicts of address keys")
             r_value = self.value_repr
-            if isinstance(dictobj, objectmodel.r_ordereddict):
-                
+            if isinstance(dictobj, objectmodel.r_dict):
+
                 if self.r_rdict_eqfn.lowleveltype != lltype.Void:
                     l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq)
                     l_dict.fnkeyeq = l_fn
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -724,15 +724,16 @@
     raise TyperError("hasattr is only suported on a constant")
 
 @typer_for(annmodel.SomeOrderedDict.knowntype)
+ at typer_for(objectmodel.r_dict)
 @typer_for(objectmodel.r_ordereddict)
-def rtype_ordered_dict(hop):
-    from rpython.rtyper.lltypesystem.rordereddict import ll_newdict
-
+def rtype_dict_constructor(hop, i_force_non_null=None):
+    # 'i_force_non_null' is ignored here; if it has any effect, it
+    # has already been applied to 'hop.r_result'
     hop.exception_cannot_occur()
     r_dict = hop.r_result
     cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
-    if hasattr(r_dict, 'r_dict_eqfn'):
+    v_result = hop.gendirectcall(r_dict.ll_newdict, cDICT)
+    if r_dict.custom_eq_hash:
         v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
         v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
         if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
@@ -743,9 +744,6 @@
             hop.genop('setfield', [v_result, cname, v_hashfn])
     return v_result
 
-from rpython.rtyper.lltypesystem.rdict import rtype_r_dict
-typer_for(objectmodel.r_dict)(rtype_r_dict)
-
 # _________________________________________________________________
 # weakrefs
 

From noreply at buildbot.pypy.org  Sun Dec 14 17:23:09 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Dec 2014 17:23:09 +0100 (CET)
Subject: [pypy-commit] pypy default: Obviously,
 adding ll_prepare_dict_update() exposed a latent bug
Message-ID: <20141214162309.9B9151D288E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74912:ef301ae5ae96
Date: 2014-12-14 17:23 +0100
http://bitbucket.org/pypy/pypy/changeset/ef301ae5ae96/

Log:	Obviously, adding ll_prepare_dict_update() exposed a latent bug

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -600,10 +600,12 @@
     else:
         newitems = d.entries
     #
-    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
+    ENTRIES = lltype.typeOf(d).TO.entries.TO
+    ENTRY = ENTRIES.OF
     isrc = 0
     idst = 0
-    while isrc < len(d.entries):
+    isrclimit = d.num_used_items
+    while isrc < isrclimit:
         if d.entries.valid(isrc):
             src = d.entries[isrc]
             dst = newitems[idst]
@@ -616,9 +618,21 @@
                 dst.f_valid = True
             idst += 1
         isrc += 1
-    d.entries = newitems
     assert d.num_items == idst
     d.num_used_items = idst
+    if ((ENTRIES.must_clear_key or ENTRIES.must_clear_value) and
+            d.entries == newitems):
+        # must clear the extra entries: they may contain valid pointers
+        # which would create a temporary memory leak
+        while idst < isrclimit:
+            entry = newitems[idst]
+            if ENTRIES.must_clear_key:
+                entry.key = lltype.nullptr(ENTRY.key.TO)
+            if ENTRIES.must_clear_value:
+                entry.value = lltype.nullptr(ENTRY.value.TO)
+            idst += 1
+    else:
+        d.entries = newitems
 
     ll_dict_reindex(d, _ll_len_of_d_indexes(d))
 
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -251,6 +251,16 @@
         assert rordereddict.ll_dict_pop_default(ll_d, llstr("k"), 40) == 40
         assert rordereddict.ll_dict_pop_default(ll_d, llstr("j"), 39) == 39
 
+    def test_bug_remove_deleted_items(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        for i in range(15):
+            rordereddict.ll_dict_setitem(ll_d, llstr(chr(i)), 5)
+        for i in range(15):
+            rordereddict.ll_dict_delitem(ll_d, llstr(chr(i)))
+        rordereddict.ll_prepare_dict_update(ll_d, 7)
+        # used to get UninitializedMemoryAccess
+
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
         ll_dummy_value = llstr("dupa")

From noreply at buildbot.pypy.org  Sun Dec 14 17:41:26 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 14 Dec 2014 17:41:26 +0100 (CET)
Subject: [pypy-commit] pypy no-silent-merging-of-int-types: No longer
 silently merge different int types in RPython.
Message-ID: <20141214164126.AB6C41C1066@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: no-silent-merging-of-int-types
Changeset: r74913:bdfb89396304
Date: 2014-12-14 17:41 +0100
http://bitbucket.org/pypy/pypy/changeset/bdfb89396304/

Log:	No longer silently merge different int types in RPython.

diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -176,31 +176,35 @@
     return newfunc
 
 class __extend__(pairtype(SomeInteger, SomeInteger)):
-    # unsignedness is considered a rare and contagious disease
 
     def union((int1, int2)):
-        if int1.unsigned == int2.unsigned:
-            knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype)
+        # Rules: int1 and int2 must be in one of the following two cases:
+        #  * int1 and int2 are the same type; or
+        #  * one of them is a constant 'int'.
+        # This rule is stricter than it used to be (Dec'14).
+        #
+        t1 = int1.knowntype
+        if t1 is bool:
+            t1 = int
+        t2 = int2.knowntype
+        if t2 is bool:
+            t2 = int
+        #
+        if t1 == t2:
+            knowntype = t1
+        elif (t1 in (int, rarithmetic.r_int) and
+              t2 in (int, rarithmetic.r_int)):
+            knowntype = rarithmetic.r_int
+        elif (t1 in (int, rarithmetic.r_int) and int1.is_constant()
+                        and long(t2(int1.const)) == int1.const):
+            knowntype = t2
+        elif (t2 in (int, rarithmetic.r_int) and int2.is_constant()
+                        and long(t1(int2.const)) == int2.const):
+            knowntype = t1
         else:
-            t1 = int1.knowntype
-            if t1 is bool:
-                t1 = int
-            t2 = int2.knowntype
-            if t2 is bool:
-                t2 = int
-
-            if t2 is int:
-                if int2.nonneg == False:
-                    raise UnionError(int1, int2, "RPython cannot prove that these " + \
-                            "integers are of the same signedness")
-                knowntype = t1
-            elif t1 is int:
-                if int1.nonneg == False:
-                    raise UnionError(int1, int2, "RPython cannot prove that these " + \
-                            "integers are of the same signedness")
-                knowntype = t2
-            else:
-                raise UnionError(int1, int2)
+            raise UnionError(int1, int2, "cannot merge two integers"
+                        " of different types; explicit casts are needed."
+                        " (exception: integer constants)")
         return SomeInteger(nonneg=int1.nonneg and int2.nonneg,
                            knowntype=knowntype)
 
@@ -218,13 +222,13 @@
     inplace_truediv = truediv
 
     def sub((int1, int2)):
-        knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype)
+        knowntype = pair(int1, int2).union().knowntype
         return SomeInteger(knowntype=knowntype)
     sub.can_only_throw = []
     sub_ovf = _clone(sub, [OverflowError])
 
     def and_((int1, int2)):
-        knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype)
+        knowntype = pair(int1, int2).union().knowntype
         return SomeInteger(nonneg=int1.nonneg or int2.nonneg,
                            knowntype=knowntype)
     and_.can_only_throw = []
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -921,9 +921,7 @@
                 c = b
             return c
         a = self.RPythonAnnotator()
-        s = a.build_types(f, [r_uint, int])
-        assert s == annmodel.SomeInteger(nonneg = True, unsigned = True)
-
+        py.test.raises(annmodel.UnionError, a.build_types, f, [r_uint, int])
 
     def test_prebuilt_long_that_is_not_too_long(self):
         small_constant = 12L
@@ -4178,9 +4176,6 @@
         with py.test.raises(annmodel.UnionError) as exc:
             a.build_types(f, [int])
 
-        assert ("RPython cannot prove that these integers are of the "
-                "same signedness" in exc.value.msg)
-
     def test_unionerror_instance(self):
         class A(object): pass
         class B(object): pass
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -440,6 +440,7 @@
                 self.max_delta = 0.125 * env.get_total_memory()
 
             gc_increment_step = env.read_uint_from_env('PYPY_GC_INCREMENT_STEP')
+            gc_increment_step = intmask(gc_increment_step)
             if gc_increment_step > 0:
                 self.gc_increment_step = gc_increment_step
             else:
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -476,7 +476,7 @@
     i = i & MASK
     ENTRY = lltype.typeOf(d.entries).TO.OF
     entry = d.entries[i]
-    if not d.entries.everused(i):
+    if not d.entries.everused(intmask(i)):
         # a new entry that was never used before
         ll_assert(not valid, "valid but not everused")
         rc = d.resize_counter - 3
@@ -586,10 +586,10 @@
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
     direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
-    mask = len(entries) - 1
-    i = r_uint(hash & mask)
+    mask = r_uint(len(entries) - 1)
+    i = r_uint(hash) & mask
     # do the first try before any looping
-    if entries.valid(i):
+    if entries.valid(intmask(i)):
         checkingkey = entries[i].key
         if direct_compare and checkingkey == key:
             return i   # found the entry
@@ -599,13 +599,14 @@
             found = d.keyeq(checkingkey, key)
             if d.paranoia:
                 if (entries != d.entries or
-                    not entries.valid(i) or entries[i].key != checkingkey):
+                         not entries.valid(intmask(i)) or
+                         entries[i].key != checkingkey):
                     # the compare did major nasty stuff to the dict: start over
                     return ll_dict_lookup(d, key, hash)
             if found:
                 return i   # found the entry
         freeslot = -1
-    elif entries.everused(i):
+    elif entries.everused(intmask(i)):
         freeslot = intmask(i)
     else:
         return i | HIGHEST_BIT # pristine entry -- lookup failed
@@ -619,11 +620,11 @@
         i = i & mask
         # keep 'i' as a signed number here, to consistently pass signed
         # arguments to the small helper methods.
-        if not entries.everused(i):
+        if not entries.everused(intmask(i)):
             if freeslot == -1:
                 freeslot = intmask(i)
             return r_uint(freeslot) | HIGHEST_BIT
-        elif entries.valid(i):
+        elif entries.valid(intmask(i)):
             checkingkey = entries[i].key
             if direct_compare and checkingkey == key:
                 return i
@@ -633,7 +634,8 @@
                 found = d.keyeq(checkingkey, key)
                 if d.paranoia:
                     if (entries != d.entries or
-                        not entries.valid(i) or entries[i].key != checkingkey):
+                            not entries.valid(intmask(i)) or
+                            entries[i].key != checkingkey):
                         # the compare did major nasty stuff to the dict:
                         # start over
                         return ll_dict_lookup(d, key, hash)
@@ -648,10 +650,10 @@
     # key is new, and the dictionary doesn't contain deleted entries.
     # It only finds the next free slot for the given hash.
     entries = d.entries
-    mask = len(entries) - 1
-    i = r_uint(hash & mask)
+    mask = r_uint(len(entries) - 1)
+    i = r_uint(hash) & mask
     perturb = r_uint(hash)
-    while entries.everused(i):
+    while entries.everused(intmask(i)):
         i = (i << 2) + i + perturb + 1
         i = i & mask
         perturb >>= PERTURB_SHIFT
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -717,8 +717,8 @@
     INDEXES = _ll_ptr_to_array_of(T)
     entries = d.entries
     indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
-    mask = len(indexes) - 1
-    i = r_uint(hash & mask)
+    mask = r_uint(len(indexes) - 1)
+    i = r_uint(hash) & mask
     # do the first try before any looping
     ENTRIES = lltype.typeOf(entries).TO
     direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
@@ -802,8 +802,8 @@
     # It only finds the next free slot for the given hash.
     INDEXES = _ll_ptr_to_array_of(T)
     indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
-    mask = len(indexes) - 1
-    i = r_uint(hash & mask)
+    mask = r_uint(len(indexes) - 1)
+    i = r_uint(hash) & mask
     perturb = r_uint(hash)
     while rffi.cast(lltype.Signed, indexes[i]) != 0:
         i = (i << 2) + i + perturb + 1

From noreply at buildbot.pypy.org  Sun Dec 14 18:05:11 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Dec 2014 18:05:11 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: Implement the correct handling of
 get_virtual_ip in the presence of
Message-ID: <20141214170511.7593F1C1066@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: vmprof
Changeset: r74914:3c0a8b430ed5
Date: 2014-12-14 19:04 +0200
http://bitbucket.org/pypy/pypy/changeset/3c0a8b430ed5/

Log:	Implement the correct handling of get_virtual_ip in the presence of
	virtualizables. Added a hack that will give you the vable_token with
	a very minimal test

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -104,6 +104,8 @@
 
 def get_virtual_ip(gc_frame):
     frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
+    if jit._is_virtualizable_token_set(frame):
+        return rffi.cast(rffi.VOIDP, 0)
     virtual_ip = do_get_virtual_ip(frame)
     return rffi.cast(rffi.VOIDP, virtual_ip)
 get_virtual_ip.c_name = 'pypy_vmprof_get_virtual_ip'
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -2,7 +2,7 @@
 from rpython.tool.udir import udir
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
-from rpython.rlib.jit import promote
+from rpython.rlib.jit import promote, _get_virtualizable_token
 from rpython.rlib import jit_hooks, rposix
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference
@@ -28,12 +28,15 @@
         # - floats neg and abs
         # - llexternal with macro=True
 
-        class Frame(object):
+        class BasicFrame(object):
             _virtualizable_ = ['i']
 
             def __init__(self, i):
                 self.i = i
 
+        class Frame(BasicFrame):
+            pass
+
         eci = ExternalCompilationInfo(post_include_bits=['''
 #define pypy_my_fabs(x)  fabs(x)
 '''])
@@ -59,6 +62,7 @@
             while frame.i > 3:
                 jitdriver.can_enter_jit(frame=frame, total=total, j=j)
                 jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+                _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
                     frame.i -= 2
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -280,6 +280,36 @@
 _we_are_jitted = CDefinedIntSymbolic('0 /* we are not jitted here */',
                                      default=0)
 
+def _get_virtualizable_token(frame):
+    """ An obscure API to get vable token.
+    Used by _vmprof
+    """
+    from rpython.rtyper.lltypesystem import lltype, llmemory
+    
+    return lltype.nullptr(llmemory.GCREF.TO)
+
+class GetVirtualizableTokenEntry(ExtRegistryEntry):
+    _about_ = _get_virtualizable_token
+
+    def compute_result_annotation(self, s_arg):
+        from rpython.rtyper.llannotation import SomePtr
+        from rpython.rtyper.lltypesystem import llmemory
+        return SomePtr(llmemory.GCREF)
+
+    def specialize_call(self, hop):
+        from rpython.rtyper.lltypesystem import lltype, llmemory
+
+        hop.exception_cannot_occur()
+        T = hop.args_r[0].lowleveltype.TO
+        v = hop.inputarg(hop.args_r[0], arg=0)
+        c_super = hop.inputconst(lltype.Void, 'super')
+        while not hasattr(T, 'vable_token'):
+            v = hop.genop('getfield', [v, c_super], resulttype=T.super)
+            T = T.super
+        c_vable_token = hop.inputconst(lltype.Void, 'vable_token')
+        return hop.genop('getfield', [v, c_vable_token],
+                         resulttype=llmemory.GCREF)
+
 class Entry(ExtRegistryEntry):
     _about_ = we_are_jitted
 

From noreply at buildbot.pypy.org  Sun Dec 14 18:13:37 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Dec 2014 18:13:37 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: typo
Message-ID: <20141214171337.2B3E21D364D@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: vmprof
Changeset: r74915:e6e97f3bddbc
Date: 2014-12-14 19:13 +0200
http://bitbucket.org/pypy/pypy/changeset/e6e97f3bddbc/

Log:	typo

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -104,7 +104,7 @@
 
 def get_virtual_ip(gc_frame):
     frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
-    if jit._is_virtualizable_token_set(frame):
+    if jit._get_virtualizable_token(frame):
         return rffi.cast(rffi.VOIDP, 0)
     virtual_ip = do_get_virtual_ip(frame)
     return rffi.cast(rffi.VOIDP, virtual_ip)

From noreply at buildbot.pypy.org  Sun Dec 14 18:36:21 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Dec 2014 18:36:21 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: disable those warnings
Message-ID: <20141214173621.357DF1D38FA@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: vmprof
Changeset: r74916:d64556a99c18
Date: 2014-12-14 19:36 +0200
http://bitbucket.org/pypy/pypy/changeset/d64556a99c18/

Log:	disable those warnings

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -100,6 +100,10 @@
         self._vmprof_virtual_ip = _vmprof.get_next_virtual_IP()
         self._vmprof_registered = 0
 
+# avoid rtyper warnings
+PyCode._vmprof_virtual_ip = 0
+PyCode._vmprof_registered = 0
+
 
 
 def get_virtual_ip(gc_frame):

From noreply at buildbot.pypy.org  Sun Dec 14 19:05:41 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Dec 2014 19:05:41 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hack unrolling enough to pass the
	first actual test
Message-ID: <20141214180541.9E7641C3688@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74917:2713ea9e60e2
Date: 2014-12-14 20:05 +0200
http://bitbucket.org/pypy/pypy/changeset/2713ea9e60e2/

Log:	hack unrolling enough to pass the first actual test

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -150,6 +150,7 @@
 
         loop.operations = self.optimizer.get_newoperations()
         final_state = self.export_state(stop_label)
+        final_state.dump(self.optimizer.metainterp_sd)
         loop.operations.append(stop_label)
         return final_state
 
@@ -386,7 +387,7 @@
         for i in range(len(short)):
             op = short[i]
             if op.is_guard():
-                op = op.clone()
+                #op = op.clone() XXXX
                 op.setfailargs(None)
                 op.setdescr(None) # will be set to a proper descr when the preamble is used
                 short[i] = op
@@ -453,13 +454,13 @@
         if op.is_guard():
             op.setdescr(None) # will be set to a proper descr when the preamble is used
 
-        if guards_needed and self.short_boxes.has_producer(op.result):
-            value_guards = self.getvalue(op.result).make_guards(op.result)
+        if guards_needed and self.short_boxes.has_producer(op):
+            value_guards = self.getvalue(op).make_guards(op)
         else:
             value_guards = []
 
         self.short.append(op)
-        self.short_seen[op.result] = None
+        self.short_seen[op] = None
         if emit and self.short_inliner:
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
@@ -577,21 +578,22 @@
         debug_stop('jit-log-virtualstate')
         return False
 
-    def _inline_short_preamble(self, short_preamble, inliner, patchguardop,
+    def _inline_short_preamble(self, short_preamble, memo, patchguardop,
                                assumed_classes):
         i = 1
         # XXX this is intentiontal :-(. short_preamble can change during the
         # loop in some cases
         while i < len(short_preamble):
             shop = short_preamble[i]
-            newop = inliner.inline_op(shop)
+            newop = shop.clone(memo)
+            newop.is_source_op = True
             if newop.is_guard():
                 if not patchguardop:
                     raise InvalidLoop("would like to have short preamble, but it has a guard and there's no guard_future_condition")
-                descr = patchguardop.getdescr().clone_if_mutable()
+                descr = patchguardop.getdescr().clone_if_mutable(memo)
                 newop.setdescr(descr)
             self.optimizer.send_extra_operation(newop)
-            if shop.result in assumed_classes:
+            if shop in assumed_classes:
                 classbox = self.getvalue(newop.result).get_constant_class(self.optimizer.cpu)
                 if not classbox or not classbox.same_constant(assumed_classes[shop.result]):
                     raise InvalidLoop('The class of an opaque pointer before the jump ' +
@@ -617,13 +619,13 @@
         self.inputarg_setup_ops = inputarg_setup_ops
         self.exported_values = exported_values
 
-    def dump(self):
+    def dump(self, metainterp_sd):
         debug_start("jit-exported-state")
-        logops = LogOperations()
+        logops = LogOperations(metainterp_sd, False)
         debug_print("   inputarg setup")
-        logops._log_operations(self.inputarg_setup_ops)
+        logops._log_operations([], self.inputarg_setup_ops)
         debug_print("   short boxes")
         self.short_boxes.dump(logops)
         debug_print("   exported values")
-        self.exported_values.dump(logops)
+        #self.exported_values.dump(logops)
         debug_stop("jit-exported-state")
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -591,32 +591,40 @@
         self.potential_ops = {}
         self.alternatives = {}
         self.synthetic = {}
-        self.memo = Memo()
         self.optimizer = optimizer
         self.available_boxes = available_boxes
         self.assumed_classes = {}
 
-        if surviving_boxes is not None:
-            for box in surviving_boxes:
-                self.potential_ops[box] = None
-            optimizer.produce_potential_short_preamble_ops(self)
+        assert surviving_boxes is not None
+        for box in surviving_boxes:
+            self.potential_ops[box] = None
+        optimizer.produce_potential_short_preamble_ops(self)
 
-            self.short_boxes = {}
-            self.short_boxes_in_production = {}
+        self.short_boxes = {}
+        self.short_boxes_in_production = {}
 
-            for op in self.potential_ops.keys():
-                try:
-                    self.produce_short_preamble_op(op)
-                except BoxNotProducable:
-                    pass
+        for op in self.potential_ops.keys():
+            try:
+                self.produce_short_preamble_op(op)
+            except BoxNotProducable:
+                pass
 
-            self.short_boxes_in_production = None # Not needed anymore
-        else:
-            self.short_boxes = {}
+        self.short_boxes_in_production = None # Not needed anymore
+        #else:
+        #    self.short_boxes = {}
+
+    def dump(self, logops):
+        for op, resop in self.short_boxes.iteritems():
+            if isinstance(op, AbstractInputArg):
+                assert resop is None
+                debug_print(logops.repr_of_arg(op))
+            else:
+                debug_print("%s:%s"% (logops.repr_of_resop(op),
+                                      logops.repr_of_resop(resop)))
 
     def prioritized_alternatives(self, box):
         if box not in self.alternatives:
-            return [box]
+            return [self.potential_ops[box]]
         alts = self.alternatives[box]
         hi, lo = 0, len(alts) - 1
         while hi < lo:
@@ -631,11 +639,13 @@
         return alts
 
     def add_to_short(self, box, op):
-        if op:
-            op = op.clone(self.memo)
-            op.is_source_op = True
+        #if op:
+        #    op = op.clone(self.memo)
+        #    op.is_source_op = True
         if box in self.short_boxes:
+            xxx
             if op is None:
+                xxxx
                 oldop = self.short_boxes[box].clone()
                 oldres = oldop.result
                 newbox = oldop.result = oldres.clonebox()
@@ -643,6 +653,7 @@
                 self.short_boxes[box] = None
                 self.short_boxes[newbox] = oldop
             else:
+                xxxx
                 newop = op.clone()
                 newbox = newop.result = op.result.clonebox()
                 self.short_boxes[newop.result] = newop
@@ -655,11 +666,11 @@
     def produce_short_preamble_op(self, op):
         if op in self.short_boxes:
             return
-        if isinstance(op, Const) or isinstance(op, AbstractInputArg):
+        if isinstance(op, Const):
             return
         if op in self.short_boxes_in_production:
             raise BoxNotProducable
-        if self.availible_boxes is not None and op not in self.available_boxes:
+        if self.available_boxes is not None and op not in self.available_boxes:
             raise BoxNotProducable
         self.short_boxes_in_production[op] = None
 
@@ -689,7 +700,7 @@
                 if classbox:
                     self.assumed_classes[op] = classbox
         if op not in self.potential_ops:
-            self.potential_ops[op] = None
+            self.potential_ops[op] = op
         else:
             if op not in self.alternatives:
                 self.alternatives[op] = [op]

From noreply at buildbot.pypy.org  Sun Dec 14 19:32:00 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 14 Dec 2014 19:32:00 +0100 (CET)
Subject: [pypy-commit] pypy vendor/stdlib: Upgrade 2.7 stdlib to 2.7.9
Message-ID: <20141214183200.C064D1C3D55@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: vendor/stdlib
Changeset: r74918:4ecd94a66a87
Date: 2014-12-14 10:31 -0800
http://bitbucket.org/pypy/pypy/changeset/4ecd94a66a87/

Log:	Upgrade 2.7 stdlib to 2.7.9

diff too long, truncating to 2000 out of 24016 lines

diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py
--- a/lib-python/2.7/CGIHTTPServer.py
+++ b/lib-python/2.7/CGIHTTPServer.py
@@ -106,16 +106,16 @@
     def run_cgi(self):
         """Execute a CGI script."""
         dir, rest = self.cgi_info
-
-        i = rest.find('/')
+        path = dir + '/' + rest
+        i = path.find('/', len(dir)+1)
         while i >= 0:
-            nextdir = rest[:i]
-            nextrest = rest[i+1:]
+            nextdir = path[:i]
+            nextrest = path[i+1:]
 
             scriptdir = self.translate_path(nextdir)
             if os.path.isdir(scriptdir):
                 dir, rest = nextdir, nextrest
-                i = rest.find('/')
+                i = path.find('/', len(dir)+1)
             else:
                 break
 
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -56,7 +56,7 @@
    >>> C = Cookie.SmartCookie()
 
 [Note: Long-time users of Cookie.py will remember using
-Cookie.Cookie() to create an Cookie object.  Although deprecated, it
+Cookie.Cookie() to create a Cookie object.  Although deprecated, it
 is still supported by the code.  See the Backward Compatibility notes
 for more information.]
 
@@ -426,6 +426,8 @@
                    "version" : "Version",
                    }
 
+    _flags = {'secure', 'httponly'}
+
     def __init__(self):
         # Set defaults
         self.key = self.value = self.coded_value = None
@@ -529,9 +531,11 @@
 _LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
+    r"\s*"                        # Optional whitespace at start of cookie
     r"(?P<key>"                   # Start of group 'key'
     ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
+    r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
     r"(?P<val>"                   # Start of group 'val'
     r'"(?:[^\\"]|\\.)*"'            # Any doublequoted string
@@ -540,7 +544,9 @@
     r"|"                            # or
     ""+ _LegalCharsPatt +"*"        # Any word or empty string
     r")"                          # End of group 'val'
-    r"\s*;?"                      # Probably ending in a semi-colon
+    r")?"                         # End of optional value group
+    r"\s*"                        # Any number of spaces.
+    r"(\s+|;|$)"                  # Ending either at space, semicolon, or EOS.
     )
 
 
@@ -585,8 +591,12 @@
 
     def __setitem__(self, key, value):
         """Dictionary style assignment."""
-        rval, cval = self.value_encode(value)
-        self.__set(key, rval, cval)
+        if isinstance(value, Morsel):
+            # allow assignment of constructed Morsels (e.g. for pickling)
+            dict.__setitem__(self, key, value)
+        else:
+            rval, cval = self.value_encode(value)
+            self.__set(key, rval, cval)
     # end __setitem__
 
     def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"):
@@ -641,7 +651,7 @@
 
         while 0 <= i < n:
             # Start looking for a cookie
-            match = patt.search(str, i)
+            match = patt.match(str, i)
             if not match: break          # No more cookies
 
             K,V = match.group("key"), match.group("val")
@@ -656,8 +666,12 @@
                     M[ K[1:] ] = V
             elif K.lower() in Morsel._reserved:
                 if M:
-                    M[ K ] = _unquote(V)
-            else:
+                    if V is None:
+                        if K.lower() in Morsel._flags:
+                            M[K] = True
+                    else:
+                        M[K] = _unquote(V)
+            elif V is not None:
                 rval, cval = self.value_decode(V)
                 self.__set(K, rval, cval)
                 M = self[K]
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -416,8 +416,12 @@
         self.socket = socket.socket(self.address_family,
                                     self.socket_type)
         if bind_and_activate:
-            self.server_bind()
-            self.server_activate()
+            try:
+                self.server_bind()
+                self.server_activate()
+            except:
+                self.server_close()
+                raise
 
     def server_bind(self):
         """Called by constructor to bind the socket.
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -143,7 +143,7 @@
     methods except for __contains__, __iter__ and __len__.
 
     To override the comparisons (presumably for speed, as the
-    semantics are fixed), all you have to do is redefine __le__ and
+    semantics are fixed), redefine __le__ and __ge__,
     then the other operations will automatically follow suit.
     """
 
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -1089,7 +1089,14 @@
         # parse all the remaining options into the namespace
         # store any unrecognized options on the object, so that the top
         # level parser can decide what to do with them
-        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
+
+        # In case this subparser defines new defaults, we parse them
+        # in a new namespace object and then update the original
+        # namespace for the relevant parts.
+        subnamespace, arg_strings = parser.parse_known_args(arg_strings, None)
+        for key, value in vars(subnamespace).items():
+            setattr(namespace, key, value)
+
         if arg_strings:
             vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
             getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py
--- a/lib-python/2.7/asynchat.py
+++ b/lib-python/2.7/asynchat.py
@@ -46,12 +46,17 @@
 you - by calling your self.found_terminator() method.
 """
 
+import asyncore
+import errno
 import socket
-import asyncore
 from collections import deque
 from sys import py3kwarning
 from warnings import filterwarnings, catch_warnings
 
+_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS,
+                       errno.EWOULDBLOCK)
+
+
 class async_chat (asyncore.dispatcher):
     """This is an abstract class.  You must derive from this class, and add
     the two methods collect_incoming_data() and found_terminator()"""
@@ -109,6 +114,8 @@
         try:
             data = self.recv (self.ac_in_buffer_size)
         except socket.error, why:
+            if why.args[0] in _BLOCKING_IO_ERRORS:
+                return
             self.handle_error()
             return
 
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,6 +10,7 @@
 
 #----------------------------------------------------------------------
 
+ at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -1719,12 +1719,12 @@
     def __repr__(self):
         r = []
         for cookie in self: r.append(repr(cookie))
-        return "<%s[%s]>" % (self.__class__, ", ".join(r))
+        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
 
     def __str__(self):
         r = []
         for cookie in self: r.append(str(cookie))
-        return "<%s[%s]>" % (self.__class__, ", ".join(r))
+        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
 
 
 # derives from IOError for backwards-compatibility with Python 2.4.0
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,6 +7,8 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
+LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
+large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -188,5 +190,11 @@
             mth = WINFUNCTYPE(None)(42, "name", (), None)
             self.assertEqual(bool(mth), True)
 
+    def test_pointer_type_name(self):
+        self.assertTrue(POINTER(LargeNamedType))
+
+    def test_pointer_type_str_name(self):
+        self.assertTrue(POINTER(large_string))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py
--- a/lib-python/2.7/ctypes/test/test_python_api.py
+++ b/lib-python/2.7/ctypes/test/test_python_api.py
@@ -40,8 +40,8 @@
     # This test is unreliable, because it is possible that code in
     # unittest changes the refcount of the '42' integer.  So, it
     # is disabled by default.
-    @requires("refcount")
     def test_PyInt_Long(self):
+        requires("refcount")
         ref42 = grc(42)
         pythonapi.PyInt_FromLong.restype = py_object
         self.assertEqual(pythonapi.PyInt_FromLong(42), 42)
diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py
--- a/lib-python/2.7/ctypes/test/test_win32.py
+++ b/lib-python/2.7/ctypes/test/test_win32.py
@@ -38,8 +38,11 @@
 
 @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
 class FunctionCallTestCase(unittest.TestCase):
-    @requires("SEH")
+    @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC")
+    @unittest.skipIf(sys.executable.endswith('_d.exe'),
+                     "SEH not enabled in debug builds")
     def test_SEH(self):
+        requires("SEH")
         # Call functions with invalid arguments, and make sure
         # that access violations are trapped and raise an
         # exception.
@@ -87,9 +90,29 @@
 
         dll = CDLL(_ctypes_test.__file__)
 
-        pt = POINT(10, 10)
-        rect = RECT(0, 0, 20, 20)
-        self.assertEqual(1, dll.PointInRect(byref(rect), pt))
+        pt = POINT(15, 25)
+        left = c_long.in_dll(dll, 'left')
+        top = c_long.in_dll(dll, 'top')
+        right = c_long.in_dll(dll, 'right')
+        bottom = c_long.in_dll(dll, 'bottom')
+        rect = RECT(left, top, right, bottom)
+        PointInRect = dll.PointInRect
+        PointInRect.argtypes = [POINTER(RECT), POINT]
+        self.assertEqual(1, PointInRect(byref(rect), pt))
+
+        ReturnRect = dll.ReturnRect
+        ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT,
+                               POINTER(RECT), POINT, RECT]
+        ReturnRect.restype = RECT
+        for i in range(4):
+            ret = ReturnRect(i, rect, pointer(rect), pt, rect,
+                         byref(rect), pt, rect)
+            # the c function will check and modify ret if something is
+            # passed in improperly
+            self.assertEqual(ret.left, left.value)
+            self.assertEqual(ret.right, right.value)
+            self.assertEqual(ret.top, top.value)
+            self.assertEqual(ret.bottom, bottom.value)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py
--- a/lib-python/2.7/decimal.py
+++ b/lib-python/2.7/decimal.py
@@ -136,7 +136,6 @@
 
 __version__ = '1.70'    # Highest version of the spec this complies with
 
-import copy as _copy
 import math as _math
 import numbers as _numbers
 
@@ -3665,6 +3664,8 @@
         if self._is_special:
             sign = _format_sign(self._sign, spec)
             body = str(self.copy_abs())
+            if spec['type'] == '%':
+                body += '%'
             return _format_align(sign, body, spec)
 
         # a type of None defaults to 'g' or 'G', depending on context
@@ -6033,7 +6034,10 @@
         format_dict['decimal_point'] = '.'
 
     # record whether return type should be str or unicode
-    format_dict['unicode'] = isinstance(format_spec, unicode)
+    try:
+        format_dict['unicode'] = isinstance(format_spec, unicode)
+    except NameError:
+        format_dict['unicode'] = False
 
     return format_dict
 
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.8"
+__version__ = "2.7.9"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -235,7 +235,7 @@
         # Python's library directory must be appended to library_dirs
         # See Issues: #1600860, #4366
         if (sysconfig.get_config_var('Py_ENABLE_SHARED')):
-            if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
+            if not sysconfig.python_build:
                 # building third party extensions
                 self.library_dirs.append(sysconfig.get_config_var('LIBDIR'))
             else:
diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py
--- a/lib-python/2.7/distutils/command/upload.py
+++ b/lib-python/2.7/distutils/command/upload.py
@@ -136,8 +136,8 @@
 
         # Build up the MIME payload for the POST data
         boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
-        sep_boundary = '\n--' + boundary
-        end_boundary = sep_boundary + '--'
+        sep_boundary = '\r\n--' + boundary
+        end_boundary = sep_boundary + '--\r\n'
         body = StringIO.StringIO()
         for key, value in data.items():
             # handle multiple entries for the same name
@@ -151,14 +151,13 @@
                     fn = ""
 
                 body.write(sep_boundary)
-                body.write('\nContent-Disposition: form-data; name="%s"'%key)
+                body.write('\r\nContent-Disposition: form-data; name="%s"' % key)
                 body.write(fn)
-                body.write("\n\n")
+                body.write("\r\n\r\n")
                 body.write(value)
                 if value and value[-1] == '\r':
                     body.write('\n')  # write an extra newline (lurve Macs)
         body.write(end_boundary)
-        body.write("\n")
         body = body.getvalue()
 
         self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO)
diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py
--- a/lib-python/2.7/distutils/file_util.py
+++ b/lib-python/2.7/distutils/file_util.py
@@ -85,7 +85,8 @@
     (os.symlink) instead of copying: set it to "hard" or "sym"; if it is
     None (the default), files are copied.  Don't set 'link' on systems that
     don't support it: 'copy_file()' doesn't check if hard or symbolic
-    linking is available.
+    linking is available. If hardlink fails, falls back to
+    _copy_file_contents().
 
     Under Mac OS, uses the native file copy function in macostools; on
     other systems, uses '_copy_file_contents()' to copy file contents.
@@ -137,24 +138,31 @@
     # (Unix only, of course, but that's the caller's responsibility)
     if link == 'hard':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
-            os.link(src, dst)
+            try:
+                os.link(src, dst)
+                return (dst, 1)
+            except OSError:
+                # If hard linking fails, fall back on copying file
+                # (some special filesystems don't support hard linking
+                #  even under Unix, see issue #8876).
+                pass
     elif link == 'sym':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
             os.symlink(src, dst)
+            return (dst, 1)
 
     # Otherwise (non-Mac, not linking), copy the file contents and
     # (optionally) copy the times and mode.
-    else:
-        _copy_file_contents(src, dst)
-        if preserve_mode or preserve_times:
-            st = os.stat(src)
+    _copy_file_contents(src, dst)
+    if preserve_mode or preserve_times:
+        st = os.stat(src)
 
-            # According to David Ascher <da at ski.org>, utime() should be done
-            # before chmod() (at least under NT).
-            if preserve_times:
-                os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
-            if preserve_mode:
-                os.chmod(dst, S_IMODE(st[ST_MODE]))
+        # According to David Ascher <da at ski.org>, utime() should be done
+        # before chmod() (at least under NT).
+        if preserve_times:
+            os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
+        if preserve_mode:
+            os.chmod(dst, S_IMODE(st[ST_MODE]))
 
     return (dst, 1)
 
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -165,7 +165,8 @@
             # version and build tools may not support the same set
             # of CPU architectures for universal builds.
             global _config_vars
-            if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''):
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
                 import _osx_support
                 _osx_support.customize_compiler(_config_vars)
                 _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
--- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py
+++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
@@ -25,6 +25,7 @@
 """
 
 class BuildRpmTestCase(support.TempdirManager,
+                       support.EnvironGuard,
                        support.LoggingSilencer,
                        unittest.TestCase):
 
@@ -50,6 +51,7 @@
     def test_quiet(self):
         # let's create a package
         tmp_dir = self.mkdtemp()
+        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
         os.mkdir(pkg_dir)
         self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
@@ -92,6 +94,7 @@
     def test_no_optimize_flag(self):
         # let's create a package that brakes bdist_rpm
         tmp_dir = self.mkdtemp()
+        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
         os.mkdir(pkg_dir)
         self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py
--- a/lib-python/2.7/distutils/tests/test_dist.py
+++ b/lib-python/2.7/distutils/tests/test_dist.py
@@ -11,7 +11,7 @@
 from distutils.dist import Distribution, fix_help_options
 from distutils.cmd import Command
 import distutils.dist
-from test.test_support import TESTFN, captured_stdout, run_unittest
+from test.test_support import TESTFN, captured_stdout, run_unittest, unlink
 from distutils.tests import support
 
 
@@ -64,6 +64,7 @@
         with open(TESTFN, "w") as f:
             f.write("[global]\n")
             f.write("command_packages = foo.bar, splat")
+        self.addCleanup(unlink, TESTFN)
 
         files = [TESTFN]
         sys.argv.append("build")
diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py
--- a/lib-python/2.7/distutils/tests/test_file_util.py
+++ b/lib-python/2.7/distutils/tests/test_file_util.py
@@ -8,6 +8,11 @@
 from distutils.tests import support
 from test.test_support import run_unittest
 
+
+requires_os_link = unittest.skipUnless(hasattr(os, "link"),
+                                       "test requires os.link()")
+
+
 class FileUtilTestCase(support.TempdirManager, unittest.TestCase):
 
     def _log(self, msg, *args):
@@ -74,6 +79,44 @@
         copy_file(foo, dst_dir)
         self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo')))
 
+    @requires_os_link
+    def test_copy_file_hard_link(self):
+        with open(self.source, 'w') as f:
+            f.write('some content')
+        st = os.stat(self.source)
+        copy_file(self.source, self.target, link='hard')
+        st2 = os.stat(self.source)
+        st3 = os.stat(self.target)
+        self.assertTrue(os.path.samestat(st, st2), (st, st2))
+        self.assertTrue(os.path.samestat(st2, st3), (st2, st3))
+        with open(self.source, 'r') as f:
+            self.assertEqual(f.read(), 'some content')
+
+    @requires_os_link
+    def test_copy_file_hard_link_failure(self):
+        # If hard linking fails, copy_file() falls back on copying file
+        # (some special filesystems don't support hard linking even under
+        #  Unix, see issue #8876).
+        with open(self.source, 'w') as f:
+            f.write('some content')
+        st = os.stat(self.source)
+        def _os_link(*args):
+            raise OSError(0, "linking unsupported")
+        old_link = os.link
+        os.link = _os_link
+        try:
+            copy_file(self.source, self.target, link='hard')
+        finally:
+            os.link = old_link
+        st2 = os.stat(self.source)
+        st3 = os.stat(self.target)
+        self.assertTrue(os.path.samestat(st, st2), (st, st2))
+        self.assertFalse(os.path.samestat(st2, st3), (st2, st3))
+        for fn in (self.source, self.target):
+            with open(fn, 'r') as f:
+                self.assertEqual(f.read(), 'some content')
+
+
 def test_suite():
     return unittest.makeSuite(FileUtilTestCase)
 
diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py
--- a/lib-python/2.7/distutils/tests/test_sysconfig.py
+++ b/lib-python/2.7/distutils/tests/test_sysconfig.py
@@ -3,6 +3,9 @@
 import test
 import unittest
 import shutil
+import subprocess
+import sys
+import textwrap
 
 from distutils import sysconfig
 from distutils.tests import support
@@ -99,6 +102,24 @@
         self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED'))
         self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC'))
 
+    def test_customize_compiler_before_get_config_vars(self):
+        # Issue #21923: test that a Distribution compiler
+        # instance can be called without an explicit call to
+        # get_config_vars().
+        with open(TESTFN, 'w') as f:
+            f.writelines(textwrap.dedent('''\
+                from distutils.core import Distribution
+                config = Distribution().get_command_obj('config')
+                # try_compile may pass or it may fail if no compiler
+                # is found but it should not raise an exception.
+                rc = config.try_compile('int x;')
+                '''))
+        p = subprocess.Popen([str(sys.executable), TESTFN],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                universal_newlines=True)
+        outs, errs = p.communicate()
+        self.assertEqual(0, p.returncode, "Subprocess failed: " + outs)
 
 
 def test_suite():
diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py
--- a/lib-python/2.7/distutils/tests/test_upload.py
+++ b/lib-python/2.7/distutils/tests/test_upload.py
@@ -119,7 +119,7 @@
         # what did we send ?
         self.assertIn('d&#233;d&#233;', self.last_open.req.data)
         headers = dict(self.last_open.req.headers)
-        self.assertEqual(headers['Content-length'], '2085')
+        self.assertEqual(headers['Content-length'], '2159')
         self.assertTrue(headers['Content-type'].startswith('multipart/form-data'))
         self.assertEqual(self.last_open.req.get_method(), 'POST')
         self.assertEqual(self.last_open.req.get_full_url(),
diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py
--- a/lib-python/2.7/doctest.py
+++ b/lib-python/2.7/doctest.py
@@ -216,7 +216,7 @@
                 # get_data() opens files as 'rb', so one must do the equivalent
                 # conversion as universal newlines would do.
                 return file_contents.replace(os.linesep, '\n'), filename
-    with open(filename) as f:
+    with open(filename, 'U') as f:
         return f.read(), filename
 
 # Use sys.stdout encoding for ouput.
diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py
--- a/lib-python/2.7/email/feedparser.py
+++ b/lib-python/2.7/email/feedparser.py
@@ -49,8 +49,8 @@
     simple abstraction -- it parses until EOF closes the current message.
     """
     def __init__(self):
-        # The last partial line pushed into this object.
-        self._partial = ''
+        # Chunks of the last partial line pushed into this object.
+        self._partial = []
         # The list of full, pushed lines, in reverse order
         self._lines = []
         # The stack of false-EOF checking predicates.
@@ -66,8 +66,8 @@
 
     def close(self):
         # Don't forget any trailing partial line.
-        self._lines.append(self._partial)
-        self._partial = ''
+        self.pushlines(''.join(self._partial).splitlines(True))
+        self._partial = []
         self._closed = True
 
     def readline(self):
@@ -95,8 +95,29 @@
 
     def push(self, data):
         """Push some new data into this object."""
-        # Handle any previous leftovers
-        data, self._partial = self._partial + data, ''
+        # Crack into lines, but preserve the linesep characters on the end of each
+        parts = data.splitlines(True)
+
+        if not parts or not parts[0].endswith(('\n', '\r')):
+            # No new complete lines, so just accumulate partials
+            self._partial += parts
+            return
+
+        if self._partial:
+            # If there are previous leftovers, complete them now
+            self._partial.append(parts[0])
+            parts[0:1] = ''.join(self._partial).splitlines(True)
+            del self._partial[:]
+
+        # If the last element of the list does not end in a newline, then treat
+        # it as a partial line.  We only check for '\n' here because a line
+        # ending with '\r' might be a line that was split in the middle of a
+        # '\r\n' sequence (see bugs 1555570 and 1721862).
+        if not parts[-1].endswith('\n'):
+            self._partial = [parts.pop()]
+        self.pushlines(parts)
+
+    def pushlines(self, lines):
         # Crack into lines, but preserve the newlines on the end of each
         parts = NLCRE_crack.split(data)
         # The *ahem* interesting behaviour of re.split when supplied grouping
diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py
--- a/lib-python/2.7/email/mime/nonmultipart.py
+++ b/lib-python/2.7/email/mime/nonmultipart.py
@@ -12,7 +12,7 @@
 
 

 class MIMENonMultipart(MIMEBase):
-    """Base class for MIME multipart/* type messages."""
+    """Base class for MIME non-multipart type messages."""
 
     def attach(self, payload):
         # The public API prohibits attaching multiple subparts to MIMEBase
diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py
--- a/lib-python/2.7/email/test/test_email.py
+++ b/lib-python/2.7/email/test/test_email.py
@@ -11,6 +11,7 @@
 import warnings
 import textwrap
 from cStringIO import StringIO
+from random import choice
 
 import email
 
@@ -2578,16 +2579,64 @@
             bsf.push(il)
             nt += n
             n1 = 0
-            while True:
-                ol = bsf.readline()
-                if ol == NeedMoreData:
-                    break
+            for ol in iter(bsf.readline, NeedMoreData):
                 om.append(ol)
                 n1 += 1
             self.assertEqual(n, n1)
         self.assertEqual(len(om), nt)
         self.assertEqual(''.join([il for il, n in imt]), ''.join(om))
 
+    def test_push_random(self):
+        from email.feedparser import BufferedSubFile, NeedMoreData
+
+        n = 10000
+        chunksize = 5
+        chars = 'abcd \t\r\n'
+
+        s = ''.join(choice(chars) for i in range(n)) + '\n'
+        target = s.splitlines(True)
+
+        bsf = BufferedSubFile()
+        lines = []
+        for i in range(0, len(s), chunksize):
+            chunk = s[i:i+chunksize]
+            bsf.push(chunk)
+            lines.extend(iter(bsf.readline, NeedMoreData))
+        self.assertEqual(lines, target)
+
+
+class TestFeedParsers(TestEmailBase):
+
+    def parse(self, chunks):
+        from email.feedparser import FeedParser
+        feedparser = FeedParser()
+        for chunk in chunks:
+            feedparser.feed(chunk)
+        return feedparser.close()
+
+    def test_newlines(self):
+        m = self.parse(['a:\nb:\rc:\r\nd:\n'])
+        self.assertEqual(m.keys(), ['a', 'b', 'c', 'd'])
+        m = self.parse(['a:\nb:\rc:\r\nd:'])
+        self.assertEqual(m.keys(), ['a', 'b', 'c', 'd'])
+        m = self.parse(['a:\rb', 'c:\n'])
+        self.assertEqual(m.keys(), ['a', 'bc'])
+        m = self.parse(['a:\r', 'b:\n'])
+        self.assertEqual(m.keys(), ['a', 'b'])
+        m = self.parse(['a:\r', '\nb:\n'])
+        self.assertEqual(m.keys(), ['a', 'b'])
+
+    def test_long_lines(self):
+        # Expected peak memory use on 32-bit platform: 4*N*M bytes.
+        M, N = 1000, 20000
+        m = self.parse(['a:b\n\n'] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', 'b')])
+        self.assertEqual(m.get_payload(), 'x'*M*N)
+        m = self.parse(['a:b\r\r'] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', 'b')])
+        self.assertEqual(m.get_payload(), 'x'*M*N)
+        m = self.parse(['a:\r', 'b: '] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)])
 
 
 class TestParsers(TestEmailBase):
@@ -3180,7 +3229,6 @@
         self.assertEqual(res, '=?iso-8859-2?q?abc?=')
         self.assertIsInstance(res, str)
 
-
 # Test RFC 2231 header parameters (en/de)coding
 class TestRFC2231(TestEmailBase):
     def test_get_param(self):
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python2
+from __future__ import print_function
+
+import os
+import os.path
+import pkgutil
+import shutil
+import sys
+import tempfile
+
+
+__all__ = ["version", "bootstrap"]
+
+
+_SETUPTOOLS_VERSION = "7.0"
+
+_PIP_VERSION = "1.5.6"
+
+# pip currently requires ssl support, so we try to provide a nicer
+# error message when that is missing (http://bugs.python.org/issue19744)
+_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION))
+try:
+    import ssl
+except ImportError:
+    ssl = None
+
+    def _require_ssl_for_pip():
+        raise RuntimeError(_MISSING_SSL_MESSAGE)
+else:
+    def _require_ssl_for_pip():
+        pass
+
+_PROJECTS = [
+    ("setuptools", _SETUPTOOLS_VERSION),
+    ("pip", _PIP_VERSION),
+]
+
+
+def _run_pip(args, additional_paths=None):
+    # Add our bundled software to the sys.path so we can import it
+    if additional_paths is not None:
+        sys.path = additional_paths + sys.path
+
+    # Install the bundled software
+    import pip
+    pip.main(args)
+
+
+def version():
+    """
+    Returns a string specifying the bundled version of pip.
+    """
+    return _PIP_VERSION
+
+
+def _disable_pip_configuration_settings():
+    # We deliberately ignore all pip environment variables
+    # when invoking pip
+    # See http://bugs.python.org/issue19734 for details
+    keys_to_remove = [k for k in os.environ if k.startswith("PIP_")]
+    for k in keys_to_remove:
+        del os.environ[k]
+    # We also ignore the settings in the default pip configuration file
+    # See http://bugs.python.org/issue20053 for details
+    os.environ['PIP_CONFIG_FILE'] = os.devnull
+
+
+def bootstrap(root=None, upgrade=False, user=False,
+              altinstall=False, default_pip=True,
+              verbosity=0):
+    """
+    Bootstrap pip into the current Python installation (or the given root
+    directory).
+
+    Note that calling this function will alter both sys.path and os.environ.
+    """
+    if altinstall and default_pip:
+        raise ValueError("Cannot use altinstall and default_pip together")
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # By default, installing pip and setuptools installs all of the
+    # following scripts (X.Y == running Python version):
+    #
+    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
+    #
+    # pip 1.5+ allows ensurepip to request that some of those be left out
+    if altinstall:
+        # omit pip, pipX and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
+    elif not default_pip:
+        # omit pip and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "install"
+
+    tmpdir = tempfile.mkdtemp()
+    try:
+        # Put our bundled wheels into a temporary directory and construct the
+        # additional paths that need added to sys.path
+        additional_paths = []
+        for project, version in _PROJECTS:
+            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
+            whl = pkgutil.get_data(
+                "ensurepip",
+                "_bundled/{}".format(wheel_name),
+            )
+            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+                fp.write(whl)
+
+            additional_paths.append(os.path.join(tmpdir, wheel_name))
+
+        # Construct the arguments to be passed to the pip command
+        args = ["install", "--no-index", "--find-links", tmpdir]
+        if root:
+            args += ["--root", root]
+        if upgrade:
+            args += ["--upgrade"]
+        if user:
+            args += ["--user"]
+        if verbosity:
+            args += ["-" + "v" * verbosity]
+
+        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def _uninstall_helper(verbosity=0):
+    """Helper to support a clean default uninstall process on Windows
+
+    Note that calling this function may alter os.environ.
+    """
+    # Nothing to do if pip was never installed, or has been removed
+    try:
+        import pip
+    except ImportError:
+        return
+
+    # If the pip version doesn't match the bundled one, leave it alone
+    if pip.__version__ != _PIP_VERSION:
+        msg = ("ensurepip will only uninstall a matching version "
+               "({!r} installed, {!r} bundled)")
+        print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr)
+        return
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # Construct the arguments to be passed to the pip command
+    args = ["uninstall", "-y"]
+    if verbosity:
+        args += ["-" + "v" * verbosity]
+
+    _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+
+
+def _main(argv=None):
+    if ssl is None:
+        print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE),
+              file=sys.stderr)
+        return
+
+    import argparse
+    parser = argparse.ArgumentParser(prog="python -m ensurepip")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(version()),
+        help="Show the version of pip that is bundled with this Python.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+    parser.add_argument(
+        "-U", "--upgrade",
+        action="store_true",
+        default=False,
+        help="Upgrade pip and dependencies, even if already installed.",
+    )
+    parser.add_argument(
+        "--user",
+        action="store_true",
+        default=False,
+        help="Install using the user scheme.",
+    )
+    parser.add_argument(
+        "--root",
+        default=None,
+        help="Install everything relative to this alternate root directory.",
+    )
+    parser.add_argument(
+        "--altinstall",
+        action="store_true",
+        default=False,
+        help=("Make an alternate install, installing only the X.Y versioned"
+              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"),
+    )
+    parser.add_argument(
+        "--default-pip",
+        action="store_true",
+        default=True,
+        dest="default_pip",
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-default-pip",
+        action="store_false",
+        dest="default_pip",
+        help=("Make a non default install, installing only the X and X.Y "
+              "versioned scripts."),
+    )
+
+    args = parser.parse_args(argv)
+
+    bootstrap(
+        root=args.root,
+        upgrade=args.upgrade,
+        user=args.user,
+        verbosity=args.verbosity,
+        altinstall=args.altinstall,
+        default_pip=args.default_pip,
+    )
diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/__main__.py
@@ -0,0 +1,4 @@
+import ensurepip
+
+if __name__ == "__main__":
+    ensurepip._main()
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/_uninstall.py
@@ -0,0 +1,30 @@
+"""Basic pip uninstallation support, helper for the Windows uninstaller"""
+
+import argparse
+import ensurepip
+
+
+def _main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(ensurepip.version()),
+        help="Show the version of pip this will attempt to uninstall.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+
+    args = parser.parse_args(argv)
+
+    ensurepip._uninstall_helper(verbosity=args.verbosity)
+
+
+if __name__ == "__main__":
+    _main()
diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py
--- a/lib-python/2.7/glob.py
+++ b/lib-python/2.7/glob.py
@@ -35,11 +35,16 @@
     patterns.
 
     """
+    dirname, basename = os.path.split(pathname)
     if not has_magic(pathname):
-        if os.path.lexists(pathname):
-            yield pathname
+        if basename:
+            if os.path.lexists(pathname):
+                yield pathname
+        else:
+            # Patterns ending with a slash should match only directories
+            if os.path.isdir(dirname):
+                yield pathname
         return
-    dirname, basename = os.path.split(pathname)
     if not dirname:
         for name in glob1(os.curdir, basename):
             yield name
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -164,9 +164,16 @@
     def _write_gzip_header(self):
         self.fileobj.write('\037\213')             # magic header
         self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
+        try:
+            # RFC 1952 requires the FNAME field to be Latin-1. Do not
+            # include filenames that cannot be represented that way.
+            fname = os.path.basename(self.name)
+            if not isinstance(fname, str):
+                fname = fname.encode('latin-1')
+            if fname.endswith('.gz'):
+                fname = fname[:-3]
+        except UnicodeEncodeError:
+            fname = ''
         flags = 0
         if fname:
             flags = FNAME
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -15,8 +15,9 @@
 
 md5(), sha1(), sha224(), sha256(), sha384(), and sha512()
 
-More algorithms may be available on your platform but the above are
-guaranteed to exist.
+More algorithms may be available on your platform but the above are guaranteed
+to exist.  See the algorithms_guaranteed and algorithms_available attributes
+to find out what algorithm names can be passed to new().
 
 NOTE: If you want the adler32 or crc32 hash functions they are available in
 the zlib module.
@@ -58,9 +59,14 @@
 # always available algorithm is added.
 __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
+algorithms_guaranteed = set(__always_supported)
+algorithms_available = set(__always_supported)
+
 algorithms = __always_supported
 
-__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac')
+__all__ = __always_supported + ('new', 'algorithms_guaranteed',
+                                'algorithms_available', 'algorithms',
+                                'pbkdf2_hmac')
 
 
 def __get_builtin_constructor(name):
@@ -128,6 +134,8 @@
     import _hashlib
     new = __hash_new
     __get_hash = __get_openssl_constructor
+    algorithms_available = algorithms_available.union(
+        _hashlib.openssl_md_meth_names)
 except ImportError:
     new = __py_new
     __get_hash = __get_builtin_constructor
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -215,6 +215,10 @@
 # maximal line length when calling readline().
 _MAXLINE = 65536
 
+# maximum amount of headers accepted
+_MAXHEADERS = 100
+
+
 class HTTPMessage(mimetools.Message):
 
     def addheader(self, key, value):
@@ -271,6 +275,8 @@
         elif self.seekable:
             tell = self.fp.tell
         while True:
+            if len(hlist) > _MAXHEADERS:
+                raise HTTPException("got more than %d headers" % _MAXHEADERS)
             if tell:
                 try:
                     startofline = tell()
@@ -1181,21 +1187,29 @@
 
         def __init__(self, host, port=None, key_file=None, cert_file=None,
                      strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-                     source_address=None):
+                     source_address=None, context=None):
             HTTPConnection.__init__(self, host, port, strict, timeout,
                                     source_address)
             self.key_file = key_file
             self.cert_file = cert_file
+            if context is None:
+                context = ssl._create_default_https_context()
+            if key_file or cert_file:
+                context.load_cert_chain(cert_file, key_file)
+            self._context = context
 
         def connect(self):
             "Connect to a host on a given (SSL) port."
 
-            sock = self._create_connection((self.host, self.port),
-                                          self.timeout, self.source_address)
+            HTTPConnection.connect(self)
+
             if self._tunnel_host:
-                self.sock = sock
-                self._tunnel()
-            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                server_hostname = self._tunnel_host
+            else:
+                server_hostname = self.host
+
+            self.sock = self._context.wrap_socket(self.sock,
+                                                  server_hostname=server_hostname)
 
     __all__.append("HTTPSConnection")
 
@@ -1210,14 +1224,15 @@
         _connection_class = HTTPSConnection
 
         def __init__(self, host='', port=None, key_file=None, cert_file=None,
-                     strict=None):
+                     strict=None, context=None):
             # provide a default host, pass the X509 cert info
 
             # urf. compensate for bad input.
             if port == 0:
                 port = None
             self._setup(self._connection_class(host, port, key_file,
-                                               cert_file, strict))
+                                               cert_file, strict,
+                                               context=context))
 
             # we never actually use these for anything, but we keep them
             # here for compatibility with post-1.5.2 CVS.
diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py
--- a/lib-python/2.7/idlelib/Bindings.py
+++ b/lib-python/2.7/idlelib/Bindings.py
@@ -75,7 +75,8 @@
    ('!_Auto-open Stack Viewer', '<<toggle-jit-stack-viewer>>'),
    ]),
  ('options', [
-   ('_Configure IDLE...', '<<open-config-dialog>>'),
+   ('Configure _IDLE', '<<open-config-dialog>>'),
+   ('Configure _Extensions', '<<open-config-extensions-dialog>>'),
    None,
    ]),
  ('help', [
diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py
--- a/lib-python/2.7/idlelib/CallTipWindow.py
+++ b/lib-python/2.7/idlelib/CallTipWindow.py
@@ -2,9 +2,8 @@
 
 After ToolTip.py, which uses ideas gleaned from PySol
 Used by the CallTips IDLE extension.
-
 """
-from Tkinter import *
+from Tkinter import Toplevel, Label, LEFT, SOLID, TclError
 
 HIDE_VIRTUAL_EVENT_NAME = "<<calltipwindow-hide>>"
 HIDE_SEQUENCES = ("<Key-Escape>", "<FocusOut>")
@@ -133,35 +132,28 @@
         return bool(self.tipwindow)
 
 
-def _calltip_window(parent):
-    root = Tk()
-    root.title("Test calltips")
-    width, height, x, y = list(map(int, re.split('[x+]', parent.geometry())))
-    root.geometry("+%d+%d"%(x, y + 150))
+def _calltip_window(parent):  # htest #
+    from Tkinter import Toplevel, Text, LEFT, BOTH
 
-    class MyEditWin: # comparenceptually an editor_window
-        def __init__(self):
-            text = self.text = Text(root)
-            text.pack(side=LEFT, fill=BOTH, expand=1)
-            text.insert("insert", "string.split")
-            root.update()
-            self.calltip = CallTip(text)
+    top = Toplevel(parent)
+    top.title("Test calltips")
+    top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200,
+                  parent.winfo_rooty() + 150))
+    text = Text(top)
+    text.pack(side=LEFT, fill=BOTH, expand=1)
+    text.insert("insert", "string.split")
+    top.update()
+    calltip = CallTip(text)
 
-            text.event_add("<<calltip-show>>", "(")
-            text.event_add("<<calltip-hide>>", ")")
-            text.bind("<<calltip-show>>", self.calltip_show)
-            text.bind("<<calltip-hide>>", self.calltip_hide)
-
-            text.focus_set()
-            root.mainloop()
-
-        def calltip_show(self, event):
-            self.calltip.showtip("Hello world", "insert", "end")
-
-        def calltip_hide(self, event):
-            self.calltip.hidetip()
-
-    editwin = MyEditWin()
+    def calltip_show(event):
+        calltip.showtip("(s=Hello world)", "insert", "end")
+    def calltip_hide(event):
+        calltip.hidetip()
+    text.event_add("<<calltip-show>>", "(")
+    text.event_add("<<calltip-hide>>", ")")
+    text.bind("<<calltip-show>>", calltip_show)
+    text.bind("<<calltip-hide>>", calltip_hide)
+    text.focus_set()
 
 if __name__=='__main__':
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py
--- a/lib-python/2.7/idlelib/ClassBrowser.py
+++ b/lib-python/2.7/idlelib/ClassBrowser.py
@@ -19,6 +19,9 @@
 from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas
 from idlelib.configHandler import idleConf
 
+file_open = None  # Method...Item and Class...Item use this.
+# Normally PyShell.flist.open, but there is no PyShell.flist for htest.
+
 class ClassBrowser:
 
     def __init__(self, flist, name, path, _htest=False):
@@ -27,6 +30,9 @@
         """
         _htest - bool, change box when location running htest.
         """
+        global file_open
+        if not _htest:
+            file_open = PyShell.flist.open
         self.name = name
         self.file = os.path.join(path[0], self.name + ".py")
         self._htest = _htest
@@ -101,7 +107,7 @@
             return []
         try:
             dict = pyclbr.readmodule_ex(name, [dir] + sys.path)
-        except ImportError, msg:
+        except ImportError:
             return []
         items = []
         self.classes = {}
@@ -170,7 +176,7 @@
     def OnDoubleClick(self):
         if not os.path.exists(self.file):
             return
-        edit = PyShell.flist.open(self.file)
+        edit = file_open(self.file)
         if hasattr(self.cl, 'lineno'):
             lineno = self.cl.lineno
             edit.gotoline(lineno)
@@ -206,7 +212,7 @@
     def OnDoubleClick(self):
         if not os.path.exists(self.file):
             return
-        edit = PyShell.flist.open(self.file)
+        edit = file_open(self.file)
         edit.gotoline(self.cl.methods[self.name])
 
 def _class_browser(parent): #Wrapper for htest
@@ -221,8 +227,9 @@
     dir, file = os.path.split(file)
     name = os.path.splitext(file)[0]
     flist = PyShell.PyShellFileList(parent)
+    global file_open
+    file_open = flist.open
     ClassBrowser(flist, name, [dir], _htest=True)
-    parent.mainloop()
 
 if __name__ == "__main__":
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py
--- a/lib-python/2.7/idlelib/ColorDelegator.py
+++ b/lib-python/2.7/idlelib/ColorDelegator.py
@@ -2,7 +2,6 @@
 import re
 import keyword
 import __builtin__
-from Tkinter import *
 from idlelib.Delegator import Delegator
 from idlelib.configHandler import idleConf
 
@@ -34,7 +33,6 @@
 
 prog = re.compile(make_pat(), re.S)
 idprog = re.compile(r"\s+(\w+)", re.S)
-asprog = re.compile(r".*?\b(as)\b")
 
 class ColorDelegator(Delegator):
 
@@ -42,7 +40,6 @@
         Delegator.__init__(self)
         self.prog = prog
         self.idprog = idprog
-        self.asprog = asprog
         self.LoadTagDefs()
 
     def setdelegate(self, delegate):
@@ -74,7 +71,6 @@
             "DEFINITION": idleConf.GetHighlight(theme, "definition"),
             "SYNC": {'background':None,'foreground':None},
             "TODO": {'background':None,'foreground':None},
-            "BREAK": idleConf.GetHighlight(theme, "break"),
             "ERROR": idleConf.GetHighlight(theme, "error"),
             # The following is used by ReplaceDialog:
             "hit": idleConf.GetHighlight(theme, "hit"),
@@ -216,22 +212,6 @@
                                     self.tag_add("DEFINITION",
                                                  head + "+%dc" % a,
                                                  head + "+%dc" % b)
-                            elif value == "import":
-                                # color all the "as" words on same line, except
-                                # if in a comment; cheap approximation to the
-                                # truth
-                                if '#' in chars:
-                                    endpos = chars.index('#')
-                                else:
-                                    endpos = len(chars)
-                                while True:
-                                    m1 = self.asprog.match(chars, b, endpos)
-                                    if not m1:
-                                        break
-                                    a, b = m1.span(1)
-                                    self.tag_add("KEYWORD",
-                                                 head + "+%dc" % a,
-                                                 head + "+%dc" % b)
                     m = self.prog.search(chars, m.end())
                 if "SYNC" in self.tag_names(next + "-1c"):
                     head = next
@@ -255,20 +235,23 @@
         for tag in self.tagdefs.keys():
             self.tag_remove(tag, "1.0", "end")
 
-def _color_delegator(parent):
+def _color_delegator(parent):  # htest #
+    from Tkinter import Toplevel, Text
     from idlelib.Percolator import Percolator
-    root = Tk()
-    root.title("Test ColorDelegator")
-    width, height, x, y = list(map(int, re.split('[x+]', parent.geometry())))
-    root.geometry("+%d+%d"%(x, y + 150))
-    source = "if somename: x = 'abc' # comment\nprint"
-    text = Text(root, background="white")
+
+    top = Toplevel(parent)
+    top.title("Test ColorDelegator")
+    top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200,
+                  parent.winfo_rooty() + 150))
+    source = "if somename: x = 'abc' # comment\nprint\n"
+    text = Text(top, background="white")
+    text.pack(expand=1, fill="both")
     text.insert("insert", source)
-    text.pack(expand=1, fill="both")
+    text.focus_set()
+
     p = Percolator(text)
     d = ColorDelegator()
     p.insertfilter(d)
-    root.mainloop()
 
 if __name__ == "__main__":
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/Debugger.py b/lib-python/2.7/idlelib/Debugger.py
--- a/lib-python/2.7/idlelib/Debugger.py
+++ b/lib-python/2.7/idlelib/Debugger.py
@@ -1,6 +1,5 @@
 import os
 import bdb
-import types
 from Tkinter import *
 from idlelib.WindowList import ListedToplevel
 from idlelib.ScrolledList import ScrolledList
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -1,6 +1,6 @@
 import sys
 import os
-from platform import python_version
+import platform
 import re
 import imp
 from Tkinter import *
@@ -22,6 +22,8 @@
 # The default tab setting for a Text widget, in average-width characters.
 TK_TABWIDTH_DEFAULT = 8
 
+_py_version = ' (%s)' % platform.python_version()
+
 def _sphinx_version():
     "Format sys.version_info to produce the Sphinx version string used to install the chm docs"
     major, minor, micro, level, serial = sys.version_info
@@ -151,7 +153,7 @@
                     # Safari requires real file:-URLs
                     EditorWindow.help_url = 'file://' + EditorWindow.help_url
             else:
-                EditorWindow.help_url = "http://docs.python.org/%d.%d" % sys.version_info[:2]
+                EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2]
         currentTheme=idleConf.CurrentTheme()
         self.flist = flist
         root = root or flist.root
@@ -214,6 +216,8 @@
         text.bind("<<python-docs>>", self.python_docs)
         text.bind("<<about-idle>>", self.about_dialog)
         text.bind("<<open-config-dialog>>", self.config_dialog)
+        text.bind("<<open-config-extensions-dialog>>",
+                  self.config_extensions_dialog)
         text.bind("<<open-module>>", self.open_module)
         text.bind("<<do-nothing>>", lambda event: "break")
         text.bind("<<select-all>>", self.select_all)
@@ -568,6 +572,8 @@
 
     def config_dialog(self, event=None):
         configDialog.ConfigDialog(self.top,'Settings')
+    def config_extensions_dialog(self, event=None):
+        configDialog.ConfigExtensionsDialog(self.top)
 
     def help_dialog(self, event=None):
         if self.root:
@@ -691,30 +697,29 @@
             return
         # XXX Ought to insert current file's directory in front of path
         try:
-            (f, file, (suffix, mode, type)) = _find_module(name)
+            (f, file_path, (suffix, mode, mtype)) = _find_module(name)
         except (NameError, ImportError) as msg:
             tkMessageBox.showerror("Import error", str(msg), parent=self.text)
             return
-        if type != imp.PY_SOURCE:
+        if mtype != imp.PY_SOURCE:
             tkMessageBox.showerror("Unsupported type",
                 "%s is not a source module" % name, parent=self.text)
             return
         if f:
             f.close()
         if self.flist:
-            self.flist.open(file)
+            self.flist.open(file_path)
         else:
-            self.io.loadfile(file)
+            self.io.loadfile(file_path)
+        return file_path
 
     def open_class_browser(self, event=None):
         filename = self.io.filename
-        if not filename:
-            tkMessageBox.showerror(
-                "No filename",
-                "This buffer has no associated filename",
-                master=self.text)
-            self.text.focus_set()
-            return None
+        if not (self.__class__.__name__ == 'PyShellEditorWindow'
+                and filename):
+            filename = self.open_module()
+            if filename is None:
+                return
         head, tail = os.path.split(filename)
         base, ext = os.path.splitext(tail)
         from idlelib import ClassBrowser
@@ -779,7 +784,7 @@
         self.color = None
 
     def ResetColorizer(self):
-        "Update the colour theme"
+        "Update the color theme"
         # Called from self.filename_change_hook and from configDialog.py
         self._rmcolorizer()
         self._addcolorizer()
@@ -944,7 +949,7 @@
         short = self.short_title()
         long = self.long_title()
         if short and long:
-            title = short + " - " + long
+            title = short + " - " + long + _py_version
         elif short:
             title = short
         elif long:
@@ -968,14 +973,13 @@
         self.undo.reset_undo()
 
     def short_title(self):
-        pyversion = "Python " + python_version() + ": "
         filename = self.io.filename
         if filename:
             filename = os.path.basename(filename)
         else:
             filename = "Untitled"
         # return unicode string to display non-ASCII chars correctly
-        return pyversion + self._filename_to_unicode(filename)
+        return self._filename_to_unicode(filename)
 
     def long_title(self):
         # return unicode string to display non-ASCII chars correctly
@@ -1711,7 +1715,8 @@
     tk.call('set', 'tcl_nonwordchars', '[^a-zA-Z0-9_]')
 
 
-def _editor_window(parent):
+def _editor_window(parent):  # htest #
+    # error if close master window first - timer event, after script
     root = parent
     fixwordbreaks(root)
     if sys.argv[1:]:
@@ -1721,7 +1726,8 @@
     macosxSupport.setupApp(root, None)
     edit = EditorWindow(root=root, filename=filename)
     edit.text.bind("<<close-all-windows>>", edit.close_event)
-    parent.mainloop()
+    # Does not stop error, neither does following
+    # edit.text.bind("<<close-window>>", edit.close_event)
 
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/idlelib/GrepDialog.py b/lib-python/2.7/idlelib/GrepDialog.py
--- a/lib-python/2.7/idlelib/GrepDialog.py
+++ b/lib-python/2.7/idlelib/GrepDialog.py
@@ -45,10 +45,10 @@
 
     def create_entries(self):
         SearchDialogBase.create_entries(self)
-        self.globent = self.make_entry("In files:", self.globvar)
+        self.globent = self.make_entry("In files:", self.globvar)[0]
 
     def create_other_buttons(self):
-        f = self.make_frame()
+        f = self.make_frame()[0]
 
         btn = Checkbutton(f, anchor="w",
                 variable=self.recvar,
@@ -131,7 +131,7 @@
             self.top.withdraw()
 
 
-def _grep_dialog(parent):  # for htest
+def _grep_dialog(parent):  # htest #
     from idlelib.PyShell import PyShellFileList
     root = Tk()
     root.title("Test GrepDialog")
diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py
--- a/lib-python/2.7/idlelib/IOBinding.py
+++ b/lib-python/2.7/idlelib/IOBinding.py
@@ -19,11 +19,7 @@
 
 from idlelib.configHandler import idleConf
 
-try:
-    from codecs import BOM_UTF8
-except ImportError:
-    # only available since Python 2.3
-    BOM_UTF8 = '\xef\xbb\xbf'
+from codecs import BOM_UTF8
 
 # Try setting the locale, so that we can find out
 # what encoding to use
@@ -72,6 +68,7 @@
 encoding = encoding.lower()
 
 coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)')
+blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)')
 
 class EncodingMessage(SimpleDialog):
     "Inform user that an encoding declaration is needed."
@@ -130,6 +127,8 @@
         match = coding_re.match(line)
         if match is not None:
             break
+        if not blank_re.match(line):
+            return None
     else:
         return None
     name = match.group(1)
@@ -529,6 +528,8 @@
         ("All files", "*"),
         ]
 
+    defaultextension = '.py' if sys.platform == 'darwin' else ''
+
     def askopenfile(self):
         dir, base = self.defaultfilename("open")
         if not self.opendialog:
@@ -554,8 +555,10 @@
     def asksavefile(self):
         dir, base = self.defaultfilename("save")
         if not self.savedialog:
-            self.savedialog = tkFileDialog.SaveAs(master=self.text,
-                                                  filetypes=self.filetypes)
+            self.savedialog = tkFileDialog.SaveAs(
+                    master=self.text,
+                    filetypes=self.filetypes,
+                    defaultextension=self.defaultextension)
         filename = self.savedialog.show(initialdir=dir, initialfile=base)
         if isinstance(filename, unicode):
             filename = filename.encode(filesystemencoding)
diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt
--- a/lib-python/2.7/idlelib/NEWS.txt
+++ b/lib-python/2.7/idlelib/NEWS.txt
@@ -1,6 +1,183 @@
+What's New in IDLE 2.7.9?
+=========================
+
+*Release data: 2014-12-07* (projected)
+
+- Issue #16893: Update Idle doc chapter to match current Idle and add new
+  information.
+
+- Issue #3068: Add Idle extension configuration dialog to Options menu.
+  Changes are written to HOME/.idlerc/config-extensions.cfg.
+  Original patch by Tal Einat.
+
+- Issue #16233: A module browser (File : Class Browser, Alt+C) requires a
+  editor window with a filename.  When Class Browser is requested otherwise,
+  from a shell, output window, or 'Untitled' editor, Idle no longer displays
+  an error box.  It now pops up an  Open Module box (Alt+M). If a valid name
+  is entered and a module is opened, a corresponding browser is also opened.
+
+- Issue #4832: Save As to type Python files automatically adds .py to the
+  name you enter (even if your system does not display it).  Some systems
+  automatically add .txt when type is Text files.
+
+- Issue #21986: Code objects are not normally pickled by the pickle module.
+  To match this, they are no longer pickled when running under Idle.
+
+- Issue #22221: IDLE now ignores the source encoding declaration on the second
+  line if the first line contains anything except a comment.
+
+- Issue #17390: Adjust Editor window title; remove 'Python',
+  move version to end.
+
+- Issue #14105: Idle debugger breakpoints no longer disappear
+  when inseting or deleting lines.
+
+
+What's New in IDLE 2.7.8?
+=========================
+
+*Release date: 2014-06-29*
+
+- Issue #21940: Add unittest for WidgetRedirector. Initial patch by Saimadhav
+  Heblikar.
+
+- Issue #18592: Add unittest for SearchDialogBase. Patch by Phil Webster.
+
+- Issue #21694: Add unittest for ParenMatch. Patch by Saimadhav Heblikar.
+
+- Issue #21686: add unittest for HyperParser. Original patch by Saimadhav
+  Heblikar.
+
+- Issue #12387: Add missing upper(lower)case versions of default Windows key
+  bindings for Idle so Caps Lock does not disable them. Patch by Roger Serwy.
+
+- Issue #21695: Closing a Find-in-files output window while the search is
+  still in progress no longer closes Idle.
+
+- Issue #18910: Add unittest for textView. Patch by Phil Webster.
+
+- Issue #18292: Add unittest for AutoExpand. Patch by Saihadhav Heblikar.
+
+- Issue #18409: Add unittest for AutoComplete. Patch by Phil Webster.
+
+
+What's New in IDLE 2.7.7?
+=========================
+
+*Release date: 2014-05-31*
+
+- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin
+  consolidating and improving human-validated tests of Idle. Change other files
+  as needed to work with htest.  Running the module as __main__ runs all tests.
+
+- Issue #21139: Change default paragraph width to 72, the PEP 8 recommendation.
+
+- Issue #21284: Paragraph reformat test passes after user changes reformat width.
+
+- Issue #20406: Use Python application icons for Idle window title bars.
+  Patch mostly by Serhiy Storchaka.
+
+- Issue #21029: Occurrences of "print" are now consistently colored as
+  being a keyword (the colorizer doesn't know if print functions are
+  enabled in the source).
+
+- Issue #17721: Remove non-functional configuration dialog help button until we
+  make it actually gives some help when clicked. Patch by Guilherme Sim&#65533;es.
+
+- Issue #17390: Add Python version to Idle editor window title bar.
+  Original patches by Edmond Burnett and Kent Johnson.
+
+- Issue #20058: sys.stdin.readline() in IDLE now always returns only one line.
+
+- Issue #19481: print() of unicode, str or bytearray subclass instance in IDLE
+  no more hangs.
+
+- Issue #18270: Prevent possible IDLE AttributeError on OS X when no initial
+  shell window is present.
+
+- Issue #17654: Ensure IDLE menus are customized properly on OS X for
+  non-framework builds and for all variants of Tk.
+
+
+What's New in IDLE 2.7.6?
+=========================
+
+*Release date: 2013-11-10*
+
+- Issue #19426: Fixed the opening of Python source file with specified encoding.
+
+- Issue #18873: IDLE now detects Python source code encoding only in comment
+  lines.
+
+- Issue #18988: The "Tab" key now works when a word is already autocompleted.
+
+- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster.
+
+- Issue #18429: Format / Format Paragraph, now works when comment blocks
+  are selected. As with text blocks, this works best when the selection
+  only includes complete lines.
+
+- Issue #18226: Add docstrings and unittests for FormatParagraph.py.
+  Original patches by Todd Rovito and Phil Webster.
+
+- Issue #18279: Format - Strip trailing whitespace no longer marks a file as
+  changed when it has not been changed. This fix followed the addition of a
+  test file originally written by Phil Webster (the issue's main goal).
+
+- Issue #18539: Calltips now work for float default arguments.
+
+- Issue #7136: In the Idle File menu, "New Window" is renamed "New File".
+  Patch by Tal Einat, Roget Serwy, and Todd Rovito.
+
+- Issue #8515: Set __file__ when run file in IDLE.
+  Initial patch by Bruce Frederiksen.
+
+- Issue #5492: Avoid traceback when exiting IDLE caused by a race condition.
+
+- Issue #17511: Keep IDLE find dialog open after clicking "Find Next".
+  Original patch by Sarah K.
+
+- Issue #15392: Create a unittest framework for IDLE.
+  Preliminary patch by Rajagopalasarma Jayakrishnan
+  See Lib/idlelib/idle_test/README.txt for how to run Idle tests.
+
+- Issue #14146: Highlight source line while debugging on Windows.
+
+- Issue #17532: Always include Options menu for IDLE on OS X.
+  Patch by Guilherme Sim&#65533;es.
+
+
 What's New in IDLE 2.7.5?
 =========================
 
+*Release date: 2013-05-12*
+
+- Issue #17838: Allow sys.stdin to be reassigned.
+
+- Issue #14735: Update IDLE docs to omit "Control-z on Windows".
+
+- Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit().
+
+- Issue #17657: Show full Tk version in IDLE's about dialog.
+  Patch by Todd Rovito.
+
+- Issue #17613: Prevent traceback when removing syntax colorizer in IDLE.
+
+- Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE.
+
+- Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box.
+
+- Issue #14254: IDLE now handles readline correctly across shell restarts.
+
+- Issue #17614: IDLE no longer raises exception when quickly closing a file.
+
+- Issue #6698: IDLE now opens just an editor window when configured to do so.
+
+- Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer
+  raises an exception.
+
+- Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo.
+
 - Issue #17390: Display Python version on Idle title bar.
   Initial patch by Edmond Burnett.
 
@@ -8,17 +185,67 @@
 What's New in IDLE 2.7.4?
 =========================
 
+*Release date: 2013-04-06*
+
+- Issue #17625: In IDLE, close the replace dialog after it is used.
+
+- IDLE was displaying spurious SystemExit tracebacks when running scripts
+  that terminated by raising SystemExit (i.e. unittest and turtledemo).
+
+- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase
+  interface and support all mandatory methods and properties.
+
+- Issue #16829: IDLE printing no longer fails if there are spaces or other
+  special characters in the file path.
+
+- Issue #16819: IDLE method completion now correctly works for unicode literals.
+
+- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by
+  Roger Serwy.
+
+- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu
+  Patch by Todd Rovito.
+
+- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog
+  ended with '\'. Patch by Roger Serwy.
+
+- Issue #9803: Don't close IDLE on saving if breakpoint is open.
+  Patch by Roger Serwy.
+
+- Issue #14958: Change IDLE systax highlighting to recognize all string and byte
+  literals currently supported in Python 2.7.
+
+- Issue #14962: Update text coloring in IDLE shell window after changing
+  options.  Patch by Roger Serwy.
+
+- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu.
+
+- Issue #12510: Attempting to get invalid tooltip no longer closes IDLE.
+  Original patch by Roger Serwy.
+
+- Issue #10365: File open dialog now works instead of crashing
+  even when parent window is closed. Patch by Roger Serwy.
+
+- Issue #14876: Use user-selected font for highlight configuration.
+  Patch by Roger Serwy.
+
+- Issue #14409: IDLE now properly executes commands in the Shell window
+  when it cannot read the normal config files on startup and
+  has to use the built-in default key bindings.
+  There was previously a bug in one of the defaults.
+
+- Issue #3573: IDLE hangs when passing invalid command line args
+  (directory(ies) instead of file(s)) (Patch by Guilherme Polo)
+
+- Issue #5219: Prevent event handler cascade in IDLE.
+
 - Issue #15318: Prevent writing to sys.stdin.
 
 - Issue #13532, #15319: Check that arguments to sys.stdout.write are strings.
 
-- Issue # 12510: Attempt to get certain tool tips no longer crashes IDLE.
-
-- Issue10365: File open dialog now works instead of crashing even when
+- Issue #10365: File open dialog now works instead of crashing even when
   parent window is closed while dialog is open.
 
-- Issue 14876: use user-selected font for highlight configuration.
-
 - Issue #14018: Update checks for unstable system Tcl/Tk versions on OS X
   to include versions shipped with OS X 10.7 and 10.8 in addition to 10.6.
 
@@ -29,6 +256,27 @@
 What's New in IDLE 2.7.3?
 =========================
 
+*Release date: 2012-04-09*
+
+- Issue #964437 Make IDLE help window non-modal.
+  Patch by Guilherme Polo and Roger Serwy.
+
+- Issue #13933: IDLE auto-complete did not work with some imported
+  module, like hashlib.  (Patch by Roger Serwy)
+
+- Issue #13506: Add '' to path for IDLE Shell when started and restarted with Restart Shell.
+  Original patches by Marco Scataglini and Roger Serwy.
+
+- Issue #4625: If IDLE cannot write to its recent file or breakpoint
+  files, display a message popup and continue rather than crash.
+  (original patch by Roger Serwy)
+
+- Issue #8793: Prevent IDLE crash when given strings with invalid hex escape
+  sequences.
+
+- Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart.
+  (Patch by Roger Serwy)
+
 - Issue #14409: IDLE now properly executes commands in the Shell window
   when it cannot read the normal config files on startup and
   has to use the built-in default key bindings.
@@ -41,41 +289,85 @@
 What's New in IDLE 2.7.2?
 =========================
 
-*Release date: 29-May-2011*
+*Release date: 2011-06-11*
+
+- Issue #11718: IDLE's open module dialog couldn't find the __init__.py
+  file in a package.
+

From noreply at buildbot.pypy.org  Sun Dec 14 19:35:01 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 14 Dec 2014 19:35:01 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Merged in stdlib/vendor. There are
 merge conflict markers right now
Message-ID: <20141214183501.88D6D1C3D55@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74919:140cc8c53bcd
Date: 2014-12-14 10:34 -0800
http://bitbucket.org/pypy/pypy/changeset/140cc8c53bcd/

Log:	Merged in stdlib/vendor. There are merge conflict markers right now

diff too long, truncating to 2000 out of 24844 lines

diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py
--- a/lib-python/2.7/CGIHTTPServer.py
+++ b/lib-python/2.7/CGIHTTPServer.py
@@ -106,16 +106,16 @@
     def run_cgi(self):
         """Execute a CGI script."""
         dir, rest = self.cgi_info
-
-        i = rest.find('/')
+        path = dir + '/' + rest
+        i = path.find('/', len(dir)+1)
         while i >= 0:
-            nextdir = rest[:i]
-            nextrest = rest[i+1:]
+            nextdir = path[:i]
+            nextrest = path[i+1:]
 
             scriptdir = self.translate_path(nextdir)
             if os.path.isdir(scriptdir):
                 dir, rest = nextdir, nextrest
-                i = rest.find('/')
+                i = path.find('/', len(dir)+1)
             else:
                 break
 
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -56,7 +56,7 @@
    >>> C = Cookie.SmartCookie()
 
 [Note: Long-time users of Cookie.py will remember using
-Cookie.Cookie() to create an Cookie object.  Although deprecated, it
+Cookie.Cookie() to create a Cookie object.  Although deprecated, it
 is still supported by the code.  See the Backward Compatibility notes
 for more information.]
 
@@ -426,6 +426,8 @@
                    "version" : "Version",
                    }
 
+    _flags = {'secure', 'httponly'}
+
     def __init__(self):
         # Set defaults
         self.key = self.value = self.coded_value = None
@@ -529,9 +531,11 @@
 _LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
+    r"\s*"                        # Optional whitespace at start of cookie
     r"(?P<key>"                   # Start of group 'key'
     ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
+    r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
     r"(?P<val>"                   # Start of group 'val'
     r'"(?:[^\\"]|\\.)*"'            # Any doublequoted string
@@ -540,7 +544,9 @@
     r"|"                            # or
     ""+ _LegalCharsPatt +"*"        # Any word or empty string
     r")"                          # End of group 'val'
-    r"\s*;?"                      # Probably ending in a semi-colon
+    r")?"                         # End of optional value group
+    r"\s*"                        # Any number of spaces.
+    r"(\s+|;|$)"                  # Ending either at space, semicolon, or EOS.
     )
 
 
@@ -585,8 +591,12 @@
 
     def __setitem__(self, key, value):
         """Dictionary style assignment."""
-        rval, cval = self.value_encode(value)
-        self.__set(key, rval, cval)
+        if isinstance(value, Morsel):
+            # allow assignment of constructed Morsels (e.g. for pickling)
+            dict.__setitem__(self, key, value)
+        else:
+            rval, cval = self.value_encode(value)
+            self.__set(key, rval, cval)
     # end __setitem__
 
     def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"):
@@ -641,7 +651,7 @@
 
         while 0 <= i < n:
             # Start looking for a cookie
-            match = patt.search(str, i)
+            match = patt.match(str, i)
             if not match: break          # No more cookies
 
             K,V = match.group("key"), match.group("val")
@@ -656,8 +666,12 @@
                     M[ K[1:] ] = V
             elif K.lower() in Morsel._reserved:
                 if M:
-                    M[ K ] = _unquote(V)
-            else:
+                    if V is None:
+                        if K.lower() in Morsel._flags:
+                            M[K] = True
+                    else:
+                        M[K] = _unquote(V)
+            elif V is not None:
                 rval, cval = self.value_decode(V)
                 self.__set(K, rval, cval)
                 M = self[K]
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -416,8 +416,12 @@
         self.socket = socket.socket(self.address_family,
                                     self.socket_type)
         if bind_and_activate:
-            self.server_bind()
-            self.server_activate()
+            try:
+                self.server_bind()
+                self.server_activate()
+            except:
+                self.server_close()
+                raise
 
     def server_bind(self):
         """Called by constructor to bind the socket.
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -143,7 +143,7 @@
     methods except for __contains__, __iter__ and __len__.
 
     To override the comparisons (presumably for speed, as the
-    semantics are fixed), all you have to do is redefine __le__ and
+    semantics are fixed), redefine __le__ and __ge__,
     then the other operations will automatically follow suit.
     """
 
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -1089,7 +1089,14 @@
         # parse all the remaining options into the namespace
         # store any unrecognized options on the object, so that the top
         # level parser can decide what to do with them
-        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
+
+        # In case this subparser defines new defaults, we parse them
+        # in a new namespace object and then update the original
+        # namespace for the relevant parts.
+        subnamespace, arg_strings = parser.parse_known_args(arg_strings, None)
+        for key, value in vars(subnamespace).items():
+            setattr(namespace, key, value)
+
         if arg_strings:
             vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
             getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py
--- a/lib-python/2.7/asynchat.py
+++ b/lib-python/2.7/asynchat.py
@@ -46,12 +46,17 @@
 you - by calling your self.found_terminator() method.
 """
 
+import asyncore
+import errno
 import socket
-import asyncore
 from collections import deque
 from sys import py3kwarning
 from warnings import filterwarnings, catch_warnings
 
+_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS,
+                       errno.EWOULDBLOCK)
+
+
 class async_chat (asyncore.dispatcher):
     """This is an abstract class.  You must derive from this class, and add
     the two methods collect_incoming_data() and found_terminator()"""
@@ -109,6 +114,8 @@
         try:
             data = self.recv (self.ac_in_buffer_size)
         except socket.error, why:
+            if why.args[0] in _BLOCKING_IO_ERRORS:
+                return
             self.handle_error()
             return
 
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,6 +10,7 @@
 
 #----------------------------------------------------------------------
 
+ at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -1719,12 +1719,12 @@
     def __repr__(self):
         r = []
         for cookie in self: r.append(repr(cookie))
-        return "<%s[%s]>" % (self.__class__, ", ".join(r))
+        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
 
     def __str__(self):
         r = []
         for cookie in self: r.append(str(cookie))
-        return "<%s[%s]>" % (self.__class__, ", ".join(r))
+        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
 
 
 # derives from IOError for backwards-compatibility with Python 2.4.0
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,6 +7,8 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
+LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
+large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -188,5 +190,11 @@
             mth = WINFUNCTYPE(None)(42, "name", (), None)
             self.assertEqual(bool(mth), True)
 
+    def test_pointer_type_name(self):
+        self.assertTrue(POINTER(LargeNamedType))
+
+    def test_pointer_type_str_name(self):
+        self.assertTrue(POINTER(large_string))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py
--- a/lib-python/2.7/ctypes/test/test_python_api.py
+++ b/lib-python/2.7/ctypes/test/test_python_api.py
@@ -46,8 +46,8 @@
     # This test is unreliable, because it is possible that code in
     # unittest changes the refcount of the '42' integer.  So, it
     # is disabled by default.
-    @requires("refcount")
     def test_PyInt_Long(self):
+        requires("refcount")
         ref42 = grc(42)
         pythonapi.PyInt_FromLong.restype = py_object
         self.assertEqual(pythonapi.PyInt_FromLong(42), 42)
diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py
--- a/lib-python/2.7/ctypes/test/test_win32.py
+++ b/lib-python/2.7/ctypes/test/test_win32.py
@@ -38,8 +38,11 @@
 
 @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
 class FunctionCallTestCase(unittest.TestCase):
-    @requires("SEH")
+    @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC")
+    @unittest.skipIf(sys.executable.endswith('_d.exe'),
+                     "SEH not enabled in debug builds")
     def test_SEH(self):
+        requires("SEH")
         # Call functions with invalid arguments, and make sure
         # that access violations are trapped and raise an
         # exception.
@@ -87,9 +90,29 @@
 
         dll = CDLL(_ctypes_test.__file__)
 
-        pt = POINT(10, 10)
-        rect = RECT(0, 0, 20, 20)
-        self.assertEqual(1, dll.PointInRect(byref(rect), pt))
+        pt = POINT(15, 25)
+        left = c_long.in_dll(dll, 'left')
+        top = c_long.in_dll(dll, 'top')
+        right = c_long.in_dll(dll, 'right')
+        bottom = c_long.in_dll(dll, 'bottom')
+        rect = RECT(left, top, right, bottom)
+        PointInRect = dll.PointInRect
+        PointInRect.argtypes = [POINTER(RECT), POINT]
+        self.assertEqual(1, PointInRect(byref(rect), pt))
+
+        ReturnRect = dll.ReturnRect
+        ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT,
+                               POINTER(RECT), POINT, RECT]
+        ReturnRect.restype = RECT
+        for i in range(4):
+            ret = ReturnRect(i, rect, pointer(rect), pt, rect,
+                         byref(rect), pt, rect)
+            # the c function will check and modify ret if something is
+            # passed in improperly
+            self.assertEqual(ret.left, left.value)
+            self.assertEqual(ret.right, right.value)
+            self.assertEqual(ret.top, top.value)
+            self.assertEqual(ret.bottom, bottom.value)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py
--- a/lib-python/2.7/decimal.py
+++ b/lib-python/2.7/decimal.py
@@ -136,7 +136,6 @@
 
 __version__ = '1.70'    # Highest version of the spec this complies with
 
-import copy as _copy
 import math as _math
 import numbers as _numbers
 
@@ -3665,6 +3664,8 @@
         if self._is_special:
             sign = _format_sign(self._sign, spec)
             body = str(self.copy_abs())
+            if spec['type'] == '%':
+                body += '%'
             return _format_align(sign, body, spec)
 
         # a type of None defaults to 'g' or 'G', depending on context
@@ -6033,7 +6034,10 @@
         format_dict['decimal_point'] = '.'
 
     # record whether return type should be str or unicode
-    format_dict['unicode'] = isinstance(format_spec, unicode)
+    try:
+        format_dict['unicode'] = isinstance(format_spec, unicode)
+    except NameError:
+        format_dict['unicode'] = False
 
     return format_dict
 
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.8"
+__version__ = "2.7.9"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -245,7 +245,7 @@
         # Python's library directory must be appended to library_dirs
         # See Issues: #1600860, #4366
         if (sysconfig.get_config_var('Py_ENABLE_SHARED')):
-            if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
+            if not sysconfig.python_build:
                 # building third party extensions
                 self.library_dirs.append(sysconfig.get_config_var('LIBDIR'))
             else:
diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py
--- a/lib-python/2.7/distutils/command/upload.py
+++ b/lib-python/2.7/distutils/command/upload.py
@@ -136,8 +136,8 @@
 
         # Build up the MIME payload for the POST data
         boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
-        sep_boundary = '\n--' + boundary
-        end_boundary = sep_boundary + '--'
+        sep_boundary = '\r\n--' + boundary
+        end_boundary = sep_boundary + '--\r\n'
         body = StringIO.StringIO()
         for key, value in data.items():
             # handle multiple entries for the same name
@@ -151,14 +151,13 @@
                     fn = ""
 
                 body.write(sep_boundary)
-                body.write('\nContent-Disposition: form-data; name="%s"'%key)
+                body.write('\r\nContent-Disposition: form-data; name="%s"' % key)
                 body.write(fn)
-                body.write("\n\n")
+                body.write("\r\n\r\n")
                 body.write(value)
                 if value and value[-1] == '\r':
                     body.write('\n')  # write an extra newline (lurve Macs)
         body.write(end_boundary)
-        body.write("\n")
         body = body.getvalue()
 
         self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO)
diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py
--- a/lib-python/2.7/distutils/file_util.py
+++ b/lib-python/2.7/distutils/file_util.py
@@ -85,7 +85,8 @@
     (os.symlink) instead of copying: set it to "hard" or "sym"; if it is
     None (the default), files are copied.  Don't set 'link' on systems that
     don't support it: 'copy_file()' doesn't check if hard or symbolic
-    linking is available.
+    linking is available. If hardlink fails, falls back to
+    _copy_file_contents().
 
     Under Mac OS, uses the native file copy function in macostools; on
     other systems, uses '_copy_file_contents()' to copy file contents.
@@ -137,24 +138,31 @@
     # (Unix only, of course, but that's the caller's responsibility)
     if link == 'hard':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
-            os.link(src, dst)
+            try:
+                os.link(src, dst)
+                return (dst, 1)
+            except OSError:
+                # If hard linking fails, fall back on copying file
+                # (some special filesystems don't support hard linking
+                #  even under Unix, see issue #8876).
+                pass
     elif link == 'sym':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
             os.symlink(src, dst)
+            return (dst, 1)
 
     # Otherwise (non-Mac, not linking), copy the file contents and
     # (optionally) copy the times and mode.
-    else:
-        _copy_file_contents(src, dst)
-        if preserve_mode or preserve_times:
-            st = os.stat(src)
+    _copy_file_contents(src, dst)
+    if preserve_mode or preserve_times:
+        st = os.stat(src)
 
-            # According to David Ascher <da at ski.org>, utime() should be done
-            # before chmod() (at least under NT).
-            if preserve_times:
-                os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
-            if preserve_mode:
-                os.chmod(dst, S_IMODE(st[ST_MODE]))
+        # According to David Ascher <da at ski.org>, utime() should be done
+        # before chmod() (at least under NT).
+        if preserve_times:
+            os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
+        if preserve_mode:
+            os.chmod(dst, S_IMODE(st[ST_MODE]))
 
     return (dst, 1)
 
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -5,8 +5,477 @@
 # directly.
 
 import sys
+<<<<<<< local
 if '__pypy__' in sys.builtin_module_names:
     from distutils import sysconfig_pypy as _sysconfig_module
 else:
     from distutils import sysconfig_cpython as _sysconfig_module
 globals().update(_sysconfig_module.__dict__)
+=======
+
+from distutils.errors import DistutilsPlatformError
+
+# These are needed in a couple of spots, so just compute them once.
+PREFIX = os.path.normpath(sys.prefix)
+EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+
+# Path to the base directory of the project. On Windows the binary may
+# live in project/PCBuild9.  If we're dealing with an x64 Windows build,
+# it'll live in project/PCbuild/amd64.
+project_base = os.path.dirname(os.path.abspath(sys.executable))
+if os.name == "nt" and "pcbuild" in project_base[-8:].lower():
+    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir))
+# PC/VS7.1
+if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower():
+    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
+                                                os.path.pardir))
+# PC/AMD64
+if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower():
+    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
+                                                os.path.pardir))
+
+# set for cross builds
+if "_PYTHON_PROJECT_BASE" in os.environ:
+    # this is the build directory, at least for posix
+    project_base = os.path.normpath(os.environ["_PYTHON_PROJECT_BASE"])
+
+# python_build: (Boolean) if true, we're either building Python or
+# building an extension with an un-installed Python, so we use
+# different (hard-wired) directories.
+# Setup.local is available for Makefile builds including VPATH builds,
+# Setup.dist is available on Windows
+def _python_build():
+    for fn in ("Setup.dist", "Setup.local"):
+        if os.path.isfile(os.path.join(project_base, "Modules", fn)):
+            return True
+    return False
+python_build = _python_build()
+
+
+def get_python_version():
+    """Return a string containing the major and minor Python version,
+    leaving off the patchlevel.  Sample return values could be '1.5'
+    or '2.2'.
+    """
+    return sys.version[:3]
+
+
+def get_python_inc(plat_specific=0, prefix=None):
+    """Return the directory containing installed Python header files.
+
+    If 'plat_specific' is false (the default), this is the path to the
+    non-platform-specific header files, i.e. Python.h and so on;
+    otherwise, this is the path to platform-specific header files
+    (namely pyconfig.h).
+
+    If 'prefix' is supplied, use it instead of sys.prefix or
+    sys.exec_prefix -- i.e., ignore 'plat_specific'.
+    """
+    if prefix is None:
+        prefix = plat_specific and EXEC_PREFIX or PREFIX
+
+    if os.name == "posix":
+        if python_build:
+            buildir = os.path.dirname(sys.executable)
+            if plat_specific:
+                # python.h is located in the buildir
+                inc_dir = buildir
+            else:
+                # the source dir is relative to the buildir
+                srcdir = os.path.abspath(os.path.join(buildir,
+                                         get_config_var('srcdir')))
+                # Include is located in the srcdir
+                inc_dir = os.path.join(srcdir, "Include")
+            return inc_dir
+        return os.path.join(prefix, "include", "python" + get_python_version())
+    elif os.name == "nt":
+        return os.path.join(prefix, "include")
+    elif os.name == "os2":
+        return os.path.join(prefix, "Include")
+    else:
+        raise DistutilsPlatformError(
+            "I don't know where Python installs its C header files "
+            "on platform '%s'" % os.name)
+
+
+def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
+    """Return the directory containing the Python library (standard or
+    site additions).
+
+    If 'plat_specific' is true, return the directory containing
+    platform-specific modules, i.e. any module from a non-pure-Python
+    module distribution; otherwise, return the platform-shared library
+    directory.  If 'standard_lib' is true, return the directory
+    containing standard Python library modules; otherwise, return the
+    directory for site-specific modules.
+
+    If 'prefix' is supplied, use it instead of sys.prefix or
+    sys.exec_prefix -- i.e., ignore 'plat_specific'.
+    """
+    if prefix is None:
+        prefix = plat_specific and EXEC_PREFIX or PREFIX
+
+    if os.name == "posix":
+        libpython = os.path.join(prefix,
+                                 "lib", "python" + get_python_version())
+        if standard_lib:
+            return libpython
+        else:
+            return os.path.join(libpython, "site-packages")
+
+    elif os.name == "nt":
+        if standard_lib:
+            return os.path.join(prefix, "Lib")
+        else:
+            if get_python_version() < "2.2":
+                return prefix
+            else:
+                return os.path.join(prefix, "Lib", "site-packages")
+
+    elif os.name == "os2":
+        if standard_lib:
+            return os.path.join(prefix, "Lib")
+        else:
+            return os.path.join(prefix, "Lib", "site-packages")
+
+    else:
+        raise DistutilsPlatformError(
+            "I don't know where Python installs its library "
+            "on platform '%s'" % os.name)
+
+
+
+def customize_compiler(compiler):
+    """Do any platform-specific customization of a CCompiler instance.
+
+    Mainly needed on Unix, so we can plug in the information that
+    varies across Unices and is stored in Python's Makefile.
+    """
+    if compiler.compiler_type == "unix":
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
+
+        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
+            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
+                            'CCSHARED', 'LDSHARED', 'SO', 'AR',
+                            'ARFLAGS')
+
+        if 'CC' in os.environ:
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
+        if 'CXX' in os.environ:
+            cxx = os.environ['CXX']
+        if 'LDSHARED' in os.environ:
+            ldshared = os.environ['LDSHARED']
+        if 'CPP' in os.environ:
+            cpp = os.environ['CPP']
+        else:
+            cpp = cc + " -E"           # not always
+        if 'LDFLAGS' in os.environ:
+            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
+        if 'CFLAGS' in os.environ:
+            cflags = opt + ' ' + os.environ['CFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CFLAGS']
+        if 'CPPFLAGS' in os.environ:
+            cpp = cpp + ' ' + os.environ['CPPFLAGS']
+            cflags = cflags + ' ' + os.environ['CPPFLAGS']
+            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
+
+        cc_cmd = cc + ' ' + cflags
+        compiler.set_executables(
+            preprocessor=cpp,
+            compiler=cc_cmd,
+            compiler_so=cc_cmd + ' ' + ccshared,
+            compiler_cxx=cxx,
+            linker_so=ldshared,
+            linker_exe=cc,
+            archiver=archiver)
+
+        compiler.shared_lib_extension = so_ext
+
+
+def get_config_h_filename():
+    """Return full pathname of installed pyconfig.h file."""
+    if python_build:
+        if os.name == "nt":
+            inc_dir = os.path.join(project_base, "PC")
+        else:
+            inc_dir = project_base
+    else:
+        inc_dir = get_python_inc(plat_specific=1)
+    if get_python_version() < '2.2':
+        config_h = 'config.h'
+    else:
+        # The name of the config.h file changed in 2.2
+        config_h = 'pyconfig.h'
+    return os.path.join(inc_dir, config_h)
+
+
+def get_makefile_filename():
+    """Return full pathname of installed Makefile from the Python build."""
+    if python_build:
+        return os.path.join(project_base, "Makefile")
+    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
+    return os.path.join(lib_dir, "config", "Makefile")
+
+
+def parse_config_h(fp, g=None):
+    """Parse a config.h-style file.
+
+    A dictionary containing name/value pairs is returned.  If an
+    optional dictionary is passed in as the second argument, it is
+    used instead of a new dictionary.
+    """
+    if g is None:
+        g = {}
+    define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
+    undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
+    #
+    while 1:
+        line = fp.readline()
+        if not line:
+            break
+        m = define_rx.match(line)
+        if m:
+            n, v = m.group(1, 2)
+            try: v = int(v)
+            except ValueError: pass
+            g[n] = v
+        else:
+            m = undef_rx.match(line)
+            if m:
+                g[m.group(1)] = 0
+    return g
+
+
+# Regexes needed for parsing Makefile (and similar syntaxes,
+# like old-style Setup files).
+_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
+_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
+_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
+
+def parse_makefile(fn, g=None):
+    """Parse a Makefile-style file.
+
+    A dictionary containing name/value pairs is returned.  If an
+    optional dictionary is passed in as the second argument, it is
+    used instead of a new dictionary.
+    """
+    from distutils.text_file import TextFile
+    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)
+
+    if g is None:
+        g = {}
+    done = {}
+    notdone = {}
+
+    while 1:
+        line = fp.readline()
+        if line is None:  # eof
+            break
+        m = _variable_rx.match(line)
+        if m:
+            n, v = m.group(1, 2)
+            v = v.strip()
+            # `$$' is a literal `$' in make
+            tmpv = v.replace('$$', '')
+
+            if "$" in tmpv:
+                notdone[n] = v
+            else:
+                try:
+                    v = int(v)
+                except ValueError:
+                    # insert literal `$'
+                    done[n] = v.replace('$$', '$')
+                else:
+                    done[n] = v
+
+    # do variable interpolation here
+    while notdone:
+        for name in notdone.keys():
+            value = notdone[name]
+            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
+            if m:
+                n = m.group(1)
+                found = True
+                if n in done:
+                    item = str(done[n])
+                elif n in notdone:
+                    # get it on a subsequent round
+                    found = False
+                elif n in os.environ:
+                    # do it like make: fall back to environment
+                    item = os.environ[n]
+                else:
+                    done[n] = item = ""
+                if found:
+                    after = value[m.end():]
+                    value = value[:m.start()] + item + after
+                    if "$" in after:
+                        notdone[name] = value
+                    else:
+                        try: value = int(value)
+                        except ValueError:
+                            done[name] = value.strip()
+                        else:
+                            done[name] = value
+                        del notdone[name]
+            else:
+                # bogus variable reference; just drop it since we can't deal
+                del notdone[name]
+
+    fp.close()
+
+    # strip spurious spaces
+    for k, v in done.items():
+        if isinstance(v, str):
+            done[k] = v.strip()
+
+    # save the results in the global dictionary
+    g.update(done)
+    return g
+
+
+def expand_makefile_vars(s, vars):
+    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
+    'string' according to 'vars' (a dictionary mapping variable names to
+    values).  Variables not present in 'vars' are silently expanded to the
+    empty string.  The variable values in 'vars' should not contain further
+    variable expansions; if 'vars' is the output of 'parse_makefile()',
+    you're fine.  Returns a variable-expanded version of 's'.
+    """
+
+    # This algorithm does multiple expansion, so if vars['foo'] contains
+    # "${bar}", it will expand ${foo} to ${bar}, and then expand
+    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
+    # 'parse_makefile()', which takes care of such expansions eagerly,
+    # according to make's variable expansion semantics.
+
+    while 1:
+        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
+        if m:
+            (beg, end) = m.span()
+            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
+        else:
+            break
+    return s
+
+
+_config_vars = None
+
+def _init_posix():
+    """Initialize the module as appropriate for POSIX systems."""
+    # _sysconfigdata is generated at build time, see the sysconfig module
+    from _sysconfigdata import build_time_vars
+    global _config_vars
+    _config_vars = {}
+    _config_vars.update(build_time_vars)
+
+
+def _init_nt():
+    """Initialize the module as appropriate for NT"""
+    g = {}
+    # set basic install directories
+    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
+    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
+
+    # XXX hmmm.. a normal install puts include files here
+    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
+
+    g['SO'] = '.pyd'
+    g['EXE'] = ".exe"
+    g['VERSION'] = get_python_version().replace(".", "")
+    g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
+
+    global _config_vars
+    _config_vars = g
+
+
+def _init_os2():
+    """Initialize the module as appropriate for OS/2"""
+    g = {}
+    # set basic install directories
+    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
+    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
+
+    # XXX hmmm.. a normal install puts include files here
+    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
+
+    g['SO'] = '.pyd'
+    g['EXE'] = ".exe"
+
+    global _config_vars
+    _config_vars = g
+
+
+def get_config_vars(*args):
+    """With no arguments, return a dictionary of all configuration
+    variables relevant for the current platform.  Generally this includes
+    everything needed to build extensions and install both pure modules and
+    extensions.  On Unix, this means every variable defined in Python's
+    installed Makefile; on Windows and Mac OS it's a much smaller set.
+
+    With arguments, return a list of values that result from looking up
+    each argument in the configuration variable dictionary.
+    """
+    global _config_vars
+    if _config_vars is None:
+        func = globals().get("_init_" + os.name)
+        if func:
+            func()
+        else:
+            _config_vars = {}
+
+        # Normalized versions of prefix and exec_prefix are handy to have;
+        # in fact, these are the standard versions used most places in the
+        # Distutils.
+        _config_vars['prefix'] = PREFIX
+        _config_vars['exec_prefix'] = EXEC_PREFIX
+
+        # OS X platforms require special customization to handle
+        # multi-architecture, multi-os-version installers
+        if sys.platform == 'darwin':
+            import _osx_support
+            _osx_support.customize_config_vars(_config_vars)
+
+    if args:
+        vals = []
+        for name in args:
+            vals.append(_config_vars.get(name))
+        return vals
+    else:
+        return _config_vars
+
+def get_config_var(name):
+    """Return the value of a single variable using the dictionary
+    returned by 'get_config_vars()'.  Equivalent to
+    get_config_vars().get(name)
+    """
+    return get_config_vars().get(name)
+>>>>>>> other
diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
--- a/lib-python/2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/2.7/distutils/sysconfig_cpython.py
@@ -165,7 +165,8 @@
             # version and build tools may not support the same set
             # of CPU architectures for universal builds.
             global _config_vars
-            if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''):
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
                 import _osx_support
                 _osx_support.customize_compiler(_config_vars)
                 _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
--- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py
+++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
@@ -25,6 +25,7 @@
 """
 
 class BuildRpmTestCase(support.TempdirManager,
+                       support.EnvironGuard,
                        support.LoggingSilencer,
                        unittest.TestCase):
 
@@ -50,6 +51,7 @@
     def test_quiet(self):
         # let's create a package
         tmp_dir = self.mkdtemp()
+        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
         os.mkdir(pkg_dir)
         self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
@@ -92,6 +94,7 @@
     def test_no_optimize_flag(self):
         # let's create a package that brakes bdist_rpm
         tmp_dir = self.mkdtemp()
+        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
         os.mkdir(pkg_dir)
         self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py
--- a/lib-python/2.7/distutils/tests/test_dist.py
+++ b/lib-python/2.7/distutils/tests/test_dist.py
@@ -11,7 +11,7 @@
 from distutils.dist import Distribution, fix_help_options
 from distutils.cmd import Command
 import distutils.dist
-from test.test_support import TESTFN, captured_stdout, run_unittest
+from test.test_support import TESTFN, captured_stdout, run_unittest, unlink
 from distutils.tests import support
 
 
@@ -64,6 +64,7 @@
         with open(TESTFN, "w") as f:
             f.write("[global]\n")
             f.write("command_packages = foo.bar, splat")
+        self.addCleanup(unlink, TESTFN)
 
         files = [TESTFN]
         sys.argv.append("build")
diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py
--- a/lib-python/2.7/distutils/tests/test_file_util.py
+++ b/lib-python/2.7/distutils/tests/test_file_util.py
@@ -8,6 +8,11 @@
 from distutils.tests import support
 from test.test_support import run_unittest
 
+
+requires_os_link = unittest.skipUnless(hasattr(os, "link"),
+                                       "test requires os.link()")
+
+
 class FileUtilTestCase(support.TempdirManager, unittest.TestCase):
 
     def _log(self, msg, *args):
@@ -74,6 +79,44 @@
         copy_file(foo, dst_dir)
         self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo')))
 
+    @requires_os_link
+    def test_copy_file_hard_link(self):
+        with open(self.source, 'w') as f:
+            f.write('some content')
+        st = os.stat(self.source)
+        copy_file(self.source, self.target, link='hard')
+        st2 = os.stat(self.source)
+        st3 = os.stat(self.target)
+        self.assertTrue(os.path.samestat(st, st2), (st, st2))
+        self.assertTrue(os.path.samestat(st2, st3), (st2, st3))
+        with open(self.source, 'r') as f:
+            self.assertEqual(f.read(), 'some content')
+
+    @requires_os_link
+    def test_copy_file_hard_link_failure(self):
+        # If hard linking fails, copy_file() falls back on copying file
+        # (some special filesystems don't support hard linking even under
+        #  Unix, see issue #8876).
+        with open(self.source, 'w') as f:
+            f.write('some content')
+        st = os.stat(self.source)
+        def _os_link(*args):
+            raise OSError(0, "linking unsupported")
+        old_link = os.link
+        os.link = _os_link
+        try:
+            copy_file(self.source, self.target, link='hard')
+        finally:
+            os.link = old_link
+        st2 = os.stat(self.source)
+        st3 = os.stat(self.target)
+        self.assertTrue(os.path.samestat(st, st2), (st, st2))
+        self.assertFalse(os.path.samestat(st2, st3), (st2, st3))
+        for fn in (self.source, self.target):
+            with open(fn, 'r') as f:
+                self.assertEqual(f.read(), 'some content')
+
+
 def test_suite():
     return unittest.makeSuite(FileUtilTestCase)
 
diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py
--- a/lib-python/2.7/distutils/tests/test_sysconfig.py
+++ b/lib-python/2.7/distutils/tests/test_sysconfig.py
@@ -3,6 +3,9 @@
 import test
 import unittest
 import shutil
+import subprocess
+import sys
+import textwrap
 
 from distutils import sysconfig
 from distutils.tests import support
@@ -99,6 +102,24 @@
         self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED'))
         self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC'))
 
+    def test_customize_compiler_before_get_config_vars(self):
+        # Issue #21923: test that a Distribution compiler
+        # instance can be called without an explicit call to
+        # get_config_vars().
+        with open(TESTFN, 'w') as f:
+            f.writelines(textwrap.dedent('''\
+                from distutils.core import Distribution
+                config = Distribution().get_command_obj('config')
+                # try_compile may pass or it may fail if no compiler
+                # is found but it should not raise an exception.
+                rc = config.try_compile('int x;')
+                '''))
+        p = subprocess.Popen([str(sys.executable), TESTFN],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                universal_newlines=True)
+        outs, errs = p.communicate()
+        self.assertEqual(0, p.returncode, "Subprocess failed: " + outs)
 
 
 def test_suite():
diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py
--- a/lib-python/2.7/distutils/tests/test_upload.py
+++ b/lib-python/2.7/distutils/tests/test_upload.py
@@ -119,7 +119,7 @@
         # what did we send ?
         self.assertIn('d&#233;d&#233;', self.last_open.req.data)
         headers = dict(self.last_open.req.headers)
-        self.assertEqual(headers['Content-length'], '2085')
+        self.assertEqual(headers['Content-length'], '2159')
         self.assertTrue(headers['Content-type'].startswith('multipart/form-data'))
         self.assertEqual(self.last_open.req.get_method(), 'POST')
         self.assertEqual(self.last_open.req.get_full_url(),
diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py
--- a/lib-python/2.7/doctest.py
+++ b/lib-python/2.7/doctest.py
@@ -216,7 +216,7 @@
                 # get_data() opens files as 'rb', so one must do the equivalent
                 # conversion as universal newlines would do.
                 return file_contents.replace(os.linesep, '\n'), filename
-    with open(filename) as f:
+    with open(filename, 'U') as f:
         return f.read(), filename
 
 # Use sys.stdout encoding for ouput.
diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py
--- a/lib-python/2.7/email/feedparser.py
+++ b/lib-python/2.7/email/feedparser.py
@@ -49,8 +49,8 @@
     simple abstraction -- it parses until EOF closes the current message.
     """
     def __init__(self):
-        # The last partial line pushed into this object.
-        self._partial = ''
+        # Chunks of the last partial line pushed into this object.
+        self._partial = []
         # The list of full, pushed lines, in reverse order
         self._lines = []
         # The stack of false-EOF checking predicates.
@@ -66,8 +66,8 @@
 
     def close(self):
         # Don't forget any trailing partial line.
-        self._lines.append(self._partial)
-        self._partial = ''
+        self.pushlines(''.join(self._partial).splitlines(True))
+        self._partial = []
         self._closed = True
 
     def readline(self):
@@ -95,8 +95,29 @@
 
     def push(self, data):
         """Push some new data into this object."""
-        # Handle any previous leftovers
-        data, self._partial = self._partial + data, ''
+        # Crack into lines, but preserve the linesep characters on the end of each
+        parts = data.splitlines(True)
+
+        if not parts or not parts[0].endswith(('\n', '\r')):
+            # No new complete lines, so just accumulate partials
+            self._partial += parts
+            return
+
+        if self._partial:
+            # If there are previous leftovers, complete them now
+            self._partial.append(parts[0])
+            parts[0:1] = ''.join(self._partial).splitlines(True)
+            del self._partial[:]
+
+        # If the last element of the list does not end in a newline, then treat
+        # it as a partial line.  We only check for '\n' here because a line
+        # ending with '\r' might be a line that was split in the middle of a
+        # '\r\n' sequence (see bugs 1555570 and 1721862).
+        if not parts[-1].endswith('\n'):
+            self._partial = [parts.pop()]
+        self.pushlines(parts)
+
+    def pushlines(self, lines):
         # Crack into lines, but preserve the newlines on the end of each
         parts = NLCRE_crack.split(data)
         # The *ahem* interesting behaviour of re.split when supplied grouping
diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py
--- a/lib-python/2.7/email/mime/nonmultipart.py
+++ b/lib-python/2.7/email/mime/nonmultipart.py
@@ -12,7 +12,7 @@
 
 

 class MIMENonMultipart(MIMEBase):
-    """Base class for MIME multipart/* type messages."""
+    """Base class for MIME non-multipart type messages."""
 
     def attach(self, payload):
         # The public API prohibits attaching multiple subparts to MIMEBase
diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py
--- a/lib-python/2.7/email/test/test_email.py
+++ b/lib-python/2.7/email/test/test_email.py
@@ -11,6 +11,7 @@
 import warnings
 import textwrap
 from cStringIO import StringIO
+from random import choice
 
 import email
 
@@ -2578,16 +2579,64 @@
             bsf.push(il)
             nt += n
             n1 = 0
-            while True:
-                ol = bsf.readline()
-                if ol == NeedMoreData:
-                    break
+            for ol in iter(bsf.readline, NeedMoreData):
                 om.append(ol)
                 n1 += 1
             self.assertEqual(n, n1)
         self.assertEqual(len(om), nt)
         self.assertEqual(''.join([il for il, n in imt]), ''.join(om))
 
+    def test_push_random(self):
+        from email.feedparser import BufferedSubFile, NeedMoreData
+
+        n = 10000
+        chunksize = 5
+        chars = 'abcd \t\r\n'
+
+        s = ''.join(choice(chars) for i in range(n)) + '\n'
+        target = s.splitlines(True)
+
+        bsf = BufferedSubFile()
+        lines = []
+        for i in range(0, len(s), chunksize):
+            chunk = s[i:i+chunksize]
+            bsf.push(chunk)
+            lines.extend(iter(bsf.readline, NeedMoreData))
+        self.assertEqual(lines, target)
+
+
+class TestFeedParsers(TestEmailBase):
+
+    def parse(self, chunks):
+        from email.feedparser import FeedParser
+        feedparser = FeedParser()
+        for chunk in chunks:
+            feedparser.feed(chunk)
+        return feedparser.close()
+
+    def test_newlines(self):
+        m = self.parse(['a:\nb:\rc:\r\nd:\n'])
+        self.assertEqual(m.keys(), ['a', 'b', 'c', 'd'])
+        m = self.parse(['a:\nb:\rc:\r\nd:'])
+        self.assertEqual(m.keys(), ['a', 'b', 'c', 'd'])
+        m = self.parse(['a:\rb', 'c:\n'])
+        self.assertEqual(m.keys(), ['a', 'bc'])
+        m = self.parse(['a:\r', 'b:\n'])
+        self.assertEqual(m.keys(), ['a', 'b'])
+        m = self.parse(['a:\r', '\nb:\n'])
+        self.assertEqual(m.keys(), ['a', 'b'])
+
+    def test_long_lines(self):
+        # Expected peak memory use on 32-bit platform: 4*N*M bytes.
+        M, N = 1000, 20000
+        m = self.parse(['a:b\n\n'] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', 'b')])
+        self.assertEqual(m.get_payload(), 'x'*M*N)
+        m = self.parse(['a:b\r\r'] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', 'b')])
+        self.assertEqual(m.get_payload(), 'x'*M*N)
+        m = self.parse(['a:\r', 'b: '] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)])
 
 
 class TestParsers(TestEmailBase):
@@ -3180,7 +3229,6 @@
         self.assertEqual(res, '=?iso-8859-2?q?abc?=')
         self.assertIsInstance(res, str)
 
-
 # Test RFC 2231 header parameters (en/de)coding
 class TestRFC2231(TestEmailBase):
     def test_get_param(self):
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python2
+from __future__ import print_function
+
+import os
+import os.path
+import pkgutil
+import shutil
+import sys
+import tempfile
+
+
+__all__ = ["version", "bootstrap"]
+
+
+_SETUPTOOLS_VERSION = "7.0"
+
+_PIP_VERSION = "1.5.6"
+
+# pip currently requires ssl support, so we try to provide a nicer
+# error message when that is missing (http://bugs.python.org/issue19744)
+_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION))
+try:
+    import ssl
+except ImportError:
+    ssl = None
+
+    def _require_ssl_for_pip():
+        raise RuntimeError(_MISSING_SSL_MESSAGE)
+else:
+    def _require_ssl_for_pip():
+        pass
+
+_PROJECTS = [
+    ("setuptools", _SETUPTOOLS_VERSION),
+    ("pip", _PIP_VERSION),
+]
+
+
+def _run_pip(args, additional_paths=None):
+    # Add our bundled software to the sys.path so we can import it
+    if additional_paths is not None:
+        sys.path = additional_paths + sys.path
+
+    # Install the bundled software
+    import pip
+    pip.main(args)
+
+
+def version():
+    """
+    Returns a string specifying the bundled version of pip.
+    """
+    return _PIP_VERSION
+
+
+def _disable_pip_configuration_settings():
+    # We deliberately ignore all pip environment variables
+    # when invoking pip
+    # See http://bugs.python.org/issue19734 for details
+    keys_to_remove = [k for k in os.environ if k.startswith("PIP_")]
+    for k in keys_to_remove:
+        del os.environ[k]
+    # We also ignore the settings in the default pip configuration file
+    # See http://bugs.python.org/issue20053 for details
+    os.environ['PIP_CONFIG_FILE'] = os.devnull
+
+
+def bootstrap(root=None, upgrade=False, user=False,
+              altinstall=False, default_pip=True,
+              verbosity=0):
+    """
+    Bootstrap pip into the current Python installation (or the given root
+    directory).
+
+    Note that calling this function will alter both sys.path and os.environ.
+    """
+    if altinstall and default_pip:
+        raise ValueError("Cannot use altinstall and default_pip together")
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # By default, installing pip and setuptools installs all of the
+    # following scripts (X.Y == running Python version):
+    #
+    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
+    #
+    # pip 1.5+ allows ensurepip to request that some of those be left out
+    if altinstall:
+        # omit pip, pipX and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
+    elif not default_pip:
+        # omit pip and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "install"
+
+    tmpdir = tempfile.mkdtemp()
+    try:
+        # Put our bundled wheels into a temporary directory and construct the
+        # additional paths that need added to sys.path
+        additional_paths = []
+        for project, version in _PROJECTS:
+            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
+            whl = pkgutil.get_data(
+                "ensurepip",
+                "_bundled/{}".format(wheel_name),
+            )
+            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+                fp.write(whl)
+
+            additional_paths.append(os.path.join(tmpdir, wheel_name))
+
+        # Construct the arguments to be passed to the pip command
+        args = ["install", "--no-index", "--find-links", tmpdir]
+        if root:
+            args += ["--root", root]
+        if upgrade:
+            args += ["--upgrade"]
+        if user:
+            args += ["--user"]
+        if verbosity:
+            args += ["-" + "v" * verbosity]
+
+        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def _uninstall_helper(verbosity=0):
+    """Helper to support a clean default uninstall process on Windows
+
+    Note that calling this function may alter os.environ.
+    """
+    # Nothing to do if pip was never installed, or has been removed
+    try:
+        import pip
+    except ImportError:
+        return
+
+    # If the pip version doesn't match the bundled one, leave it alone
+    if pip.__version__ != _PIP_VERSION:
+        msg = ("ensurepip will only uninstall a matching version "
+               "({!r} installed, {!r} bundled)")
+        print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr)
+        return
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # Construct the arguments to be passed to the pip command
+    args = ["uninstall", "-y"]
+    if verbosity:
+        args += ["-" + "v" * verbosity]
+
+    _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+
+
+def _main(argv=None):
+    if ssl is None:
+        print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE),
+              file=sys.stderr)
+        return
+
+    import argparse
+    parser = argparse.ArgumentParser(prog="python -m ensurepip")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(version()),
+        help="Show the version of pip that is bundled with this Python.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+    parser.add_argument(
+        "-U", "--upgrade",
+        action="store_true",
+        default=False,
+        help="Upgrade pip and dependencies, even if already installed.",
+    )
+    parser.add_argument(
+        "--user",
+        action="store_true",
+        default=False,
+        help="Install using the user scheme.",
+    )
+    parser.add_argument(
+        "--root",
+        default=None,
+        help="Install everything relative to this alternate root directory.",
+    )
+    parser.add_argument(
+        "--altinstall",
+        action="store_true",
+        default=False,
+        help=("Make an alternate install, installing only the X.Y versioned"
+              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"),
+    )
+    parser.add_argument(
+        "--default-pip",
+        action="store_true",
+        default=True,
+        dest="default_pip",
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-default-pip",
+        action="store_false",
+        dest="default_pip",
+        help=("Make a non default install, installing only the X and X.Y "
+              "versioned scripts."),
+    )
+
+    args = parser.parse_args(argv)
+
+    bootstrap(
+        root=args.root,
+        upgrade=args.upgrade,
+        user=args.user,
+        verbosity=args.verbosity,
+        altinstall=args.altinstall,
+        default_pip=args.default_pip,
+    )
diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/__main__.py
@@ -0,0 +1,4 @@
+import ensurepip
+
+if __name__ == "__main__":
+    ensurepip._main()
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/_uninstall.py
@@ -0,0 +1,30 @@
+"""Basic pip uninstallation support, helper for the Windows uninstaller"""
+
+import argparse
+import ensurepip
+
+
+def _main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(ensurepip.version()),
+        help="Show the version of pip this will attempt to uninstall.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+
+    args = parser.parse_args(argv)
+
+    ensurepip._uninstall_helper(verbosity=args.verbosity)
+
+
+if __name__ == "__main__":
+    _main()
diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py
--- a/lib-python/2.7/glob.py
+++ b/lib-python/2.7/glob.py
@@ -35,11 +35,16 @@
     patterns.
 
     """
+    dirname, basename = os.path.split(pathname)
     if not has_magic(pathname):
-        if os.path.lexists(pathname):
-            yield pathname
+        if basename:
+            if os.path.lexists(pathname):
+                yield pathname
+        else:
+            # Patterns ending with a slash should match only directories
+            if os.path.isdir(dirname):
+                yield pathname
         return
-    dirname, basename = os.path.split(pathname)
     if not dirname:
         for name in glob1(os.curdir, basename):
             yield name
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -164,9 +164,16 @@
     def _write_gzip_header(self):
         self.fileobj.write('\037\213')             # magic header
         self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
+        try:
+            # RFC 1952 requires the FNAME field to be Latin-1. Do not
+            # include filenames that cannot be represented that way.
+            fname = os.path.basename(self.name)
+            if not isinstance(fname, str):
+                fname = fname.encode('latin-1')
+            if fname.endswith('.gz'):
+                fname = fname[:-3]
+        except UnicodeEncodeError:
+            fname = ''
         flags = 0
         if fname:
             flags = FNAME
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -15,8 +15,9 @@
 
 md5(), sha1(), sha224(), sha256(), sha384(), and sha512()
 
-More algorithms may be available on your platform but the above are
-guaranteed to exist.
+More algorithms may be available on your platform but the above are guaranteed
+to exist.  See the algorithms_guaranteed and algorithms_available attributes
+to find out what algorithm names can be passed to new().
 
 NOTE: If you want the adler32 or crc32 hash functions they are available in
 the zlib module.
@@ -58,9 +59,14 @@
 # always available algorithm is added.
 __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
+algorithms_guaranteed = set(__always_supported)
+algorithms_available = set(__always_supported)
+
 algorithms = __always_supported
 
-__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac')
+__all__ = __always_supported + ('new', 'algorithms_guaranteed',
+                                'algorithms_available', 'algorithms',
+                                'pbkdf2_hmac')
 
 
 def __get_builtin_constructor(name):
@@ -128,6 +134,8 @@
     import _hashlib
     new = __hash_new
     __get_hash = __get_openssl_constructor
+    algorithms_available = algorithms_available.union(
+        _hashlib.openssl_md_meth_names)
 except ImportError:
     new = __py_new
     __get_hash = __get_builtin_constructor
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -215,6 +215,10 @@
 # maximal line length when calling readline().
 _MAXLINE = 65536
 
+# maximum amount of headers accepted
+_MAXHEADERS = 100
+
+
 class HTTPMessage(mimetools.Message):
 
     def addheader(self, key, value):
@@ -271,6 +275,8 @@
         elif self.seekable:
             tell = self.fp.tell
         while True:
+            if len(hlist) > _MAXHEADERS:
+                raise HTTPException("got more than %d headers" % _MAXHEADERS)
             if tell:
                 try:
                     startofline = tell()
@@ -1185,21 +1191,29 @@
 
         def __init__(self, host, port=None, key_file=None, cert_file=None,
                      strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-                     source_address=None):
+                     source_address=None, context=None):
             HTTPConnection.__init__(self, host, port, strict, timeout,
                                     source_address)
             self.key_file = key_file
             self.cert_file = cert_file
+            if context is None:
+                context = ssl._create_default_https_context()
+            if key_file or cert_file:
+                context.load_cert_chain(cert_file, key_file)
+            self._context = context
 
         def connect(self):
             "Connect to a host on a given (SSL) port."
 
-            sock = self._create_connection((self.host, self.port),
-                                          self.timeout, self.source_address)
+            HTTPConnection.connect(self)
+
             if self._tunnel_host:
-                self.sock = sock
-                self._tunnel()
-            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                server_hostname = self._tunnel_host
+            else:
+                server_hostname = self.host
+
+            self.sock = self._context.wrap_socket(self.sock,
+                                                  server_hostname=server_hostname)
 
     __all__.append("HTTPSConnection")
 
@@ -1214,14 +1228,15 @@
         _connection_class = HTTPSConnection
 
         def __init__(self, host='', port=None, key_file=None, cert_file=None,
-                     strict=None):
+                     strict=None, context=None):
             # provide a default host, pass the X509 cert info
 
             # urf. compensate for bad input.
             if port == 0:
                 port = None
             self._setup(self._connection_class(host, port, key_file,
-                                               cert_file, strict))
+                                               cert_file, strict,
+                                               context=context))
 
             # we never actually use these for anything, but we keep them
             # here for compatibility with post-1.5.2 CVS.
diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py
--- a/lib-python/2.7/idlelib/Bindings.py
+++ b/lib-python/2.7/idlelib/Bindings.py
@@ -75,7 +75,8 @@
    ('!_Auto-open Stack Viewer', '<<toggle-jit-stack-viewer>>'),
    ]),
  ('options', [
-   ('_Configure IDLE...', '<<open-config-dialog>>'),
+   ('Configure _IDLE', '<<open-config-dialog>>'),
+   ('Configure _Extensions', '<<open-config-extensions-dialog>>'),
    None,
    ]),
  ('help', [
diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py
--- a/lib-python/2.7/idlelib/CallTipWindow.py
+++ b/lib-python/2.7/idlelib/CallTipWindow.py
@@ -2,9 +2,8 @@
 
 After ToolTip.py, which uses ideas gleaned from PySol
 Used by the CallTips IDLE extension.
-
 """
-from Tkinter import *
+from Tkinter import Toplevel, Label, LEFT, SOLID, TclError
 
 HIDE_VIRTUAL_EVENT_NAME = "<<calltipwindow-hide>>"
 HIDE_SEQUENCES = ("<Key-Escape>", "<FocusOut>")
@@ -133,35 +132,28 @@
         return bool(self.tipwindow)
 
 
-def _calltip_window(parent):
-    root = Tk()
-    root.title("Test calltips")
-    width, height, x, y = list(map(int, re.split('[x+]', parent.geometry())))
-    root.geometry("+%d+%d"%(x, y + 150))
+def _calltip_window(parent):  # htest #
+    from Tkinter import Toplevel, Text, LEFT, BOTH
 
-    class MyEditWin: # comparenceptually an editor_window
-        def __init__(self):
-            text = self.text = Text(root)
-            text.pack(side=LEFT, fill=BOTH, expand=1)
-            text.insert("insert", "string.split")
-            root.update()
-            self.calltip = CallTip(text)
+    top = Toplevel(parent)
+    top.title("Test calltips")
+    top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200,
+                  parent.winfo_rooty() + 150))
+    text = Text(top)
+    text.pack(side=LEFT, fill=BOTH, expand=1)
+    text.insert("insert", "string.split")
+    top.update()
+    calltip = CallTip(text)
 
-            text.event_add("<<calltip-show>>", "(")
-            text.event_add("<<calltip-hide>>", ")")
-            text.bind("<<calltip-show>>", self.calltip_show)
-            text.bind("<<calltip-hide>>", self.calltip_hide)
-
-            text.focus_set()
-            root.mainloop()
-
-        def calltip_show(self, event):
-            self.calltip.showtip("Hello world", "insert", "end")
-
-        def calltip_hide(self, event):
-            self.calltip.hidetip()
-
-    editwin = MyEditWin()
+    def calltip_show(event):
+        calltip.showtip("(s=Hello world)", "insert", "end")
+    def calltip_hide(event):
+        calltip.hidetip()
+    text.event_add("<<calltip-show>>", "(")
+    text.event_add("<<calltip-hide>>", ")")
+    text.bind("<<calltip-show>>", calltip_show)
+    text.bind("<<calltip-hide>>", calltip_hide)
+    text.focus_set()
 
 if __name__=='__main__':
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py
--- a/lib-python/2.7/idlelib/ClassBrowser.py
+++ b/lib-python/2.7/idlelib/ClassBrowser.py
@@ -19,6 +19,9 @@
 from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas
 from idlelib.configHandler import idleConf
 
+file_open = None  # Method...Item and Class...Item use this.
+# Normally PyShell.flist.open, but there is no PyShell.flist for htest.
+
 class ClassBrowser:
 
     def __init__(self, flist, name, path, _htest=False):
@@ -27,6 +30,9 @@
         """
         _htest - bool, change box when location running htest.
         """
+        global file_open
+        if not _htest:
+            file_open = PyShell.flist.open
         self.name = name
         self.file = os.path.join(path[0], self.name + ".py")
         self._htest = _htest
@@ -101,7 +107,7 @@
             return []
         try:
             dict = pyclbr.readmodule_ex(name, [dir] + sys.path)
-        except ImportError, msg:
+        except ImportError:
             return []
         items = []
         self.classes = {}
@@ -170,7 +176,7 @@
     def OnDoubleClick(self):
         if not os.path.exists(self.file):
             return
-        edit = PyShell.flist.open(self.file)
+        edit = file_open(self.file)
         if hasattr(self.cl, 'lineno'):
             lineno = self.cl.lineno
             edit.gotoline(lineno)
@@ -206,7 +212,7 @@
     def OnDoubleClick(self):
         if not os.path.exists(self.file):
             return
-        edit = PyShell.flist.open(self.file)
+        edit = file_open(self.file)
         edit.gotoline(self.cl.methods[self.name])
 
 def _class_browser(parent): #Wrapper for htest
@@ -221,8 +227,9 @@
     dir, file = os.path.split(file)
     name = os.path.splitext(file)[0]
     flist = PyShell.PyShellFileList(parent)
+    global file_open
+    file_open = flist.open
     ClassBrowser(flist, name, [dir], _htest=True)
-    parent.mainloop()
 
 if __name__ == "__main__":
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py
--- a/lib-python/2.7/idlelib/ColorDelegator.py
+++ b/lib-python/2.7/idlelib/ColorDelegator.py
@@ -2,7 +2,6 @@
 import re
 import keyword
 import __builtin__
-from Tkinter import *
 from idlelib.Delegator import Delegator
 from idlelib.configHandler import idleConf
 
@@ -34,7 +33,6 @@
 
 prog = re.compile(make_pat(), re.S)
 idprog = re.compile(r"\s+(\w+)", re.S)
-asprog = re.compile(r".*?\b(as)\b")
 
 class ColorDelegator(Delegator):
 
@@ -42,7 +40,6 @@
         Delegator.__init__(self)
         self.prog = prog
         self.idprog = idprog
-        self.asprog = asprog
         self.LoadTagDefs()
 
     def setdelegate(self, delegate):
@@ -74,7 +71,6 @@
             "DEFINITION": idleConf.GetHighlight(theme, "definition"),
             "SYNC": {'background':None,'foreground':None},
             "TODO": {'background':None,'foreground':None},
-            "BREAK": idleConf.GetHighlight(theme, "break"),
             "ERROR": idleConf.GetHighlight(theme, "error"),
             # The following is used by ReplaceDialog:
             "hit": idleConf.GetHighlight(theme, "hit"),
@@ -216,22 +212,6 @@
                                     self.tag_add("DEFINITION",
                                                  head + "+%dc" % a,
                                                  head + "+%dc" % b)
-                            elif value == "import":
-                                # color all the "as" words on same line, except
-                                # if in a comment; cheap approximation to the
-                                # truth
-                                if '#' in chars:
-                                    endpos = chars.index('#')
-                                else:
-                                    endpos = len(chars)
-                                while True:
-                                    m1 = self.asprog.match(chars, b, endpos)
-                                    if not m1:
-                                        break
-                                    a, b = m1.span(1)
-                                    self.tag_add("KEYWORD",
-                                                 head + "+%dc" % a,
-                                                 head + "+%dc" % b)
                     m = self.prog.search(chars, m.end())
                 if "SYNC" in self.tag_names(next + "-1c"):
                     head = next
@@ -255,20 +235,23 @@
         for tag in self.tagdefs.keys():
             self.tag_remove(tag, "1.0", "end")
 
-def _color_delegator(parent):
+def _color_delegator(parent):  # htest #
+    from Tkinter import Toplevel, Text
     from idlelib.Percolator import Percolator
-    root = Tk()
-    root.title("Test ColorDelegator")
-    width, height, x, y = list(map(int, re.split('[x+]', parent.geometry())))
-    root.geometry("+%d+%d"%(x, y + 150))
-    source = "if somename: x = 'abc' # comment\nprint"
-    text = Text(root, background="white")
+
+    top = Toplevel(parent)
+    top.title("Test ColorDelegator")
+    top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200,
+                  parent.winfo_rooty() + 150))
+    source = "if somename: x = 'abc' # comment\nprint\n"
+    text = Text(top, background="white")
+    text.pack(expand=1, fill="both")
     text.insert("insert", source)
-    text.pack(expand=1, fill="both")
+    text.focus_set()
+
     p = Percolator(text)
     d = ColorDelegator()
     p.insertfilter(d)
-    root.mainloop()
 
 if __name__ == "__main__":
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/Debugger.py b/lib-python/2.7/idlelib/Debugger.py
--- a/lib-python/2.7/idlelib/Debugger.py
+++ b/lib-python/2.7/idlelib/Debugger.py
@@ -1,6 +1,5 @@
 import os
 import bdb
-import types
 from Tkinter import *
 from idlelib.WindowList import ListedToplevel
 from idlelib.ScrolledList import ScrolledList
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -1,6 +1,6 @@
 import sys
 import os
-from platform import python_version
+import platform
 import re
 import imp
 from Tkinter import *
@@ -22,6 +22,8 @@
 # The default tab setting for a Text widget, in average-width characters.
 TK_TABWIDTH_DEFAULT = 8
 
+_py_version = ' (%s)' % platform.python_version()
+
 def _sphinx_version():
     "Format sys.version_info to produce the Sphinx version string used to install the chm docs"
     major, minor, micro, level, serial = sys.version_info
@@ -151,7 +153,7 @@
                     # Safari requires real file:-URLs
                     EditorWindow.help_url = 'file://' + EditorWindow.help_url
             else:
-                EditorWindow.help_url = "http://docs.python.org/%d.%d" % sys.version_info[:2]
+                EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2]
         currentTheme=idleConf.CurrentTheme()
         self.flist = flist
         root = root or flist.root
@@ -214,6 +216,8 @@
         text.bind("<<python-docs>>", self.python_docs)
         text.bind("<<about-idle>>", self.about_dialog)
         text.bind("<<open-config-dialog>>", self.config_dialog)
+        text.bind("<<open-config-extensions-dialog>>",
+                  self.config_extensions_dialog)
         text.bind("<<open-module>>", self.open_module)
         text.bind("<<do-nothing>>", lambda event: "break")
         text.bind("<<select-all>>", self.select_all)
@@ -568,6 +572,8 @@

From noreply at buildbot.pypy.org  Sun Dec 14 19:36:14 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 14 Dec 2014 19:36:14 +0100 (CET)
Subject: [pypy-commit] pypy default: Fixed typo in the stdlib upgrade guide
Message-ID: <20141214183614.70DCC1C3D55@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r74920:799a6016cf54
Date: 2014-12-14 10:36 -0800
http://bitbucket.org/pypy/pypy/changeset/799a6016cf54/

Log:	Fixed typo in the stdlib upgrade guide

diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt
--- a/lib-python/stdlib-upgrade.txt
+++ b/lib-python/stdlib-upgrade.txt
@@ -7,7 +7,7 @@
 
 1. check out the branch vendor/stdlib
 2. upgrade the files there
-3. update stdlib-versions.txt with the output of hg -id from the cpython repo
+3. update stdlib-version.txt with the output of hg -id from the cpython repo
 4. commit
 5. update to default/py3k
 6. create a integration branch for the new stdlib

From noreply at buildbot.pypy.org  Sun Dec 14 19:38:26 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 14 Dec 2014 19:38:26 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Attempted to resolve merge
	conflicts in ssl.py
Message-ID: <20141214183826.BF1E41C3D55@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74921:8c0194b34140
Date: 2014-12-14 10:38 -0800
http://bitbucket.org/pypy/pypy/changeset/8c0194b34140/

Log:	Attempted to resolve merge conflicts in ssl.py

diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py
--- a/lib-python/2.7/ssl.py
+++ b/lib-python/2.7/ssl.py
@@ -129,18 +129,12 @@
 except NameError:
     _SSLv2_IF_EXISTS = None
 
-<<<<<<< local
 from socket import socket, _fileobject, error as socket_error
-from socket import getnameinfo as _getnameinfo
-from socket import SOL_SOCKET, SO_TYPE, SOCK_STREAM
-=======
-from socket import socket, _fileobject, _delegate_methods, error as socket_error
 if sys.platform == "win32":
     from _ssl import enum_certificates, enum_crls
 
 from socket import socket, AF_INET, SOCK_STREAM, create_connection
 from socket import SOL_SOCKET, SO_TYPE
->>>>>>> other
 import base64        # for DER-to-PEM translation
 import errno
 
@@ -528,21 +522,12 @@
         if sock.getsockopt(SOL_SOCKET, SO_TYPE) != SOCK_STREAM:
             raise NotImplementedError("only stream sockets are supported")
         socket.__init__(self, _sock=sock._sock)
-<<<<<<< local
 
         # "close" the original socket: it is not usable any more. which should
         # not actually call the operating system's close() because the
         # reference counter is greater than 1 (we hold one too).
         sock._sock._drop()
-=======
-        # The initializer for socket overrides the methods send(), recv(), etc.
-        # in the instancce, which we don't need -- but we want to provide the
-        # methods defined in SSLSocket.
-        for attr in _delegate_methods:
-            try:
-                delattr(self, attr)
-            except AttributeError:
-                pass
+
         if server_side and server_hostname:
             raise ValueError("server_hostname can only be specified "
                              "in client mode")
@@ -552,7 +537,6 @@
         self.server_hostname = server_hostname
         self.do_handshake_on_connect = do_handshake_on_connect
         self.suppress_ragged_eofs = suppress_ragged_eofs
->>>>>>> other
 
         # See if we are connected
         try:
@@ -864,7 +848,6 @@
         # the file-like object.
         return _fileobject(self, mode, bufsize, close=True)
 
-<<<<<<< local
     def _reuse(self):
         self._makefile_refs += 1
 
@@ -874,7 +857,6 @@
         else:
             self._makefile_refs -= 1
 
-=======
     def get_channel_binding(self, cb_type="tls-unique"):
         """Get channel binding data for current connection.  Raise ValueError
         if the requested `cb_type` is not supported.  Return bytes of the data
@@ -898,7 +880,6 @@
         if self._sslobj is None:
             return None
         return self._sslobj.version()
->>>>>>> other
 
 
 def wrap_socket(sock, keyfile=None, certfile=None,

From noreply at buildbot.pypy.org  Sun Dec 14 19:40:26 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 14 Dec 2014 19:40:26 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Resolved merge conflicts in
	test_ssl.py
Message-ID: <20141214184026.7284C1C1066@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74922:5eed2065b581
Date: 2014-12-14 10:40 -0800
http://bitbucket.org/pypy/pypy/changeset/5eed2065b581/

Log:	Resolved merge conflicts in test_ssl.py

diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -1995,51 +1995,6 @@
 
     class ThreadedTests(unittest.TestCase):
 
-<<<<<<< local
-        def test_rude_shutdown(self):
-            """A brutal shutdown of an SSL server should raise an IOError
-            in the client when attempting handshake.
-            """
-            listener_ready = threading.Event()
-            listener_gone = threading.Event()
-
-            s = socket.socket()
-            port = test_support.bind_port(s, HOST)
-
-            # `listener` runs in a thread.  It sits in an accept() until
-            # the main thread connects.  Then it rudely closes the socket,
-            # and sets Event `listener_gone` to let the main thread know
-            # the socket is gone.
-            def listener():
-                s.listen(5)
-                listener_ready.set()
-                s.accept()
-                s.close()
-                listener_gone.set()
-
-            def connector():
-                listener_ready.wait()
-                c = socket.socket()
-                c.connect((HOST, port))
-                listener_gone.wait()
-                # XXX why is it necessary?
-                test_support.gc_collect()
-                try:
-                    ssl_sock = ssl.wrap_socket(c)
-                except IOError:
-                    pass
-                else:
-                    self.fail('connecting to closed SSL socket should have failed')
-
-            t = threading.Thread(target=listener)
-            t.start()
-            try:
-                connector()
-            finally:
-                t.join()
-
-=======
->>>>>>> other
         @skip_if_broken_ubuntu_ssl
         def test_echo(self):
             """Basic test of an SSL client connecting to a server"""

From noreply at buildbot.pypy.org  Sun Dec 14 19:42:22 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Dec 2014 19:42:22 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Add method TkApp.merge()
Message-ID: <20141214184222.B7C551C1066@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r74923:97d75d169b30
Date: 2014-10-30 22:18 +0100
http://bitbucket.org/pypy/pypy/changeset/97d75d169b30/

Log:	Add method TkApp.merge()

diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -8,6 +8,7 @@
 import sys
 import threading
 import time
+import warnings
 
 
 class _DummyLock(object):
@@ -326,7 +327,7 @@
             if res == tklib.TCL_ERROR:
                 self.raiseTclError()
             result = tkffi.string(tklib.Tcl_GetStringResult(self.interp))
-            return result.decode('utf-8')
+            return FromTclString(result)
 
     def evalfile(self, filename):
         self._check_tcl_appartment()
@@ -335,7 +336,7 @@
             if res == tklib.TCL_ERROR:
                 self.raiseTclError()
             result = tkffi.string(tklib.Tcl_GetStringResult(self.interp))
-            return result.decode('utf-8')
+            return FromTclString(result)
 
     def split(self, arg):
         if isinstance(arg, TclObject):
@@ -435,6 +436,32 @@
         finally:
             tklib.Tcl_Free(argv[0])
 
+    def merge(self, *args):
+        warnings.warn("merge is deprecated and will be removed in 3.4",
+                      DeprecationWarning)
+        s = self._merge(args)
+        return s.decode('utf-8')
+
+    def _merge(self, args):
+        argv = []
+        for arg in args:
+            if isinstance(arg, tuple):
+                argv.append(self._merge(arg))
+            elif arg is None:
+                break
+            elif isinstance(arg, bytes):
+                argv.append(arg)
+            else:
+                argv.append(str(arg).encode('utf-8'))
+        argv_array = [tkffi.new("char[]", arg) for arg in argv]
+        res = tklib.Tcl_Merge(len(argv), argv_array)
+        if not res:
+            raise TclError("merge failed")
+        try:
+            return tkffi.string(res)
+        finally:
+            tklib.Tcl_Free(res)
+
     def getboolean(self, s):
         if isinstance(s, int):
             return s
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -19,7 +19,7 @@
     except UnicodeDecodeError:
         # Tcl encodes null character as \xc0\x80
         try:
-            return s.replace('\xc0\x80', '\x00').decode('utf-8')
+            return s.replace(b'\xc0\x80', b'\x00').decode('utf-8')
         except UnicodeDecodeError:
             pass
     return s
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -89,6 +89,7 @@
 int Tcl_ListObjLength(Tcl_Interp* interp, Tcl_Obj* listPtr, int* intPtr);
 int Tcl_ListObjIndex(Tcl_Interp* interp, Tcl_Obj* listPtr, int index, Tcl_Obj** objPtrPtr);
 int Tcl_SplitList(Tcl_Interp* interp, char* list, int* argcPtr, const char*** argvPtr);
+char* Tcl_Merge(int argc, char** argv);
 
 int Tcl_Eval(Tcl_Interp* interp, const char* script);
 int Tcl_EvalFile(Tcl_Interp* interp, const char* filename);

From noreply at buildbot.pypy.org  Sun Dec 14 19:42:23 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Dec 2014 19:42:23 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: In import errors,
	hide frames from importlib._bootstrap
Message-ID: <20141214184223.EE66B1C1066@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r74924:97034435e2d0
Date: 2014-11-01 10:54 +0100
http://bitbucket.org/pypy/pypy/changeset/97034435e2d0/

Log:	In import errors, hide frames from importlib._bootstrap

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -320,6 +320,15 @@
         """
         self._application_traceback = traceback
 
+    def remove_traceback_module_frames(self, module_name):
+        from pypy.interpreter.pytraceback import PyTraceback
+        tb = self._application_traceback
+        while tb is not None and isinstance(tb, PyTraceback):
+            if tb.frame.pycode.co_filename != module_name:
+                break
+            tb = tb.next
+        self._application_traceback = tb
+
     def record_context(self, space, frame):
         """Record a __context__ for this exception from the current
         frame if one exists.
diff --git a/pypy/module/_frozen_importlib/__init__.py b/pypy/module/_frozen_importlib/__init__.py
--- a/pypy/module/_frozen_importlib/__init__.py
+++ b/pypy/module/_frozen_importlib/__init__.py
@@ -1,6 +1,7 @@
 import os
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.module.sys import initpath
+from pypy.module._frozen_importlib import interp_import
 
 lib_python = os.path.join(os.path.dirname(__file__),
                           '..', '..', '..', 'lib-python', '3')
@@ -29,11 +30,12 @@
                       space.wrap(space.builtin))
         code_w.exec_code(space, self.w_dict, self.w_dict)
 
+        self.w_import = space.wrap(interp_import.import_with_frames_removed)
+
     def startup(self, space):
         """Copy our __import__ to builtins."""
         w_install = self.getdictvalue(space, '_install')
         space.call_function(w_install,
                             space.getbuiltinmodule('sys'),
                             space.getbuiltinmodule('_imp'))
-        self.space.builtin.setdictvalue(space, '__import__',
-                                        self.getdictvalue(space, '__import__'))
+        self.space.builtin.setdictvalue(space, '__import__', self.w_import)
diff --git a/pypy/module/_frozen_importlib/interp_import.py b/pypy/module/_frozen_importlib/interp_import.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_frozen_importlib/interp_import.py
@@ -0,0 +1,12 @@
+from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.error import OperationError
+
+ at interp2app
+def import_with_frames_removed(space, __args__):
+    try:
+        return space.call_args(
+            space.getbuiltinmodule('_frozen_importlib').getdictvalue(
+                space, '__import__'), __args__)
+    except OperationError as e:
+        e.remove_traceback_module_frames('<frozen importlib._bootstrap>')
+        raise

From noreply at buildbot.pypy.org  Sun Dec 14 19:42:25 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Dec 2014 19:42:25 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Start to define posix._have_functions
Message-ID: <20141214184225.30AA21C1066@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r74925:e563bf673157
Date: 2014-11-01 12:01 +0100
http://bitbucket.org/pypy/pypy/changeset/e563bf673157/

Log:	Start to define posix._have_functions

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -188,7 +188,8 @@
 
     # os.py uses this list to build os.supports_dir_fd() and os.supports_fd().
     # Fill with e.g. HAVE_FCHDIR, when os.chdir() supports file descriptors.
-    interpleveldefs['_have_functions'] = 'space.newlist([])'
+    interpleveldefs['_have_functions'] = (
+        'space.newlist([space.wrap(x) for x in interp_posix.have_functions])')
 
     def startup(self, space):
         from pypy.module.posix import interp_posix
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1436,3 +1436,8 @@
         except OSError as e:
             raise wrap_oserror2(space, e, w_path)
         return space.wrap(result)
+
+have_functions = []
+for name in """FSTAT FCHDIR OPENAT""".split():
+    if getattr(rposix, "HAVE_%s" % name):
+        have_functions.append("HAVE_%s" % name)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1248,3 +1248,16 @@
         f.write(source)
         child = self.spawn([str(f)])
         child.expect('ok!')
+
+
+class AppTestFdVariants:
+    # Tests variant functions which also accept file descriptors,
+    # dir_fd and follow_symlinks.
+    def test_have_functions(self):
+        import os
+        assert os.stat in os.supports_fd  # fstat() is supported everywhere
+        if os.name != 'nt':
+            assert os.chdir in os.supports_fd  # fchdir()
+        if os.name == 'posix':
+            assert os.open in os.supports_dir_fd  # openat()
+
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1,5 +1,6 @@
 import os
 from rpython.rtyper.lltypesystem.rffi import CConstant, CExternVariable, INT
+from rpython.rtyper.tool import rffi_platform
 from rpython.rtyper.lltypesystem import ll2ctypes, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rlib.rarithmetic import intmask
@@ -8,6 +9,19 @@
 from rpython.translator.platform import platform
 
 
+class CConfig:
+    _compilation_info_ = ExternalCompilationInfo(
+        includes=['sys/stat.h', 
+                  'unistd.h',
+                  'fcntl.h'],
+    )
+    HAVE_FSTAT = rffi_platform.Has('fstat')
+    HAVE_FCHDIR = rffi_platform.Has('fchdir')
+    HAVE_OPENAT = rffi_platform.Has('openat')
+cConfig = rffi_platform.configure(CConfig)
+globals().update(cConfig)
+
+
 class CConstantErrno(CConstant):
     # these accessors are used when calling get_errno() or set_errno()
     # on top of CPython

From noreply at buildbot.pypy.org  Sun Dec 14 19:42:26 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Dec 2014 19:42:26 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Add a negative test for win32
Message-ID: <20141214184226.64EC61C1066@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r74926:29979a710c73
Date: 2014-11-01 12:02 +0100
http://bitbucket.org/pypy/pypy/changeset/29979a710c73/

Log:	Add a negative test for win32

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1258,6 +1258,8 @@
         assert os.stat in os.supports_fd  # fstat() is supported everywhere
         if os.name != 'nt':
             assert os.chdir in os.supports_fd  # fchdir()
+        else:
+            assert os.chdir not in os.supports_fd
         if os.name == 'posix':
             assert os.open in os.supports_dir_fd  # openat()
 

From noreply at buildbot.pypy.org  Sun Dec 14 19:42:27 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Dec 2014 19:42:27 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Try to fix the encoding bootstrap issue.
Message-ID: <20141214184227.A693D1C1066@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r74927:7470b99dce17
Date: 2014-12-14 19:12 +0100
http://bitbucket.org/pypy/pypy/changeset/7470b99dce17/

Log:	Try to fix the encoding bootstrap issue. Now pypy starts with
	LANG=C.

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -162,12 +162,13 @@
 def _lookup_codec_loop(space, encoding, normalized_encoding):
     state = space.fromcache(CodecState)
     if state.codec_need_encodings:
+        # registers new codecs.
+        # This import uses the "builtin" import method, and is needed
+        # to bootstrap the full importlib module.
         w_import = space.getattr(space.builtin, space.wrap("__import__"))
-        # registers new codecs
         space.call_function(w_import, space.wrap("encodings"))
-        # XXX needed to bootstrap the importlib module.
-        # I hope we don't have to import all encodings here.
-        space.call_function(w_import, space.wrap("encodings.utf_8"))
+        from pypy.module.sys.interp_encoding import base_encoding
+        space.call_function(w_import, space.wrap("encodings." + base_encoding))
         state.codec_need_encodings = False
         if len(state.codec_search_path) == 0:
             raise OperationError(

From noreply at buildbot.pypy.org  Sun Dec 14 19:42:30 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Dec 2014 19:42:30 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: hg merge heads
Message-ID: <20141214184230.BD4471C1066@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r74928:362d1c6f3870
Date: 2014-12-14 19:41 +0100
http://bitbucket.org/pypy/pypy/changeset/362d1c6f3870/

Log:	hg merge heads

diff too long, truncating to 2000 out of 4604 lines

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -250,10 +250,6 @@
         BoolOption("optimized_list_getitem",
                    "special case the 'list[integer]' expressions",
                    default=False),
-        BoolOption("builtinshortcut",
-                   "a shortcut for operations between built-in types. XXX: "
-                   "deprecated, not really a shortcut any more.",
-                   default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
                    default=False,
@@ -265,9 +261,6 @@
                    # weakrefs needed, because of get_subclasses()
                    requires=[("translation.rweakref", True)]),
 
-        ChoiceOption("multimethods", "the multimethod implementation to use",
-                     ["doubledispatch", "mrd"],
-                     default="mrd"),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
                    default=False,
diff --git a/pypy/doc/config/objspace.std.builtinshortcut.txt b/pypy/doc/config/objspace.std.builtinshortcut.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.std.builtinshortcut.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-A shortcut speeding up primitive operations between built-in types.
-
-This is a space-time trade-off: at the moment, this option makes a
-translated pypy-c executable bigger by about 1.7 MB.  (This can probably
-be improved with careful analysis.)
diff --git a/pypy/doc/config/objspace.std.multimethods.txt b/pypy/doc/config/objspace.std.multimethods.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.std.multimethods.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Choose the multimethod implementation.
-
-* ``doubledispatch`` turns
-  a multimethod call into a sequence of normal method calls.
-
-* ``mrd`` uses a technique known as Multiple Row Displacement
-  which precomputes a few compact tables of numbers and
-  function pointers.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -35,3 +35,7 @@
 Split RPython documentation from PyPy documentation and clean up.  There now is
 a clearer separation between documentation for users, developers and people
 interested in background information.
+
+.. branch: kill-multimethod
+
+Kill multimethod machinery, all multimethods were removed earlier.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -215,23 +215,6 @@
         from pypy.config.pypyoption import set_pypy_opt_level
         set_pypy_opt_level(config, translateconfig.opt)
 
-        # as of revision 27081, multimethod.py uses the InstallerVersion1 by default
-        # because it is much faster both to initialize and run on top of CPython.
-        # The InstallerVersion2 is optimized for making a translator-friendly
-        # structure for low level backends. However, InstallerVersion1 is still
-        # preferable for high level backends, so we patch here.
-
-        from pypy.objspace.std import multimethod
-        if config.objspace.std.multimethods == 'mrd':
-            assert multimethod.InstallerVersion1.instance_counter == 0,\
-                   'The wrong Installer version has already been instatiated'
-            multimethod.Installer = multimethod.InstallerVersion2
-        elif config.objspace.std.multimethods == 'doubledispatch':
-            # don't rely on the default, set again here
-            assert multimethod.InstallerVersion2.instance_counter == 0,\
-                   'The wrong Installer version has already been instatiated'
-            multimethod.Installer = multimethod.InstallerVersion1
-
     def print_help(self, config):
         self.opt_parser(config).print_help()
 
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -630,6 +630,7 @@
 from pypy.interpreter.nestedscope import Cell
 from pypy.interpreter.special import NotImplemented, Ellipsis
 
+
 def descr_get_dict(space, w_obj):
     w_dict = w_obj.getdict(space)
     if w_dict is None:
@@ -650,6 +651,11 @@
         return space.w_None
     return lifeline.get_any_weakref(space)
 
+dict_descr = GetSetProperty(descr_get_dict, descr_set_dict, descr_del_dict,
+                            doc="dictionary for instance variables (if defined)")
+dict_descr.name = '__dict__'
+
+
 def generic_ne(space, w_obj1, w_obj2):
     if space.eq_w(w_obj1, w_obj2):
         return space.w_False
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -52,16 +52,6 @@
 class BlockingIOError(Exception):
     pass
 
-def trap_EINTR(space, e):
-    """Return True if an EnvironmentError with errno == EINTR is set."""
-    if not e.match(space, space.w_EnvironmentError):
-        return False
-    w_value = e.get_w_value(space)
-    w_errno = space.getattr(w_value, space.wrap("errno"))
-    if space.eq_w(w_errno, space.wrap(errno.EINTR)):
-        return True
-    return False
-
 class W_BufferedIOBase(W_IOBase):
     def _check_init(self, space):
         raise NotImplementedError
@@ -816,11 +806,6 @@
         self._check_closed(space, "flush of closed file")
         with self.lock:
             self._flush_and_rewind_unlocked(space)
-            if self.readable:
-                # Rewind the raw stream so that its position corresponds to
-                # the current logical position.
-                self._raw_seek(space, -self._raw_offset(), 1)
-                self._reader_reset_buf()
 
     def _flush_and_rewind_unlocked(self, space):
         self._writer_flush_unlocked(space)
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -24,8 +24,7 @@
     try:
         w_value = error.get_w_value(space)
         w_errno = space.getattr(w_value, space.wrap("errno"))
-        return space.is_true(
-            space.eq(w_errno, space.wrap(EINTR)))
+        return space.eq_w(w_errno, space.wrap(EINTR))
     except OperationError:
         return False
 
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -390,3 +390,13 @@
                 assert mod == 'io'
             else:
                 assert mod == '_io'
+
+    def test_issue1902(self):
+        import _io
+        with _io.open(self.tmpfile, 'w+b', 4096) as f:
+            f.write(b'\xff' * 13569)
+            f.flush()
+            f.seek(0, 0)
+            f.read(1)
+            f.seek(-1, 1)
+            f.write(b'')
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,5 +1,3 @@
-from __future__ import with_statement
-
 from rpython.rlib import jit
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.objectmodel import keepalive_until_here
@@ -15,7 +13,6 @@
     interp2app, interpindirect2app, unwrap_spec)
 from pypy.interpreter.typedef import (
     GetSetProperty, TypeDef, make_weakref_descr)
-from pypy.objspace.std.floatobject import W_FloatObject
 
 
 @unwrap_spec(typecode=str)
@@ -691,7 +688,7 @@
             try:
                 item = unwrap(w_item)
             except OperationError, e:
-                if isinstance(w_item, W_FloatObject):
+                if space.isinstance_w(w_item, space.w_float):
                     # Odd special case from cpython
                     raise
                 if mytype.method != '' and e.match(space, space.w_TypeError):
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -1038,11 +1038,6 @@
         assert bytes(memoryview(self.array('i'))) == b''
 
 
-class AppTestArrayBuiltinShortcut(AppTestArray):
-    spaceconfig = AppTestArray.spaceconfig.copy()
-    spaceconfig['objspace.std.builtinshortcut'] = True
-
-
 class AppTestArrayReconstructor:
     spaceconfig = dict(usemodules=('array', 'struct'))
 
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -303,11 +303,6 @@
 
 @bootstrap_function
 def init_typeobject(space):
-    # Probably a hack
-    space.model.typeorder[W_PyCTypeObject] = [(W_PyCTypeObject, None),
-                                              (W_TypeObject, None),
-                                              (W_Root, None)]
-
     make_typedescr(space.w_type.instancetypedef,
                    basestruct=PyTypeObject,
                    alloc=type_alloc,
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -2,7 +2,6 @@
 from pypy.interpreter import argument, gateway
 from pypy.interpreter.baseobjspace import W_Root, ObjSpace, SpaceCache
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.objspace.std.stdtypedef import StdTypeDef
 from pypy.objspace.std.sliceobject import W_SliceObject
 from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.objectmodel import instantiate, we_are_translated, specialize
@@ -113,6 +112,10 @@
 # ____________________________________________________________
 
 
+BUILTIN_TYPES = ['int', 'str', 'float', 'tuple', 'list', 'dict', 'bytes',
+                 'unicode', 'complex', 'slice', 'bool', 'text', 'object',
+                 'bytearray', 'memoryview']
+
 class FakeObjSpace(ObjSpace):
     def __init__(self, config=None):
         self._seen_extras = []
@@ -352,9 +355,7 @@
     def setup(space):
         for name in (ObjSpace.ConstantTable +
                      ObjSpace.ExceptionTable +
-                     ['int', 'str', 'float', 'tuple', 'list',
-                      'dict', 'bytes', 'complex', 'slice', 'bool',
-                      'text', 'object', 'unicode', 'bytearray', 'memoryview']):
+                     BUILTIN_TYPES):
             setattr(space, 'w_' + name, w_some_obj())
         space.w_type = w_some_type()
         #
@@ -385,7 +386,7 @@
 @specialize.memo()
 def see_typedef(space, typedef):
     assert isinstance(typedef, TypeDef)
-    if not isinstance(typedef, StdTypeDef):
+    if typedef.name not in BUILTIN_TYPES:
         for name, value in typedef.rawdict.items():
             space.wrap(value)
 
diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py
--- a/pypy/objspace/std/boolobject.py
+++ b/pypy/objspace/std/boolobject.py
@@ -6,8 +6,8 @@
 from rpython.tool.sourcetools import func_renamer, func_with_new_name
 
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
+from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.intobject import W_AbstractIntObject, W_IntObject
-from pypy.objspace.std.stdtypedef import StdTypeDef
 
 
 class W_BoolObject(W_IntObject):
@@ -80,7 +80,7 @@
 W_BoolObject.w_True = W_BoolObject(True)
 
 
-W_BoolObject.typedef = StdTypeDef("bool", W_IntObject.typedef,
+W_BoolObject.typedef = TypeDef("bool", W_IntObject.typedef,
     __doc__ = """bool(x) -> bool
 
 Returns True when the argument x is true, False otherwise.
diff --git a/pypy/objspace/std/builtinshortcut.py b/pypy/objspace/std/builtinshortcut.py
deleted file mode 100644
--- a/pypy/objspace/std/builtinshortcut.py
+++ /dev/null
@@ -1,132 +0,0 @@
-from pypy.interpreter.baseobjspace import ObjSpace
-from pypy.interpreter.error import OperationError
-from pypy.objspace.descroperation import DescrOperation
-from pypy.objspace.std.multimethod import FailedToImplement
-from pypy.objspace.std.boolobject import W_BoolObject
-from rpython.tool.sourcetools import func_with_new_name
-
-# ____________________________________________________________
-#
-#  The sole purpose of this file is performance.
-#  It speeds up the dispatch of operations between
-#  built-in objects.
-#
-
-# this is a selection... a few operations are missing because they are
-# thought to be very rare or most commonly used with non-builtin types
-METHODS_WITH_SHORTCUT = dict.fromkeys(
-    ['add', 'sub', 'mul', 'truediv', 'floordiv', 'div',
-     'mod', 'lshift', 'rshift', 'and_', 'or_', 'xor', 'pow',
-     'lt', 'le', 'eq', 'ne', 'gt', 'ge', 'contains',
-     # unary
-     'len', 'nonzero', 'repr', 'str', 'hash',
-     'neg', 'invert', 'index', 'iter', 'next', 'buffer',
-     'getitem', 'setitem', 'int',
-     # in-place
-     'inplace_add', 'inplace_sub', 'inplace_mul', 'inplace_truediv',
-     'inplace_floordiv', 'inplace_div', 'inplace_mod', 'inplace_pow',
-     'inplace_lshift', 'inplace_rshift', 'inplace_and', 'inplace_or',
-     'inplace_xor',
-     # other
-     'format',
- ])
-
-KNOWN_MISSING = ['getattr',   # mostly non-builtins or optimized by CALL_METHOD
-                 'setattr', 'delattr', 'userdel',  # mostly for non-builtins
-                 'get', 'set', 'delete',   # uncommon (except on functions)
-                 'delitem', 'trunc',              # rare stuff?
-                 'abs',                           # rare stuff?
-                 'pos', 'divmod',                 # rare stuff?
-                 'float', 'long',                 # rare stuff?
-                 'isinstance', 'issubtype',
-                 ]
-
-for _name, _, _, _specialmethods in ObjSpace.MethodTable:
-    if _specialmethods:
-        assert _name in METHODS_WITH_SHORTCUT or _name in KNOWN_MISSING, (
-            "operation %r should be in METHODS_WITH_SHORTCUT or KNOWN_MISSING"
-            % (_name,))
-
-
-def filter_out_conversions(typeorder):
-    res = {}
-    for cls, order in typeorder.iteritems():
-        res[cls] = [(target_type, converter) for (target_type, converter) in
-                                                 order if converter is None]
-    return res
-
-
-def install(space, mm, fallback_mm=None):
-    """Install a function <name>() on the space instance which invokes
-    a shortcut for built-in types.  Returns the shortcutting multimethod
-    object or None.
-    """
-    name = mm.name
-    if name not in METHODS_WITH_SHORTCUT:
-        return None
-
-    # can be called multiple times without re-installing
-    if name in space.__dict__:
-        mm1, shortcut_method = space.__dict__[name].builtinshortcut
-        assert mm1 is mm
-        return shortcut_method
-
-    #print 'installing shortcut for:', name
-    assert hasattr(DescrOperation, name)
-
-    base_method = getattr(space.__class__, name)
-
-    # Basic idea: we first try to dispatch the operation using purely
-    # the multimethod.  If this is done naively, subclassing a built-in
-    # type like 'int' and overriding a special method like '__add__'
-    # doesn't work any more, because the multimethod will accept the int
-    # subclass and compute the result in the built-in way.  To avoid
-    # this issue, we tweak the shortcut multimethods so that these ones
-    # (and only these ones) never match the interp-level subclasses
-    # built in pypy.interpreter.typedef.get_unique_interplevel_subclass.
-    expanded_order = space.model.get_typeorder_with_empty_usersubcls()
-    if fallback_mm:
-        mm = mm.merge_with(fallback_mm)
-    shortcut_method = mm.install_not_sliced(filter_out_conversions(expanded_order))
-
-    def operate(*args_w):
-        try:
-            return shortcut_method(space, *args_w)
-        except FailedToImplement:
-            pass
-        return base_method(space, *args_w)
-
-    operate = func_with_new_name(operate, name)
-    operate.builtinshortcut = (mm, shortcut_method)
-    setattr(space, name, operate)
-    return shortcut_method
-
-
-def install_is_true(space, mm_nonzero, mm_len):
-    shortcut = install(space, mm_nonzero, fallback_mm = mm_len)
-    assert 'is_true' not in space.__dict__
-
-    def is_true(w_obj):
-        # a bit of duplication of the logic from DescrOperation.is_true...
-        try:
-            w_res = shortcut(space, w_obj)
-        except FailedToImplement:
-            pass
-        else:
-            # the __bool__ method of built-in objects should
-            # always directly return a Bool; however, the __len__ method
-            # of built-in objects typically returns an unwrappable integer
-            if isinstance(w_res, W_BoolObject):
-                return bool(w_res.intval)
-            try:
-                return space.int_w(w_res) != 0
-            except OperationError:
-                # I think no OperationError other than w_OverflowError
-                # could occur here
-                w_obj = w_res
-
-        # general case fallback
-        return _DescrOperation_is_true(space, w_obj)
-
-    _DescrOperation_is_true = DescrOperation.is_true.im_func
-    space.is_true = is_true
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -11,8 +11,8 @@
 from pypy.objspace.std.bytesobject import (
     getbytevalue, makebytesdata_w, newbytesdata_w)
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
+from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.sliceobject import W_SliceObject
-from pypy.objspace.std.stdtypedef import StdTypeDef
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.util import get_positive_index
@@ -909,7 +909,7 @@
         """
 
 
-W_BytearrayObject.typedef = StdTypeDef(
+W_BytearrayObject.typedef = TypeDef(
     "bytearray",
     __doc__ = BytearrayDocstrings.__doc__,
     __new__ = interp2app(W_BytearrayObject.descr_new),
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -11,7 +11,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import (
     WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
-from pypy.objspace.std.stdtypedef import StdTypeDef
+from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.stringmethods import StringMethods
 
 
@@ -769,7 +769,7 @@
     return data
 
 
-W_BytesObject.typedef = StdTypeDef(
+W_BytesObject.typedef = TypeDef(
     "bytes",
     __new__ = interp2app(W_BytesObject.descr_new),
     __doc__ = """bytes(iterable_of_ints) -> bytes
diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -1,20 +1,21 @@
 import math
 
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from pypy.objspace.std import newformat
-from pypy.objspace.std.floatobject import _hash_float
-from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef
-from pypy.objspace.std.unicodeobject import unicode_to_decimal_w
 from rpython.rlib import jit, rcomplex
 from rpython.rlib.rarithmetic import intmask, r_ulonglong
 from rpython.rlib.rbigint import rbigint
 from rpython.rlib.rfloat import (
-    formatd, DTSF_STR_PRECISION, isinf, isnan, copysign, string_to_float)
+    DTSF_STR_PRECISION, copysign, formatd, isinf, isnan, string_to_float)
 from rpython.rlib.rstring import ParseStringError
 from rpython.tool.sourcetools import func_with_new_name
 
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
+from pypy.interpreter.typedef import GetSetProperty, TypeDef
+from pypy.objspace.std import newformat
+from pypy.objspace.std.floatobject import _hash_float
+from pypy.objspace.std.unicodeobject import unicode_to_decimal_w
+
 HASH_IMAG = 1000003
 
 
@@ -248,7 +249,7 @@
         if self.user_overridden_class:
             return None
         from rpython.rlib.longlong2float import float2longlong
-        from pypy.objspace.std.model import IDTAG_COMPLEX as tag
+        from pypy.objspace.std.util import IDTAG_COMPLEX as tag
         real = space.float_w(space.getattr(self, space.wrap("real")))
         imag = space.float_w(space.getattr(self, space.wrap("imag")))
         real_b = rbigint.fromrarith_int(float2longlong(real))
@@ -511,7 +512,7 @@
         return space.newfloat(getattr(w_obj, name))
     return GetSetProperty(fget, doc=doc)
 
-W_ComplexObject.typedef = StdTypeDef("complex",
+W_ComplexObject.typedef = TypeDef("complex",
     __doc__ = """complex(real[, imag]) -> complex number
 
 Create a complex number from a real part and an optional imaginary part.
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -11,7 +11,7 @@
     WrappedDefault, applevel, interp2app, unwrap_spec)
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.interpreter.signature import Signature
-from pypy.objspace.std.stdtypedef import StdTypeDef
+from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.util import negate
 
 
@@ -312,7 +312,7 @@
 dictrepr = app.interphook("dictrepr")
 
 
-W_DictMultiObject.typedef = StdTypeDef("dict",
+W_DictMultiObject.typedef = TypeDef("dict",
     __doc__ = '''dict() -> new empty dictionary.
 dict(mapping) -> new dictionary initialized from a mapping object\'s
     (key, value) pairs.
@@ -1146,8 +1146,6 @@
 class W_BaseDictMultiIterObject(W_Root):
     _immutable_fields_ = ["iteratorimplementation"]
 
-    ignore_for_isinstance_cache = True
-
     def __init__(self, space, iteratorimplementation):
         self.space = space
         self.iteratorimplementation = iteratorimplementation
@@ -1238,7 +1236,7 @@
             return space.newtuple([w_key, w_value])
         raise OperationError(space.w_StopIteration, space.w_None)
 
-W_DictMultiIterItemsObject.typedef = StdTypeDef(
+W_DictMultiIterItemsObject.typedef = TypeDef(
     "dict_itemiterator",
     __iter__ = interp2app(W_DictMultiIterItemsObject.descr_iter),
     __next__ = interp2app(W_DictMultiIterItemsObject.descr_next),
@@ -1246,7 +1244,7 @@
     __reduce__ = interp2app(W_BaseDictMultiIterObject.descr_reduce),
     )
 
-W_DictMultiIterKeysObject.typedef = StdTypeDef(
+W_DictMultiIterKeysObject.typedef = TypeDef(
     "dict_keyiterator",
     __iter__ = interp2app(W_DictMultiIterKeysObject.descr_iter),
     __next__ = interp2app(W_DictMultiIterKeysObject.descr_next),
@@ -1254,7 +1252,7 @@
     __reduce__ = interp2app(W_BaseDictMultiIterObject.descr_reduce),
     )
 
-W_DictMultiIterValuesObject.typedef = StdTypeDef(
+W_DictMultiIterValuesObject.typedef = TypeDef(
     "dict_valueiterator",
     __iter__ = interp2app(W_DictMultiIterValuesObject.descr_iter),
     __next__ = interp2app(W_DictMultiIterValuesObject.descr_next),
@@ -1383,7 +1381,7 @@
     def descr_iter(self, space):
         return W_DictMultiIterValuesObject(space, self.w_dict.itervalues())
 
-W_DictViewItemsObject.typedef = StdTypeDef(
+W_DictViewItemsObject.typedef = TypeDef(
     "dict_items",
     __repr__ = interp2app(W_DictViewItemsObject.descr_repr),
     __len__ = interp2app(W_DictViewItemsObject.descr_len),
@@ -1407,7 +1405,7 @@
     isdisjoint = interp2app(W_DictViewItemsObject.descr_isdisjoint),
     )
 
-W_DictViewKeysObject.typedef = StdTypeDef(
+W_DictViewKeysObject.typedef = TypeDef(
     "dict_keys",
     __repr__ = interp2app(W_DictViewKeysObject.descr_repr),
     __len__ = interp2app(W_DictViewKeysObject.descr_len),
@@ -1431,7 +1429,7 @@
     isdisjoint = interp2app(W_DictViewKeysObject.descr_isdisjoint),
     )
 
-W_DictViewValuesObject.typedef = StdTypeDef(
+W_DictViewValuesObject.typedef = TypeDef(
     "dict_values",
     __repr__ = interp2app(W_DictViewValuesObject.descr_repr),
     __len__ = interp2app(W_DictViewValuesObject.descr_len),
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -1,11 +1,11 @@
-from pypy.objspace.std.dictmultiobject import (
-    W_DictMultiObject, DictStrategy, create_iterator_classes)
-from pypy.objspace.std.typeobject import unwrap_cell
-from pypy.objspace.std.stdtypedef import StdTypeDef
+from rpython.rlib import rerased
+
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
-
-from rpython.rlib import rerased
+from pypy.interpreter.typedef import TypeDef
+from pypy.objspace.std.dictmultiobject import (
+    DictStrategy, W_DictMultiObject, create_iterator_classes)
+from pypy.objspace.std.typeobject import unwrap_cell
 
 
 class W_DictProxyObject(W_DictMultiObject):
@@ -24,7 +24,7 @@
         return space.wrap(u"mappingproxy(%s)" % (
             space.unicode_w(W_DictMultiObject.descr_repr(self, space))))
 
-W_DictProxyObject.typedef = StdTypeDef(
+W_DictProxyObject.typedef = TypeDef(
     "mappingproxy", W_DictMultiObject.typedef,
     __new__ = interp2app(W_DictProxyObject.descr_new),
     __init__ = interp2app(W_DictProxyObject.descr_init),
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -2,27 +2,28 @@
 import operator
 import sys
 
+from rpython.rlib import rarithmetic, rfloat
+from rpython.rlib.rarithmetic import LONG_BIT, intmask, ovfcheck_float_to_int
+from rpython.rlib.rbigint import rbigint
+from rpython.rlib.rfloat import (
+    DTSF_ADD_DOT_0, INFINITY, NAN, copysign,
+    float_as_rbigint_ratio, formatd, isfinite, isinf, isnan)
+from rpython.rlib.rstring import ParseStringError
+from rpython.rlib.unroll import unrolling_iterable
+from rpython.rtyper.lltypesystem.module.ll_math import math_fmod
+from rpython.tool.sourcetools import func_with_new_name
+
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from pypy.interpreter.typedef import GetSetProperty
+from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
+from pypy.interpreter.typedef import GetSetProperty, TypeDef
 from pypy.objspace.std import newformat
 from pypy.objspace.std.intobject import HASH_BITS, HASH_MODULUS, W_IntObject
 from pypy.objspace.std.longobject import (
     W_AbstractLongObject, newlong_from_float)
 from rpython.rlib.rarithmetic import (
     LONG_BIT, intmask, ovfcheck_float_to_int, r_uint)
-from pypy.objspace.std.stdtypedef import StdTypeDef
 from pypy.objspace.std.util import wrap_parsestringerror
-from rpython.rlib import rarithmetic, rfloat
-from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rfloat import (
-    isinf, isnan, isfinite, INFINITY, NAN, copysign, formatd,
-    DTSF_ADD_DOT_0, float_as_rbigint_ratio)
-from rpython.rlib.rstring import ParseStringError
-from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.unroll import unrolling_iterable
-from rpython.rtyper.lltypesystem.module.ll_math import math_fmod
 
 HASH_INF  = 314159
 HASH_NAN  = 0
@@ -191,7 +192,7 @@
         if self.user_overridden_class:
             return None
         from rpython.rlib.longlong2float import float2longlong
-        from pypy.objspace.std.model import IDTAG_FLOAT as tag
+        from pypy.objspace.std.util import IDTAG_FLOAT as tag
         val = float2longlong(space.float_w(self))
         b = rbigint.fromrarith_int(val)
         b = b.lshift(3).or_(rbigint.fromint(tag))
@@ -640,7 +641,7 @@
             return space.wrap("0x%sp%s%d" % (s, sign, exp))
 
 
-W_FloatObject.typedef = StdTypeDef("float",
+W_FloatObject.typedef = TypeDef("float",
     __doc__ = '''float(x) -> floating point number
 
 Convert a string or number to a floating point number, if possible.''',
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -1,15 +1,15 @@
-"""
-String formatting routines.
-"""
+"""String formatting routines"""
 import sys
-from pypy.interpreter.error import OperationError, oefmt
+
 from rpython.rlib import jit
-from rpython.rlib.rfloat import formatd, DTSF_ALT, isnan, isinf
+from rpython.rlib.rarithmetic import INT_MAX
+from rpython.rlib.rfloat import DTSF_ALT, formatd, isnan, isinf
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.rarithmetic import INT_MAX
 from rpython.tool.sourcetools import func_with_new_name
 
+from pypy.interpreter.error import OperationError, oefmt
+
 
 class BaseStringFormatter(object):
     def __init__(self, space, values_w, w_valuedict):
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -25,11 +25,10 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import (
     WrappedDefault, applevel, interp2app, interpindirect2app, unwrap_spec)
+from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std import newformat
-from pypy.objspace.std.model import (
-    BINARY_OPS, CMP_OPS, COMMUTATIVE_OPS, IDTAG_INT)
-from pypy.objspace.std.stdtypedef import StdTypeDef
-from pypy.objspace.std.util import wrap_parsestringerror
+from pypy.objspace.std.util import (
+    BINARY_OPS, CMP_OPS, COMMUTATIVE_OPS, IDTAG_INT, wrap_parsestringerror)
 
 SENTINEL = object()
 
@@ -761,6 +760,16 @@
         _divmod, ovf2small=_divmod_ovf2small)
 
 
+def setup_prebuilt(space):
+    if space.config.objspace.std.withprebuiltint:
+        W_IntObject.PREBUILT = []
+        for i in range(space.config.objspace.std.prebuiltintfrom,
+                       space.config.objspace.std.prebuiltintto):
+            W_IntObject.PREBUILT.append(W_IntObject(i))
+    else:
+        W_IntObject.PREBUILT = None
+
+
 def wrapint(space, x):
     if not space.config.objspace.std.withprebuiltint:
         return W_IntObject(x)
@@ -916,7 +925,7 @@
     return newbigint(space, w_inttype, space.bigint_w(w_obj))
 
 
-W_AbstractIntObject.typedef = StdTypeDef("int",
+W_AbstractIntObject.typedef = TypeDef("int",
     __doc__ = """int(x=0) -> integer
 int(x, base=10) -> integer
 
diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.gateway import interp2app, interpindirect2app
 from pypy.interpreter.error import OperationError
-from pypy.objspace.std.stdtypedef import StdTypeDef
+from pypy.interpreter.typedef import TypeDef
 
 
 class W_AbstractSeqIterObject(W_Root):
@@ -40,7 +40,7 @@
     def descr_length_hint(self, space):
         return self.getlength(space)
 
-W_AbstractSeqIterObject.typedef = StdTypeDef(
+W_AbstractSeqIterObject.typedef = TypeDef(
     "sequenceiterator",
     __doc__ = '''iter(collection) -> iterator
 iter(callable, sentinel) -> iterator
@@ -159,7 +159,7 @@
             raise OperationError(space.w_StopIteration, space.w_None)
         return w_item
 
-W_ReverseSeqIterObject.typedef = StdTypeDef(
+W_ReverseSeqIterObject.typedef = TypeDef(
     "reversesequenceiterator",
     __iter__ = interp2app(W_ReverseSeqIterObject.descr_iter),
     __next__ = interp2app(W_ReverseSeqIterObject.descr_next),
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -10,27 +10,28 @@
 import operator
 import sys
 
+from rpython.rlib import debug, jit, rerased
+from rpython.rlib.listsort import make_timsort_class
+from rpython.rlib.objectmodel import (
+    import_from_mixin, instantiate, newlist_hint, resizelist_hint, specialize)
+from rpython.tool.sourcetools import func_with_new_name
+
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter.gateway import (WrappedDefault, unwrap_spec, applevel,
-    interp2app)
+from pypy.interpreter.gateway import (
+    WrappedDefault, applevel, interp2app, unwrap_spec)
 from pypy.interpreter.generator import GeneratorIterator
 from pypy.interpreter.signature import Signature
+from pypy.interpreter.typedef import TypeDef
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.floatobject import W_FloatObject
 from pypy.objspace.std.intobject import W_IntObject
-from pypy.objspace.std.iterobject import (W_FastListIterObject,
-    W_ReverseSeqIterObject)
+from pypy.objspace.std.iterobject import (
+    W_FastListIterObject, W_ReverseSeqIterObject)
 from pypy.objspace.std.sliceobject import W_SliceObject, unwrap_start_stop
-from pypy.objspace.std.stdtypedef import StdTypeDef
 from pypy.objspace.std.tupleobject import W_AbstractTupleObject
 from pypy.objspace.std.unicodeobject import W_UnicodeObject
 from pypy.objspace.std.util import get_positive_index, negate
-from rpython.rlib import debug, jit, rerased
-from rpython.rlib.listsort import make_timsort_class
-from rpython.rlib.objectmodel import (
-    instantiate, newlist_hint, resizelist_hint, specialize, import_from_mixin)
-from rpython.tool.sourcetools import func_with_new_name
 
 __all__ = ['W_ListObject', 'make_range_list', 'make_empty_list_with_size']
 
@@ -1801,7 +1802,7 @@
         return space.is_true(space.lt(a.w_key, b.w_key))
 
 
-W_ListObject.typedef = StdTypeDef("list",
+W_ListObject.typedef = TypeDef("list",
     __doc__ = """list() -> new empty list
 list(iterable) -> new list initialized from iterable's items""",
     __new__ = interp2app(W_ListObject.descr_new),
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -13,7 +13,7 @@
 from pypy.objspace.std import newformat
 from pypy.objspace.std.intobject import (
     HASH_BITS, HASH_MODULUS, W_AbstractIntObject, W_IntObject)
-from pypy.objspace.std.model import COMMUTATIVE_OPS
+from pypy.objspace.std.util import COMMUTATIVE_OPS
 
 
 def delegate_other(func):
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
deleted file mode 100644
--- a/pypy/objspace/std/model.py
+++ /dev/null
@@ -1,353 +0,0 @@
-"""
-The full list of which Python types and which implementation we want
-to provide in this version of PyPy, along with conversion rules.
-"""
-
-from pypy.objspace.std.multimethod import MultiMethodTable, FailedToImplement
-from pypy.interpreter.baseobjspace import W_Root, ObjSpace
-import pypy.interpreter.pycode
-import pypy.interpreter.special
-
-_registered_implementations = set()
-def registerimplementation(implcls):
-    """Hint to objspace.std.model to register the implementation class."""
-    assert issubclass(implcls, W_Object)
-    _registered_implementations.add(implcls)
-
-option_to_typename = {
-    "withsmalllong"  : ["smalllongobject.W_SmallLongObject"],
-    "withstrbuf"     : ["strbufobject.W_StringBufferObject"],
-}
-
-IDTAG_INT     = 1
-IDTAG_FLOAT   = 5
-IDTAG_COMPLEX = 7
-
-class StdTypeModel:
-
-    def __init__(self, config):
-        """NOT_RPYTHON: inititialization only"""
-        self.config = config
-        # All the Python types that we want to provide in this StdObjSpace
-
-        # The object implementations that we want to 'link' into PyPy must be
-        # imported here.  This registers them into the multimethod tables,
-        # *before* the type objects are built from these multimethod tables.
-        from pypy.objspace.std import objectobject
-        from pypy.objspace.std import boolobject
-        from pypy.objspace.std import intobject
-        from pypy.objspace.std import floatobject
-        from pypy.objspace.std import complexobject
-        from pypy.objspace.std import tupleobject
-        from pypy.objspace.std import listobject
-        from pypy.objspace.std import dictmultiobject
-        from pypy.objspace.std import setobject
-        from pypy.objspace.std import bytesobject
-        from pypy.objspace.std import bytearrayobject
-        from pypy.objspace.std import typeobject
-        from pypy.objspace.std import sliceobject
-        from pypy.objspace.std import longobject
-        from pypy.objspace.std import noneobject
-        from pypy.objspace.std import iterobject
-        from pypy.objspace.std import unicodeobject
-        from pypy.objspace.std import dictproxyobject
-        from pypy.objspace.std import proxyobject
-        from pypy.objspace.std import memoryobject
-
-
-        self.pythontypes = []
-        self.pythontypes.append(objectobject.W_ObjectObject.typedef)
-        self.pythontypes.append(typeobject.W_TypeObject.typedef)
-        self.pythontypes.append(noneobject.W_NoneObject.typedef)
-        self.pythontypes.append(tupleobject.W_TupleObject.typedef)
-        self.pythontypes.append(listobject.W_ListObject.typedef)
-        self.pythontypes.append(dictmultiobject.W_DictMultiObject.typedef)
-        self.pythontypes.append(setobject.W_SetObject.typedef)
-        self.pythontypes.append(setobject.W_FrozensetObject.typedef)
-        self.pythontypes.append(iterobject.W_AbstractSeqIterObject.typedef)
-        self.pythontypes.append(bytesobject.W_BytesObject.typedef)
-        self.pythontypes.append(bytearrayobject.W_BytearrayObject.typedef)
-        self.pythontypes.append(unicodeobject.W_UnicodeObject.typedef)
-        self.pythontypes.append(intobject.W_IntObject.typedef)
-        self.pythontypes.append(boolobject.W_BoolObject.typedef)
-        self.pythontypes.append(longobject.W_LongObject.typedef)
-        self.pythontypes.append(floatobject.W_FloatObject.typedef)
-        self.pythontypes.append(complexobject.W_ComplexObject.typedef)
-        self.pythontypes.append(sliceobject.W_SliceObject.typedef)
-        self.pythontypes.append(memoryobject.W_MemoryView.typedef)
-
-        # the set of implementation types
-        self.typeorder = {
-            objectobject.W_ObjectObject: [],
-            # XXX: Bool/Int/Long are pythontypes but still included here
-            # for delegation to Float/Complex
-            boolobject.W_BoolObject: [],
-            intobject.W_IntObject: [],
-            floatobject.W_FloatObject: [],
-            typeobject.W_TypeObject: [],
-            sliceobject.W_SliceObject: [],
-            longobject.W_LongObject: [],
-            noneobject.W_NoneObject: [],
-            complexobject.W_ComplexObject: [],
-            pypy.interpreter.pycode.PyCode: [],
-            pypy.interpreter.special.Ellipsis: [],
-            }
-
-        self.imported_but_not_registered = {
-            bytesobject.W_BytesObject: True,
-        }
-        for option, value in config.objspace.std:
-            if option.startswith("with") and option in option_to_typename:
-                for classname in option_to_typename[option]:
-                    modname = classname[:classname.index('.')]
-                    classname = classname[classname.index('.')+1:]
-                    d = {}
-                    exec "from pypy.objspace.std.%s import %s" % (
-                        modname, classname) in d
-                    implcls = d[classname]
-                    if value:
-                        self.typeorder[implcls] = []
-                    else:
-                        self.imported_but_not_registered[implcls] = True
-
-        # check if we missed implementations
-        for implcls in _registered_implementations:
-            if hasattr(implcls, 'register'):
-                implcls.register(self.typeorder)
-            assert (implcls in self.typeorder or
-                    implcls in self.imported_but_not_registered), (
-                "please add %r in StdTypeModel.typeorder" % (implcls,))
-
-
-        for type in self.typeorder:
-            self.typeorder[type].append((type, None))
-
-        # register the order in which types are converted into each others
-        # when trying to dispatch multimethods.
-        # XXX build these lists a bit more automatically later
-
-        if config.objspace.std.withsmalllong:
-            from pypy.objspace.std import smalllongobject
-            self.typeorder[smalllongobject.W_SmallLongObject] += [
-                (floatobject.W_FloatObject, smalllongobject.delegate_SmallLong2Float),
-                (complexobject.W_ComplexObject, smalllongobject.delegate_SmallLong2Complex),
-                ]
-
-        if config.objspace.std.withstrbuf:
-            from pypy.objspace.std import strbufobject
-
-        # put W_Root everywhere
-        self.typeorder[W_Root] = []
-        for type in self.typeorder:
-            from pypy.objspace.std import stdtypedef
-            if type is not W_Root and isinstance(type.typedef, stdtypedef.StdTypeDef):
-                self.typeorder[type].append((type.typedef.any, None))
-            self.typeorder[type].append((W_Root, None))
-
-        self._typeorder_with_empty_usersubcls = None
-
-        # ____________________________________________________________
-        # Prebuilt common integer values
-
-        if config.objspace.std.withprebuiltint:
-            # XXX: currently broken on py3k
-            intobject.W_IntObject.PREBUILT = []
-            for i in range(config.objspace.std.prebuiltintfrom,
-                           config.objspace.std.prebuiltintto):
-                intobject.W_IntObject.PREBUILT.append(intobject.W_IntObject(i))
-            del i
-        else:
-            intobject.W_IntObject.PREBUILT = None
-
-        # ____________________________________________________________
-
-    def get_typeorder_with_empty_usersubcls(self):
-        if self._typeorder_with_empty_usersubcls is None:
-            from pypy.interpreter.typedef import enum_interplevel_subclasses
-            from pypy.objspace.std import stdtypedef
-            result = self.typeorder.copy()
-            for cls in self.typeorder:
-                if (hasattr(cls, 'typedef') and cls.typedef is not None and
-                    cls.typedef.acceptable_as_base_class):
-                    subclslist = enum_interplevel_subclasses(self.config, cls)
-                    for subcls in subclslist:
-                        if cls in subcls.__bases__:   # only direct subclasses
-                            # for user subclasses we only accept "generic"
-                            # matches: "typedef.any" is the applevel-type-based
-                            # matching, and "W_Root" is ANY.
-                            matches = []
-                            if isinstance(cls.typedef, stdtypedef.StdTypeDef):
-                                matches.append((cls.typedef.any, None))
-                            matches.append((W_Root, None))
-                            result[subcls] = matches
-            self._typeorder_with_empty_usersubcls = result
-        return self._typeorder_with_empty_usersubcls
-
-def _op_negated(function):
-    def op(space, w_1, w_2):
-        return space.not_(function(space, w_1, w_2))
-    return op
-
-def _op_swapped(function):
-    def op(space, w_1, w_2):
-        return function(space, w_2, w_1)
-    return op
-
-def _op_swapped_negated(function):
-    def op(space, w_1, w_2):
-        return space.not_(function(space, w_2, w_1))
-    return op
-
-
-CMP_OPS = dict(lt='<', le='<=', eq='==', ne='!=', gt='>', ge='>=')
-CMP_CORRESPONDANCES = [
-    ('eq', 'ne', _op_negated),
-    ('lt', 'gt', _op_swapped),
-    ('le', 'ge', _op_swapped),
-    ('lt', 'ge', _op_negated),
-    ('le', 'gt', _op_negated),
-    ('lt', 'le', _op_swapped_negated),
-    ('gt', 'ge', _op_swapped_negated),
-    ]
-for op1, op2, value in CMP_CORRESPONDANCES[:]:
-    i = CMP_CORRESPONDANCES.index((op1, op2, value))
-    CMP_CORRESPONDANCES.insert(i+1, (op2, op1, value))
-BINARY_BITWISE_OPS = {'and': '&', 'lshift': '<<', 'or': '|', 'rshift': '>>',
-                      'xor': '^'}
-BINARY_OPS = dict(add='+', div='/', floordiv='//', mod='%', mul='*', sub='-',
-                  truediv='/', **BINARY_BITWISE_OPS)
-COMMUTATIVE_OPS = ('add', 'mul', 'and', 'or', 'xor')
-
-def add_extra_comparisons():
-    """
-    Add the missing comparison operators if they were not explicitly
-    defined:  eq <-> ne  and  lt <-> le <-> gt <-> ge.
-    We try to add them in the order defined by the CMP_CORRESPONDANCES
-    table, thus favouring swapping the arguments over negating the result.
-    """
-    originalentries = {}
-    for op in CMP_OPS.iterkeys():
-        originalentries[op] = getattr(MM, op).signatures()
-
-    for op1, op2, correspondance in CMP_CORRESPONDANCES:
-        mirrorfunc = getattr(MM, op2)
-        for types in originalentries[op1]:
-            t1, t2 = types
-            if t1 is t2:
-                if not mirrorfunc.has_signature(types):
-                    functions = getattr(MM, op1).getfunctions(types)
-                    assert len(functions) == 1, ('Automatic'
-                            ' registration of comparison functions'
-                            ' only work when there is a single method for'
-                            ' the operation.')
-                    mirrorfunc.register(correspondance(functions[0]), *types)
-
-
-# ____________________________________________________________
-
-W_ANY = W_Root
-
-class W_Object(W_Root):
-    "Parent base class for wrapped objects provided by the StdObjSpace."
-    # Note that not all wrapped objects in the interpreter inherit from
-    # W_Object.  (They inherit from W_Root.)
-    __slots__ = ()
-
-    def __repr__(self):
-        name = getattr(self, 'name', '')
-        if not isinstance(name, str):
-            name = ''
-        s = '%s(%s)' % (self.__class__.__name__, name)
-        w_cls = getattr(self, 'w__class__', None)
-        if w_cls is not None and w_cls is not self:
-            s += ' instance of %s' % self.w__class__
-        return '<%s>' % s
-
-
-class UnwrapError(Exception):
-    pass
-
-
-class StdObjSpaceMultiMethod(MultiMethodTable):
-
-    def __init__(self, operatorsymbol, arity, specialnames=None, **extras):
-        """NOT_RPYTHON: cannot create new multimethods dynamically.
-        """
-        MultiMethodTable.__init__(self, arity, W_ANY,
-                                  argnames_before = ['space'])
-        self.operatorsymbol = operatorsymbol
-        if specialnames is None:
-            specialnames = [operatorsymbol]
-        assert isinstance(specialnames, list)
-        self.specialnames = specialnames  # e.g. ['__xxx__', '__rxxx__']
-        self.extras = extras
-        # transform  '+'  =>  'add'  etc.
-        for line in ObjSpace.MethodTable:
-            realname, symbolname = line[:2]
-            if symbolname == operatorsymbol:
-                self.name = realname
-                break
-        else:
-            self.name = operatorsymbol
-
-        if extras.get('general__args__', False):
-            self.argnames_after = ['__args__']
-        if extras.get('varargs_w', False):
-            self.argnames_after = ['args_w']
-        self.argnames_after += extras.get('extra_args', [])
-
-    def install_not_sliced(self, typeorder, baked_perform_call=True):
-        return self.install(prefix = '__mm_' + self.name,
-                list_of_typeorders = [typeorder]*self.arity,
-                baked_perform_call=baked_perform_call)
-
-    def merge_with(self, other):
-        # Make a new 'merged' multimethod including the union of the two
-        # tables.  In case of conflict, pick the entry from 'self'.
-        if self.arity != other.arity:
-            return self      # XXX that's the case of '**'
-        operatorsymbol = '%s_merge_%s' % (self.name, other.name)
-        assert self.extras == other.extras
-        mm = StdObjSpaceMultiMethod(operatorsymbol, self.arity, **self.extras)
-        #
-        def merge(node1, node2):
-            assert type(node1) is type(node2)
-            if isinstance(node1, dict):
-                d = node1.copy()
-                d.update(node2)
-                for key in node1:
-                    if key in node2:
-                        d[key] = merge(node1[key], node2[key])
-                return d
-            else:
-                assert isinstance(node1, list)
-                assert node1
-                return node1     # pick the entry from 'self'
-        #
-        mm.dispatch_tree = merge(self.dispatch_tree, other.dispatch_tree)
-        return mm
-
-NOT_MULTIMETHODS = set(
-    ['delattr', 'delete', 'get', 'id', 'inplace_div', 'inplace_floordiv',
-     'inplace_lshift', 'inplace_mod', 'inplace_pow', 'inplace_rshift',
-     'inplace_truediv', 'is_', 'set', 'setattr', 'type', 'userdel',
-     'isinstance', 'issubtype', 'int', 'ord'])
-# XXX should we just remove those from the method table or we're happy
-#     with just not having multimethods?
-
-class MM:
-    """StdObjSpace multimethods"""
-
-    call    = StdObjSpaceMultiMethod('call', 1, ['__call__'],
-                                     general__args__=True)
-    init    = StdObjSpaceMultiMethod('__init__', 1, general__args__=True)
-    getnewargs = StdObjSpaceMultiMethod('__getnewargs__', 1)
-
-    # add all regular multimethods here
-    for _name, _symbol, _arity, _specialnames in ObjSpace.MethodTable:
-        if _name not in locals() and _name not in NOT_MULTIMETHODS:
-            mm = StdObjSpaceMultiMethod(_symbol, _arity, _specialnames)
-            locals()[_name] = mm
-            del mm
-
-    pow.extras['defaults'] = (None,)
diff --git a/pypy/objspace/std/multimethod.py b/pypy/objspace/std/multimethod.py
deleted file mode 100644
--- a/pypy/objspace/std/multimethod.py
+++ /dev/null
@@ -1,972 +0,0 @@
-
-from rpython.tool.sourcetools import compile2
-
-# This provide two compatible implementations of "multimethods".  A
-# multimethod is a callable object which chooses and calls a real
-# function from a table of pre-registered functions.  The choice depends
-# on the '__class__' of all arguments.  For example usages see
-# test_multimethod.
-
-# These multimethods support delegation: for each class A we must
-# provide a "typeorder", which is list of pairs (B, converter) where B
-# is a class and 'converter' is a function that can convert from an
-# instance of A to an instance of B.  If 'converter' is None it is
-# assumed that the instance needs no conversion.  The first entry in the
-# typeorder of a class A must almost always be (A, None).
-
-# A slightly non-standard feature of PyPy's multimethods is the way in
-# which they interact with normal subclassing.  Basically, they don't.
-# Suppose that A is a parent class of B.  Then a function registered for
-# an argument class A only accepts an instance whose __class__ is A, not
-# B.  To make it accept an instance of B, the typeorder for B must
-# contain (A, None).  An exception to this strict rule is if C is
-# another subclass of A which is not mentioned at all in the typeorder;
-# in this case C is considered to be equivalent to A.
-
-
-class FailedToImplement(Exception):
-    def __new__(cls, *args):
-        if cls is FailedToImplement:
-            assert not args, "use FailedToImplementArgs!"
-        return Exception.__new__(cls, *args)
-
-    def get_w_value(self, space):
-        return None
-
-    def get_w_type(self, space):
-        return None
-
-    def __str__(self):
-        return '<FailedToImplement(None, None)>'
-
-class FailedToImplementArgs(FailedToImplement):
-    def __init__(self, w_type=None, w_value=None):
-        self.w_type  = w_type
-        self.w_value = w_value
-
-    def get_w_value(self, space):
-        # convenience: same semantics as with OperationError
-        return self.w_value
-
-    def get_w_type(self, space):
-        return self.w_type
-
-    def __str__(self):
-        return '<FailedToImplement(%s, %s)>' % (self.w_type, self.w_value)
-
-
-
-def raiseFailedToImplement():
-    raise FailedToImplement
-
-
-class MultiMethodTable:
-
-    def __init__(self, arity, root_class, argnames_before=[], argnames_after=[]):
-        """NOT_RPYTHON: cannot create new multimethods dynamically.
-        MultiMethod-maker dispatching on exactly 'arity' arguments.
-        """
-        if arity < 1:
-            raise ValueError, "multimethods cannot dispatch on nothing"
-        self.arity = arity
-        self.root_class = root_class
-        self.dispatch_tree = {}
-        self.argnames_before = list(argnames_before)
-        self.argnames_after = list(argnames_after)
-
-    def register(self, function, *types, **kwds):
-        assert len(types) == self.arity
-        assert kwds.keys() == [] or kwds.keys() == ['order']
-        order = kwds.get('order', 0)
-        node = self.dispatch_tree
-        for type in types[:-1]:
-            node = node.setdefault(type, {})
-        lst = node.setdefault(types[-1], [])
-        if order >= len(lst):
-            lst += [None] * (order+1 - len(lst))
-        assert lst[order] is None, "duplicate function for %r@%d" % (
-            types, order)
-        lst[order] = function
-
-    def install(self, prefix, list_of_typeorders, baked_perform_call=True,
-                base_typeorder=None, installercls=None):
-        "NOT_RPYTHON: initialization-time only"
-        assert len(list_of_typeorders) == self.arity
-        installercls = installercls or Installer
-        installer = installercls(self, prefix, list_of_typeorders,
-                                 baked_perform_call=baked_perform_call,
-                                 base_typeorder=base_typeorder)
-        return installer.install()
-
-    def install_if_not_empty(self, prefix, list_of_typeorders,
-                             base_typeorder=None, installercls=None):
-        "NOT_RPYTHON: initialization-time only"
-        assert len(list_of_typeorders) == self.arity
-        installercls = installercls or Installer
-        installer = installercls(self, prefix, list_of_typeorders,
-                                 base_typeorder=base_typeorder)
-        if installer.is_empty():
-            return None
-        else:
-            return installer.install()
-
-
-
-    # ____________________________________________________________
-    # limited dict-like interface to the dispatch table
-
-    def getfunctions(self, types):
-        assert len(types) == self.arity
-        node = self.dispatch_tree
-        for type in types:
-            node = node[type]
-        return [fn for fn in node if fn is not None]
-
-    def has_signature(self, types):
-        try:
-            self.getfunctions(types)
-        except KeyError:
-            return False
-        else:
-            return True
-
-    def signatures(self):
-        """NOT_RPYTHON"""
-        result = []
-        def enum_keys(types_so_far, node):
-            for type, subnode in node.items():
-                next_types = types_so_far+(type,)
-                if isinstance(subnode, dict):
-                    enum_keys(next_types, subnode)
-                else:
-                    assert len(next_types) == self.arity
-                    result.append(next_types)
-        enum_keys((), self.dispatch_tree)
-        return result
-
-# ____________________________________________________________
-# Installer version 1
-
-class InstallerVersion1:
-    """NOT_RPYTHON"""
-
-    instance_counter = 0
-
-    mmfunccache = {}
-
-    prefix_memo = {}
-
-    def __init__(self, multimethod, prefix, list_of_typeorders,
-                 baked_perform_call=True, base_typeorder=None):
-        self.__class__.instance_counter += 1
-        self.multimethod = multimethod
-        # avoid prefix clashes, user code should supply different prefixes
-        # itself for nice names in tracebacks
-        base_prefix = prefix
-        n = 1
-        while prefix in self.prefix_memo:
-            n += 1
-            prefix = "%s%d" % (base_prefix, n)
-        self.prefix = prefix
-        self.prefix_memo[prefix] = 1
-        self.list_of_typeorders = list_of_typeorders
-        self.check_typeorders()
-        self.subtree_cache = {}
-        self.to_install = []
-        self.non_empty = self.build_tree([], multimethod.dispatch_tree)
-
-        self.baked_perform_call = baked_perform_call
-
-        if self.non_empty:
-            perform = [(None, prefix, 0)]
-        else:
-            perform = []
-
-        self.perform_call = self.build_function(None, prefix+'_perform_call',
-                                                None, perform)
-
-    def check_typeorders(self):
-        # xxx we use a '__'-separated list of the '__name__' of the types
-        # in build_single_method(), so types with the same __name__ or
-        # with '__' in them would obscurely break this logic
-        for typeorder in self.list_of_typeorders:
-            for type in typeorder:
-                assert '__' not in type.__name__, (
-                    "avoid '__' in the name of %r" % (type,))
-            names = dict.fromkeys([type.__name__ for type in typeorder])
-            assert len(names) == len(typeorder), (
-                "duplicate type.__name__ in %r" % (typeorder,))
-
-    def is_empty(self):
-        return not self.non_empty
-
-    def install(self):
-        #f = open('LOGFILE', 'a')
-        #print >> f, '_'*60
-        #import pprint
-        #pprint.pprint(self.list_of_typeorders, f)
-
-        def class_key(cls):
-            "Returns an object such that class_key(subcls) > class_key(cls)."
-            return len(cls.__mro__)
-
-        # Sort 'to_install' so that base classes come first, which is
-        # necessary for the 'parentfunc' logic in the loop below to work.
-        # Moreover, 'to_install' can contain two functions with the same
-        # name for the root class: the default fallback one and the real
-        # one.  So we have to sort the real one just after the default one
-        # so that the default one gets overridden.
-        def key(target, funcname, func, source, fallback):
-            if target is None:
-                return ()
-            return (class_key(target), not fallback)
-        self.to_install.sort(lambda a, b: cmp(key(*a), key(*b)))
-
-        for target, funcname, func, source, fallback in self.to_install:
-            if target is not None:
-                # If the parent class provides a method of the same
-                # name which is actually the same 'func', we don't need
-                # to install it again.  Useful with fallback functions.
-                parentfunc = getattr(target, funcname, None)
-                parentfunc = getattr(parentfunc, 'im_func', None)
-                if parentfunc is func:
-                    continue
-                #print >> f, target.__name__, funcname
-                #if source:
-                #    print >> f, source
-                #else:
-                #    print >> f, '*\n'
-                setattr(target, funcname, func)
-        #f.close()
-        return self.perform_call
-
-    def build_tree(self, types_so_far, dispatch_node):
-        key = tuple(types_so_far)
-        if key in self.subtree_cache:
-            return self.subtree_cache[key]
-        non_empty = False
-        typeorder = self.list_of_typeorders[len(types_so_far)]
-        for next_type in typeorder:
-            if self.build_single_method(typeorder, types_so_far, next_type,
-                                        dispatch_node):
-                non_empty = True
-        self.subtree_cache[key] = non_empty
-        return non_empty
-
-    def build_single_method(self, typeorder, types_so_far, next_type,
-                            dispatch_node):
-        funcname = '__'.join([self.prefix] + [t.__name__ for t in types_so_far])
-
-        order = typeorder[next_type]
-        #order = [(next_type, None)] + order
-
-        things_to_call = []
-        for type, conversion in order:
-            if type not in dispatch_node:
-                # there is no possible completion of types_so_far+[type]
-                # that could lead to a registered function.
-                continue
-            match = dispatch_node[type]
-            if isinstance(match, dict):
-                if self.build_tree(types_so_far+[type], match):
-                    call = funcname + '__' + type.__name__
-                    call_selfarg_index = len(types_so_far) + 1
-                    things_to_call.append((conversion, call,
-                                           call_selfarg_index))
-            else:
-                for func in match:   # list of functions
-                    if func is not None:
-                        things_to_call.append((conversion, func, None))
-
-        funcname = intern(funcname)
-        self.build_function(next_type, funcname, len(types_so_far),
-                            things_to_call)
-        return bool(things_to_call)
-
-    def build_function(self, target, funcname, func_selfarg_index,
-                       things_to_call):
-        # support for inventing names for the entries in things_to_call
-        # which are real function objects instead of strings
-        miniglobals = {'FailedToImplement': FailedToImplement, '__name__': __name__}
-        def invent_name(obj):
-            if isinstance(obj, str):
-                return obj
-            name = obj.__name__
-            n = 1
-            while name in miniglobals:
-                n += 1
-                name = '%s%d' % (obj.__name__, n)
-            miniglobals[name] = obj
-            return name
-
-        funcargs = ['arg%d' % i for i in range(self.multimethod.arity)]
-
-        bodylines = []
-        for conversion, call, call_selfarg_index in things_to_call:
-            callargs = funcargs[:]
-            if conversion is not None:
-                to_convert = func_selfarg_index
-                convert_callargs = (self.multimethod.argnames_before +
-                                    [callargs[to_convert]])
-                callargs[to_convert] = '%s(%s)' % (
-                    invent_name(conversion), ', '.join(convert_callargs))
-            callname = invent_name(call)
-            if call_selfarg_index is not None:
-                # fallback on root_class
-                self.build_function(self.multimethod.root_class,
-                                    callname, call_selfarg_index, [])
-                callname = '%s.%s' % (callargs.pop(call_selfarg_index), callname)
-            callargs = (self.multimethod.argnames_before +
-                        callargs + self.multimethod.argnames_after)
-            bodylines.append('return %s(%s)' % (callname, ', '.join(callargs)))
-
-        fallback = False
-        if not bodylines:
-            miniglobals['raiseFailedToImplement'] = raiseFailedToImplement
-            bodylines = ['return raiseFailedToImplement()']
-            fallback = True
-            # NB. make sure that there is only one fallback function object,
-            # i.e. the key used in the mmfunccache below is always the same
-            # for all functions with the same name and an empty bodylines.
-
-        # protect all lines apart from the last one by a try:except:
-        for i in range(len(bodylines)-2, -1, -1):
-            bodylines[i:i+1] = ['try:',
-                                '    ' + bodylines[i],
-                                'except FailedToImplement:',
-                                '    pass']
-
-        if func_selfarg_index is not None:
-            selfargs = [funcargs.pop(func_selfarg_index)]
-        else:
-            selfargs = []
-        funcargs = (selfargs + self.multimethod.argnames_before +
-                    funcargs + self.multimethod.argnames_after)
-
-        if target is None and not self.baked_perform_call:
-            return funcargs, bodylines[0][len('return '):], miniglobals, fallback
-
-        # indent mode
-        bodylines = ['    ' + line for line in bodylines]
-
-        bodylines.insert(0, 'def %s(%s):' % (funcname, ', '.join(funcargs)))
-        bodylines.append('')
-        source = '\n'.join(bodylines)
-
-        # XXX find a better place (or way) to avoid duplicate functions
-        l = miniglobals.items()
-        l.sort()
-        l = tuple(l)
-        key = (source, l)
-        try:
-            func = self.mmfunccache[key]
-        except KeyError:
-            exec compile2(source) in miniglobals
-            func = miniglobals[funcname]
-            self.mmfunccache[key] = func
-        #else:
-        #    print "avoided duplicate function", func
-        self.to_install.append((target, funcname, func, source, fallback))
-        return func
-
-# ____________________________________________________________
-# Installer version 2
-
-class MMDispatcher(object):
-    """NOT_RPYTHON
-    Explicit dispatcher class.  The __call__ and dispatch() methods
-    are only present for documentation purposes.  The InstallerVersion2
-    uses the expressions() method to precompute fast RPython-friendly
-    dispatch tables.
-    """
-    _revcache = None
-
-    def __init__(self, multimethod, list_of_typeorders):
-        self.multimethod = multimethod
-        self.list_of_typeorders = list_of_typeorders
-
-    def __call__(self, *args):
-        # for testing only: this is slow
-        i = len(self.multimethod.argnames_before)
-        j = i + self.multimethod.arity
-        k = j + len(self.multimethod.argnames_after)
-        assert len(args) == k
-        prefixargs = args[:i]
-        dispatchargs = args[i:j]
-        suffixargs = args[j:]
-        return self.dispatch([x.__class__ for x in dispatchargs],
-                             prefixargs,
-                             dispatchargs,
-                             suffixargs)
-
-    def dispatch(self, argtypes, prefixargs, args, suffixargs):
-        # for testing only: this is slow
-        def expr(v):
-            if isinstance(v, Call):
-                return v.function(*[expr(w) for w in v.arguments])
-            else:
-                return v
-        # XXX this is incomplete: for each type in argtypes but not
-        # in the typeorder, we should look for the first base class
-        # that is in the typeorder.
-        e = None
-        for v in self.expressions(argtypes, prefixargs, args, suffixargs):
-            try:
-                return expr(v)
-            except FailedToImplement, e:
-                pass
-        else:
-            raise e or FailedToImplement()
-
-    def expressions(self, argtypes, prefixargs, args, suffixargs):
-        """Lists the possible expressions that call the appropriate
-        function for the given argument types.  Each expression is a Call
-        object.  The intent is that at run-time the first Call that doesn't
-        cause FailedToImplement to be raised is the good one.
-        """
-        prefixargs = tuple(prefixargs)
-        suffixargs = tuple(suffixargs)
-
-        def walktree(node, args_so_far):
-            if isinstance(node, list):
-                for func in node:
-                    if func is not None:
-                        result.append(Call(func, prefixargs +
-                                                 args_so_far +
-                                                 suffixargs))
-            else:
-                index = len(args_so_far)
-                typeorder = self.list_of_typeorders[index]
-                next_type = argtypes[index]
-                for target_type, converter in typeorder[next_type]:
-                    if target_type not in node:
-                        continue
-                    next_arg = args[index]
-                    if converter:
-                        next_arg = Call(converter, prefixargs + (next_arg,))
-                    walktree(node[target_type], args_so_far + (next_arg,))
-
-        result = []
-        walktree(self.multimethod.dispatch_tree, ())
-        return result
-
-    def anychance(self, typesprefix):
-        # is there any chance that a list of types starting with typesprefix
-        # could lead to a successful dispatch?
-        # (START-UP TIME OPTIMIZATION ONLY)
-        if self._revcache is None:
-
-            def build_tree(types_so_far, dispatch_node):
-                non_empty = False
-                typeorder = self.list_of_typeorders[len(types_so_far)]
-                for next_type in typeorder:
-                    if build_single_method(typeorder, types_so_far, next_type,
-                                           dispatch_node):
-                        non_empty = True
-                if non_empty:
-                    self._revcache[types_so_far] = True
-                return non_empty
-
-            def build_single_method(typeorder, types_so_far, next_type,
-                                    dispatch_node):
-                order = typeorder[next_type]
-                things_to_call = False
-                for type, conversion in order:
-                    if type not in dispatch_node:
-                        # there is no possible completion of
-                        # types_so_far+[type] that could lead to a
-                        # registered function.
-                        continue
-                    match = dispatch_node[type]
-                    if isinstance(match, dict):
-                        if build_tree(types_so_far+(next_type,), match):
-                            things_to_call = True
-                    elif match:
-                        things_to_call = True
-                return things_to_call
-
-            self._revcache = {}
-            build_tree((), self.multimethod.dispatch_tree)
-        return tuple(typesprefix) in self._revcache
-
-
-class Call(object):
-    """ Represents a call expression.
-    The arguments may themselves be Call objects.
-    """
-    def __init__(self, function, arguments):
-        self.function = function
-        self.arguments = arguments
-
-
-class CompressedArray(object):
-    def __init__(self, null_value):
-        self.null_value = null_value
-        self.items = [null_value]
-
-    def ensure_length(self, newlen):
-        if newlen > len(self.items):
-            self.items.extend([self.null_value] * (newlen - len(self.items)))
-
-    def insert_subarray(self, array):
-        # insert the given array of numbers into the indexlist,
-        # allowing null values to become non-null
-        if array.count(self.null_value) == len(array):
-            return 0
-        test = 1
-        while True:
-            self.ensure_length(test+len(array))
-            for i in xrange(len(array)):
-                if not (array[i] == self.items[test+i] or
-                        array[i] == self.null_value or
-                        self.items[test+i] == self.null_value):
-                    break
-            else:
-                # success
-                for i in range(len(array)):
-                    if array[i] != self.null_value:
-                        self.items[test+i] = array[i]
-                return test
-            test += 1
-
-    def _freeze_(self):
-        return True
-
-
-class MRDTable(object):
-    # Multi-Method Dispatch Using Multiple Row Displacement,
-    # Candy Pang, Wade Holst, Yuri Leontiev, and Duane Szafron
-    # University of Alberta, Edmonton AB T6G 2H1 Canada
-    # can be found on http://web.cs.ualberta.ca/~yuri/publ.htm
-
-    Counter = 0
-
-    def __init__(self, list_of_types):
-        self.id = MRDTable.Counter
-        MRDTable.Counter += 1
-        self.list_of_types = list_of_types
-        self.typenum = dict(zip(list_of_types, range(len(list_of_types))))
-        self.attrname = '__mrd%d_typenum' % self.id
-        for t1, num in self.typenum.items():
-            setattr(t1, self.attrname, num)
-        self.indexarray = CompressedArray(0)
-
-    def get_typenum(self, cls):
-        return self.typenum[cls]
-
-    def is_anti_range(self, typenums):
-        # NB. typenums should be sorted.  Returns (a, b) if typenums contains
-        # at least half of all typenums and its complement is range(a, b).
-        # Returns (None, None) otherwise.  Returns (0, 0) if typenums contains
-        # everything.
-        n = len(self.list_of_types)
-        if len(typenums) <= n // 2:
-            return (None, None)
-        typenums = dict.fromkeys(typenums)
-        complement = [typenum for typenum in range(n)
-                              if typenum not in typenums]
-        if not complement:
-            return (0, 0)
-        a = min(complement)
-        b = max(complement) + 1
-        if complement == range(a, b):
-            return (a, b)
-        else:
-            return (None, None)
-
-    def normalize_length(self, next_array):
-        # make sure that the indexarray is not smaller than any funcarray
-        self.indexarray.ensure_length(len(next_array.items))
-
-
-def invent_name(miniglobals, obj):
-    if isinstance(obj, str):
-        return obj
-    name = obj.__name__
-    n = 1
-    while name in miniglobals:
-        n += 1
-        name = '%s%d' % (obj.__name__, n)
-    miniglobals[name] = obj
-    return name
-
-
-class FuncEntry(object):
-
-    def __init__(self, bodylines, miniglobals, fallback):
-        self.body = '\n    '.join(bodylines)
-        self.miniglobals = miniglobals
-        self.fallback = fallback
-        self.possiblenames = []
-        self.typetree = {}
-        self._function = None
-
-    def key(self):
-        lst = self.miniglobals.items()
-        lst.sort()
-        return self.body, tuple(lst)
-
-    def get_function_name(self):
-        # pick a name consistently based on self.possiblenames
-        length = min([len(parts) for parts in self.possiblenames])
-        result = []
-        for i in range(length):
-            choices = {}
-            for parts in self.possiblenames:
-                choices[parts[i]] = True
-            parts = choices.keys()
-            res = str(len(parts))
-            for part in parts:
-                if type(part) is str:     # there is a string at this pos
-                    if '0_fail' in choices:
-                        res = '0_fail'
-                    elif len(parts) == 1:
-                        res = part
-                    break
-            else:
-                # only types at this location, try to find a common base
-                basecls = parts[0]
-                for cls in parts[1:]:
-                    if issubclass(basecls, cls):
-                        basecls = cls
-                for cls in parts[1:]:
-                    if not issubclass(cls, basecls):
-                        break   # no common base
-                else:
-                    res = basecls.__name__
-            result.append(res)
-        return '_'.join(result)
-
-    def make_function(self, fnargs, nbargs_before, mrdtable):
-        if self._function is not None:
-            return self._function
-        name = self.get_function_name()
-        self.compress_typechecks(mrdtable)
-        checklines = self.generate_typechecks(mrdtable, fnargs[nbargs_before:])
-        if not checklines:
-            body = self.body
-        else:
-            checklines.append(self.body)
-            body = '\n    '.join(checklines)
-        source = 'def %s(%s):\n    %s\n' % (name, ', '.join(fnargs), body)
-        self.debug_dump(source)
-        exec compile2(source) in self.miniglobals
-        self._function = self.miniglobals[name]
-        return self._function
-
-    def debug_dump(self, source):
-        if 0:    # for debugging the generated mm sources
-            name = self.get_function_name()
-            f = open('/tmp/mm-source/%s' % name, 'a')
-            for possiblename in self.possiblenames:
-                print >> f, '#',
-                for part in possiblename:
-                    print >> f, getattr(part, '__name__', part),
-                print >> f
-            print >> f
-            print >> f, source
-            f.close()
-
-    def register_valid_types(self, types):
-        node = self.typetree
-        for t1 in types[:-1]:
-            if node is True:
-                return
-            node = node.setdefault(t1, {})
-        if node is True:
-            return
-        node[types[-1]] = True
-
-    def no_typecheck(self):
-        self.typetree = True
-
-    def compress_typechecks(self, mrdtable):
-        def full(node):
-            if node is True:
-                return 1
-            fulls = 0
-            for key, subnode in node.items():
-                if full(subnode):
-                    node[key] = True
-                    fulls += 1
-            if fulls == types_total:
-                return 1
-            return 0
-
-        types_total = len(mrdtable.list_of_types)
-        if full(self.typetree):
-            self.typetree = True
-
-    def generate_typechecks(self, mrdtable, args):
-        attrname = mrdtable.attrname
-        possibletypes = [{} for _ in args]
-        any_type_is_ok = [False for _ in args]
-
-        def generate(node, level=0):
-            # this generates type-checking code like the following:
-            #
-            #     _argtypenum = arg1.__typenum
-            #     if _argtypenum == 5:
-            #         ...
-            #     elif _argtypenum == 6 or _argtypenum == 8:
-            #         ...
-            #     else:
-            #         _failedtoimplement = True
-            #
-            # or, in the common particular case of an "anti-range", we optimize it to:
-            #
-            #     _argtypenum = arg1.__typenum
-            #     if _argtypenum < 5 or _argtypenum >= 10:
-            #         ...
-            #     else:
-            #         _failedtoimplement = True
-            #
-            result = []
-            indent = '    '*level
-            if node is True:
-                for i in range(level, len(args)):
-                    any_type_is_ok[i] = True
-                result.append('%s_failedtoimplement = False' % (indent,))
-                return result
-            if not node:
-                result.append('%s_failedtoimplement = True' % (indent,))
-                return result
-            result.append('%s_argtypenum = %s.%s' % (indent, args[level],
-                                                     attrname))
-            cases = {}
-            for key, subnode in node.items():
-                possibletypes[level][key] = True
-                casebody = tuple(generate(subnode, level+1))
-                typenum = mrdtable.get_typenum(key)
-                cases.setdefault(casebody, []).append(typenum)
-            for casebody, typenums in cases.items():
-                typenums.sort()
-            cases = [(typenums, casebody)
-                     for (casebody, typenums) in cases.items()]
-            cases.sort()
-            if len(cases) == 1:
-                typenums, casebody = cases[0]
-                a, b = mrdtable.is_anti_range(typenums)
-            else:
-                a, b = None, None
-            keyword = 'if'
-            for typenums, casebody in cases:
-                if a is not None:
-                    if b - a == 1:
-                        condition = '_argtypenum != %d' % a
-                    elif b == a:
-                        condition = 'True'
-                    else:
-                        condition = '_argtypenum < %d or _argtypenum >= %d' % (
-                            a, b)
-                else:
-                    conditions = ['_argtypenum == %d' % typenum
-                                  for typenum in typenums]
-                    condition = ' or '.join(conditions)
-                result.append('%s%s %s:' % (indent, keyword, condition))
-                result.extend(casebody)
-                keyword = 'elif'
-            result.append('%selse:' % (indent,))
-            result.append('%s    _failedtoimplement = True' % (indent,))
-            return result
-
-        result = []
-        if self.typetree is not True:
-            result.extend(generate(self.typetree))
-            result.append('if _failedtoimplement:')
-            result.append('    raise FailedToImplement')
-            for level in range(len(args)):

From noreply at buildbot.pypy.org  Sun Dec 14 19:43:44 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 14 Dec 2014 19:43:44 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Resolved merge conflicts in
	test_ssl.py
Message-ID: <20141214184344.E6BC61C1066@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74929:39a0aacf4e67
Date: 2014-12-14 10:43 -0800
http://bitbucket.org/pypy/pypy/changeset/39a0aacf4e67/

Log:	Resolved merge conflicts in test_ssl.py

diff --git a/lib-python/2.7/test/test_hash.py b/lib-python/2.7/test/test_hash.py
--- a/lib-python/2.7/test/test_hash.py
+++ b/lib-python/2.7/test/test_hash.py
@@ -222,12 +222,7 @@
     repr_ = 'buffer("abc")'
 
     def test_empty_string(self):
-<<<<<<< local
         self.assertEqual(hash(buffer("")), self.EMPTY_STRING_HASH)
-=======
-        with test_support.check_py3k_warnings():
-            self.assertEqual(hash(buffer("")), 0)
->>>>>>> other
 
 class DatetimeTests(HashRandomizationTests):
     def get_hash_command(self, repr_):

From noreply at buildbot.pypy.org  Sun Dec 14 19:45:23 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 14 Dec 2014 19:45:23 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Resolved merge conflicts in
	test_ssl.py
Message-ID: <20141214184523.8921B1C1066@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74930:460cc2dd9310
Date: 2014-12-14 10:45 -0800
http://bitbucket.org/pypy/pypy/changeset/460cc2dd9310/

Log:	Resolved merge conflicts in test_ssl.py

diff --git a/lib-python/2.7/test/test_socket.py b/lib-python/2.7/test/test_socket.py
--- a/lib-python/2.7/test/test_socket.py
+++ b/lib-python/2.7/test/test_socket.py
@@ -635,17 +635,8 @@
         port = test_support.find_unused_port()
         big_port = port + 65536
         neg_port = port - 65536
-<<<<<<< local
-        sock = socket.socket()
-        try:
-            self.assertRaises((OverflowError, ValueError), sock.bind, (host, big_port))
-            self.assertRaises((OverflowError, ValueError), sock.bind, (host, neg_port))
-            sock.bind((host, port))
-        finally:
-            sock.close()
-=======
-        self.assertRaises(OverflowError, sock.bind, (HOST, big_port))
-        self.assertRaises(OverflowError, sock.bind, (HOST, neg_port))
+        self.assertRaises((OverflowError, ValueError), sock.bind, (HOST, big_port))
+        self.assertRaises((OverflowError, ValueError), sock.bind, (HOST, neg_port))
         # Since find_unused_port() is inherently subject to race conditions, we
         # call it a couple times if necessary.
         for i in itertools.count():
@@ -657,7 +648,6 @@
                     raise
             else:
                 break
->>>>>>> other
 
     @unittest.skipUnless(os.name == "nt", "Windows specific")
     def test_sock_ioctl(self):

From noreply at buildbot.pypy.org  Sun Dec 14 19:46:24 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 14 Dec 2014 19:46:24 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Resolved merge conflicts in
	test_file2k.py
Message-ID: <20141214184624.31D291C1066@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74931:06eaf68b7f7c
Date: 2014-12-14 10:46 -0800
http://bitbucket.org/pypy/pypy/changeset/06eaf68b7f7c/

Log:	Resolved merge conflicts in test_file2k.py

diff --git a/lib-python/2.7/test/test_file2k.py b/lib-python/2.7/test/test_file2k.py
--- a/lib-python/2.7/test/test_file2k.py
+++ b/lib-python/2.7/test/test_file2k.py
@@ -14,11 +14,7 @@
     threading = None
 
 from test import test_support
-<<<<<<< local
-from test.test_support import TESTFN, run_unittest, gc_collect
-=======
-from test.test_support import TESTFN, run_unittest, requires
->>>>>>> other
+from test.test_support import TESTFN, run_unittest, gc_collect, requires
 from UserList import UserList
 
 class AutoFileTests(unittest.TestCase):

From noreply at buildbot.pypy.org  Sun Dec 14 19:51:39 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 14 Dec 2014 19:51:39 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Resolved merge conflicts in
	distutils/sysconfig.py
Message-ID: <20141214185139.BCBCA1C1066@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74932:2bc79e0008d8
Date: 2014-12-14 10:51 -0800
http://bitbucket.org/pypy/pypy/changeset/2bc79e0008d8/

Log:	Resolved merge conflicts in distutils/sysconfig.py

diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -5,477 +5,8 @@
 # directly.
 
 import sys
-<<<<<<< local
 if '__pypy__' in sys.builtin_module_names:
     from distutils import sysconfig_pypy as _sysconfig_module
 else:
     from distutils import sysconfig_cpython as _sysconfig_module
 globals().update(_sysconfig_module.__dict__)
-=======
-
-from distutils.errors import DistutilsPlatformError
-
-# These are needed in a couple of spots, so just compute them once.
-PREFIX = os.path.normpath(sys.prefix)
-EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
-
-# Path to the base directory of the project. On Windows the binary may
-# live in project/PCBuild9.  If we're dealing with an x64 Windows build,
-# it'll live in project/PCbuild/amd64.
-project_base = os.path.dirname(os.path.abspath(sys.executable))
-if os.name == "nt" and "pcbuild" in project_base[-8:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir))
-# PC/VS7.1
-if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
-# PC/AMD64
-if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
-
-# set for cross builds
-if "_PYTHON_PROJECT_BASE" in os.environ:
-    # this is the build directory, at least for posix
-    project_base = os.path.normpath(os.environ["_PYTHON_PROJECT_BASE"])
-
-# python_build: (Boolean) if true, we're either building Python or
-# building an extension with an un-installed Python, so we use
-# different (hard-wired) directories.
-# Setup.local is available for Makefile builds including VPATH builds,
-# Setup.dist is available on Windows
-def _python_build():
-    for fn in ("Setup.dist", "Setup.local"):
-        if os.path.isfile(os.path.join(project_base, "Modules", fn)):
-            return True
-    return False
-python_build = _python_build()
-
-
-def get_python_version():
-    """Return a string containing the major and minor Python version,
-    leaving off the patchlevel.  Sample return values could be '1.5'
-    or '2.2'.
-    """
-    return sys.version[:3]
-
-
-def get_python_inc(plat_specific=0, prefix=None):
-    """Return the directory containing installed Python header files.
-
-    If 'plat_specific' is false (the default), this is the path to the
-    non-platform-specific header files, i.e. Python.h and so on;
-    otherwise, this is the path to platform-specific header files
-    (namely pyconfig.h).
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        if python_build:
-            buildir = os.path.dirname(sys.executable)
-            if plat_specific:
-                # python.h is located in the buildir
-                inc_dir = buildir
-            else:
-                # the source dir is relative to the buildir
-                srcdir = os.path.abspath(os.path.join(buildir,
-                                         get_config_var('srcdir')))
-                # Include is located in the srcdir
-                inc_dir = os.path.join(srcdir, "Include")
-            return inc_dir
-        return os.path.join(prefix, "include", "python" + get_python_version())
-    elif os.name == "nt":
-        return os.path.join(prefix, "include")
-    elif os.name == "os2":
-        return os.path.join(prefix, "Include")
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its C header files "
-            "on platform '%s'" % os.name)
-
-
-def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
-    """Return the directory containing the Python library (standard or
-    site additions).
-
-    If 'plat_specific' is true, return the directory containing
-    platform-specific modules, i.e. any module from a non-pure-Python
-    module distribution; otherwise, return the platform-shared library
-    directory.  If 'standard_lib' is true, return the directory
-    containing standard Python library modules; otherwise, return the
-    directory for site-specific modules.
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        libpython = os.path.join(prefix,
-                                 "lib", "python" + get_python_version())
-        if standard_lib:
-            return libpython
-        else:
-            return os.path.join(libpython, "site-packages")
-
-    elif os.name == "nt":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            if get_python_version() < "2.2":
-                return prefix
-            else:
-                return os.path.join(prefix, "Lib", "site-packages")
-
-    elif os.name == "os2":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            return os.path.join(prefix, "Lib", "site-packages")
-
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its library "
-            "on platform '%s'" % os.name)
-
-
-
-def customize_compiler(compiler):
-    """Do any platform-specific customization of a CCompiler instance.
-
-    Mainly needed on Unix, so we can plug in the information that
-    varies across Unices and is stored in Python's Makefile.
-    """
-    if compiler.compiler_type == "unix":
-        if sys.platform == "darwin":
-            # Perform first-time customization of compiler-related
-            # config vars on OS X now that we know we need a compiler.
-            # This is primarily to support Pythons from binary
-            # installers.  The kind and paths to build tools on
-            # the user system may vary significantly from the system
-            # that Python itself was built on.  Also the user OS
-            # version and build tools may not support the same set
-            # of CPU architectures for universal builds.
-            global _config_vars
-            # Use get_config_var() to ensure _config_vars is initialized.
-            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
-                import _osx_support
-                _osx_support.customize_compiler(_config_vars)
-                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
-
-        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
-            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
-                            'CCSHARED', 'LDSHARED', 'SO', 'AR',
-                            'ARFLAGS')
-
-        if 'CC' in os.environ:
-            newcc = os.environ['CC']
-            if (sys.platform == 'darwin'
-                    and 'LDSHARED' not in os.environ
-                    and ldshared.startswith(cc)):
-                # On OS X, if CC is overridden, use that as the default
-                #       command for LDSHARED as well
-                ldshared = newcc + ldshared[len(cc):]
-            cc = newcc
-        if 'CXX' in os.environ:
-            cxx = os.environ['CXX']
-        if 'LDSHARED' in os.environ:
-            ldshared = os.environ['LDSHARED']
-        if 'CPP' in os.environ:
-            cpp = os.environ['CPP']
-        else:
-            cpp = cc + " -E"           # not always
-        if 'LDFLAGS' in os.environ:
-            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
-        if 'CFLAGS' in os.environ:
-            cflags = opt + ' ' + os.environ['CFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CFLAGS']
-        if 'CPPFLAGS' in os.environ:
-            cpp = cpp + ' ' + os.environ['CPPFLAGS']
-            cflags = cflags + ' ' + os.environ['CPPFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
-        if 'AR' in os.environ:
-            ar = os.environ['AR']
-        if 'ARFLAGS' in os.environ:
-            archiver = ar + ' ' + os.environ['ARFLAGS']
-        else:
-            archiver = ar + ' ' + ar_flags
-
-        cc_cmd = cc + ' ' + cflags
-        compiler.set_executables(
-            preprocessor=cpp,
-            compiler=cc_cmd,
-            compiler_so=cc_cmd + ' ' + ccshared,
-            compiler_cxx=cxx,
-            linker_so=ldshared,
-            linker_exe=cc,
-            archiver=archiver)
-
-        compiler.shared_lib_extension = so_ext
-
-
-def get_config_h_filename():
-    """Return full pathname of installed pyconfig.h file."""
-    if python_build:
-        if os.name == "nt":
-            inc_dir = os.path.join(project_base, "PC")
-        else:
-            inc_dir = project_base
-    else:
-        inc_dir = get_python_inc(plat_specific=1)
-    if get_python_version() < '2.2':
-        config_h = 'config.h'
-    else:
-        # The name of the config.h file changed in 2.2
-        config_h = 'pyconfig.h'
-    return os.path.join(inc_dir, config_h)
-
-
-def get_makefile_filename():
-    """Return full pathname of installed Makefile from the Python build."""
-    if python_build:
-        return os.path.join(project_base, "Makefile")
-    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
-    return os.path.join(lib_dir, "config", "Makefile")
-
-
-def parse_config_h(fp, g=None):
-    """Parse a config.h-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    if g is None:
-        g = {}
-    define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
-    undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
-    #
-    while 1:
-        line = fp.readline()
-        if not line:
-            break
-        m = define_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            try: v = int(v)
-            except ValueError: pass
-            g[n] = v
-        else:
-            m = undef_rx.match(line)
-            if m:
-                g[m.group(1)] = 0
-    return g
-
-
-# Regexes needed for parsing Makefile (and similar syntaxes,
-# like old-style Setup files).
-_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-def parse_makefile(fn, g=None):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    from distutils.text_file import TextFile
-    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)
-
-    if g is None:
-        g = {}
-    done = {}
-    notdone = {}
-
-    while 1:
-        line = fp.readline()
-        if line is None:  # eof
-            break
-        m = _variable_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    while notdone:
-        for name in notdone.keys():
-            value = notdone[name]
-            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
-            if m:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-                else:
-                    done[n] = item = ""
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try: value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        del notdone[name]
-            else:
-                # bogus variable reference; just drop it since we can't deal
-                del notdone[name]
-
-    fp.close()
-
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    g.update(done)
-    return g
-
-
-def expand_makefile_vars(s, vars):
-    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
-    'string' according to 'vars' (a dictionary mapping variable names to
-    values).  Variables not present in 'vars' are silently expanded to the
-    empty string.  The variable values in 'vars' should not contain further
-    variable expansions; if 'vars' is the output of 'parse_makefile()',
-    you're fine.  Returns a variable-expanded version of 's'.
-    """
-
-    # This algorithm does multiple expansion, so if vars['foo'] contains
-    # "${bar}", it will expand ${foo} to ${bar}, and then expand
-    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
-    # 'parse_makefile()', which takes care of such expansions eagerly,
-    # according to make's variable expansion semantics.
-
-    while 1:
-        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
-        if m:
-            (beg, end) = m.span()
-            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
-        else:
-            break
-    return s
-
-
-_config_vars = None
-
-def _init_posix():
-    """Initialize the module as appropriate for POSIX systems."""
-    # _sysconfigdata is generated at build time, see the sysconfig module
-    from _sysconfigdata import build_time_vars
-    global _config_vars
-    _config_vars = {}
-    _config_vars.update(build_time_vars)
-
-
-def _init_nt():
-    """Initialize the module as appropriate for NT"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-    g['VERSION'] = get_python_version().replace(".", "")
-    g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_os2():
-    """Initialize the module as appropriate for OS/2"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-
-    global _config_vars
-    _config_vars = g
-
-
-def get_config_vars(*args):
-    """With no arguments, return a dictionary of all configuration
-    variables relevant for the current platform.  Generally this includes
-    everything needed to build extensions and install both pure modules and
-    extensions.  On Unix, this means every variable defined in Python's
-    installed Makefile; on Windows and Mac OS it's a much smaller set.
-
-    With arguments, return a list of values that result from looking up
-    each argument in the configuration variable dictionary.
-    """
-    global _config_vars
-    if _config_vars is None:
-        func = globals().get("_init_" + os.name)
-        if func:
-            func()
-        else:
-            _config_vars = {}
-
-        # Normalized versions of prefix and exec_prefix are handy to have;
-        # in fact, these are the standard versions used most places in the
-        # Distutils.
-        _config_vars['prefix'] = PREFIX
-        _config_vars['exec_prefix'] = EXEC_PREFIX
-
-        # OS X platforms require special customization to handle
-        # multi-architecture, multi-os-version installers
-        if sys.platform == 'darwin':
-            import _osx_support
-            _osx_support.customize_config_vars(_config_vars)
-
-    if args:
-        vals = []
-        for name in args:
-            vals.append(_config_vars.get(name))
-        return vals
-    else:
-        return _config_vars
-
-def get_config_var(name):
-    """Return the value of a single variable using the dictionary
-    returned by 'get_config_vars()'.  Equivalent to
-    get_config_vars().get(name)
-    """
-    return get_config_vars().get(name)
->>>>>>> other

From noreply at buildbot.pypy.org  Sun Dec 14 20:16:44 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 14 Dec 2014 20:16:44 +0100 (CET)
Subject: [pypy-commit] pypy default: formatting
Message-ID: <20141214191645.07AE71D2859@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74933:f2fc3729ff99
Date: 2014-12-14 21:16 +0200
http://bitbucket.org/pypy/pypy/changeset/f2fc3729ff99/

Log:	formatting

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -306,7 +306,9 @@
         jumpop.initarglist(jumpargs)
 
         # Inline the short preamble at the end of the loop
-        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values,
+                                                         self.optimizer,
+                                                         keyboxes=True)
         assert len(short_inputargs) == len(jmp_to_short_args)
         args = {}
         for i in range(len(short_inputargs)):

From noreply at buildbot.pypy.org  Sun Dec 14 21:14:45 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 14 Dec 2014 21:14:45 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Implement
 _hashlib.openssl_md_meth_names, needed by hashlib.py
Message-ID: <20141214201445.276AB1D24B5@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74934:1d1bd5f19364
Date: 2011-10-22 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/1d1bd5f19364/

Log:	Implement _hashlib.openssl_md_meth_names, needed by hashlib.py

diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py
--- a/pypy/module/_hashlib/__init__.py
+++ b/pypy/module/_hashlib/__init__.py
@@ -5,6 +5,7 @@
 class Module(MixedModule):
     interpleveldefs = {
         'new' : 'interp_hashlib.new',
+        'openssl_md_meth_names': 'interp_hashlib.get(space).w_meth_names'
         }
 
     appleveldefs = {
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -1,17 +1,62 @@
 from __future__ import with_statement
+
+from rpython.rlib import rgc, ropenssl
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rstring import StringBuilder
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.tool.sourcetools import func_renamer
+
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec, interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.interpreter.error import OperationError
-from rpython.tool.sourcetools import func_renamer
-from pypy.interpreter.baseobjspace import W_Root
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib import rgc, ropenssl
-from rpython.rlib.rstring import StringBuilder
 from pypy.module.thread.os_lock import Lock
 
 
 algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
+def hash_name_mapper_callback(obj_name, userdata):
+    state = global_state[0]
+    assert state is not None
+    if not obj_name:
+        return
+    # Ignore aliased names, they pollute the list and OpenSSL appears
+    # to have a its own definition of alias as the resulting list
+    # still contains duplicate and alternate names for several
+    # algorithms.
+    if obj_name[0].c_alias:
+        return
+    try:
+        w_name = state.space.wrap(rffi.charp2str(obj_name[0].c_name))
+        state.space.call_method(state.w_meth_names, "add", w_name)
+    except OperationError, e:
+        state.w_error = e
+
+# XXX make it threadlocal?
+global_state = [None]
+
+class State:
+    def __init__(self, space):
+        self.space = space
+        self.generate_method_names(space)
+
+    def generate_method_names(self, space):
+        if not we_are_translated():
+            ropenssl.init_digests()
+        self.w_error = None
+        try:
+            global_state[0] = self
+            self.w_meth_names = space.call_function(space.w_set)
+            ropenssl.OBJ_NAME_do_all(
+                ropenssl.OBJ_NAME_TYPE_MD_METH,
+                hash_name_mapper_callback, None)
+        finally:
+            global_state[0] = None
+        if self.w_error:
+            raise self.w_error
+
+def get(space):
+    return space.fromcache(State)
 
 class W_Hash(W_Root):
     NULL_CTX = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -3,6 +3,11 @@
         "usemodules": ['_hashlib', 'array', 'struct', 'binascii'],
     }
 
+    def test_method_names(self):
+        import _hashlib
+        assert isinstance(_hashlib.openssl_md_meth_names, set)
+        assert "md5" in _hashlib.openssl_md_meth_names
+
     def test_simple(self):
         import _hashlib
         assert _hashlib.new('md5').__class__.__name__ == 'HASH'
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -144,6 +144,12 @@
          ('name', rffi.CCHARP),
          ])
 
+    OBJ_NAME_st = rffi_platform.Struct(
+        'OBJ_NAME',
+        [('alias', rffi.INT),
+         ('name', rffi.CCHARP),
+         ])
+
 
 for k, v in rffi_platform.configure(CConfig).items():
     globals()[k] = v
@@ -343,6 +349,11 @@
 HASH_MALLOC_SIZE = EVP_MD_SIZE + EVP_MD_CTX_SIZE \
         + rffi.sizeof(EVP_MD) * 2 + 208
 
+OBJ_NAME_CALLBACK = lltype.Ptr(lltype.FuncType(
+        [OBJ_NAME, rffi.VOIDP], lltype.Void))
+OBJ_NAME_do_all = external(
+    'OBJ_NAME_do_all', [rffi.INT, OBJ_NAME_CALLBACK, rffi.VOIDP], lltype.Void)
+
 def init_ssl():
     libssl_SSL_load_error_strings()
     libssl_SSL_library_init()

From noreply at buildbot.pypy.org  Sun Dec 14 21:17:10 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 14 Dec 2014 21:17:10 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Added the needed spaceconfig
Message-ID: <20141214201710.634721D24B5@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74935:ec3a44263c5b
Date: 2014-12-14 12:17 -0800
http://bitbucket.org/pypy/pypy/changeset/ec3a44263c5b/

Log:	Added the needed spaceconfig

diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -403,6 +403,8 @@
 
 
 class AppTestSimpleSearches:
+    spaceconfig = {"usemodules": ['array']}
+
     def test_search_simple_literal(self):
         import re
         assert re.search("bla", "bla")

From noreply at buildbot.pypy.org  Sun Dec 14 23:57:59 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sun, 14 Dec 2014 23:57:59 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: add test_ensurepip
Message-ID: <20141214225759.C42F61D2859@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: stdlib-2.7.9
Changeset: r74936:cc6be548db9d
Date: 2014-12-14 14:57 -0800
http://bitbucket.org/pypy/pypy/changeset/cc6be548db9d/

Log:	add test_ensurepip

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -205,6 +205,7 @@
     RegrTest('test_email.py'),
     RegrTest('test_email_codecs.py'),
     RegrTest('test_email_renamed.py'),
+    RegrTest('test_ensurepip.py'),
     RegrTest('test_enumerate.py', core=True),
     RegrTest('test_eof.py', core=True),
     RegrTest('test_epoll.py'),

From noreply at buildbot.pypy.org  Mon Dec 15 12:57:54 2014
From: noreply at buildbot.pypy.org (vext01)
Date: Mon, 15 Dec 2014 12:57:54 +0100 (CET)
Subject: [pypy-commit] benchmarks min_5_secs: Fix a bug in the Richards
	benchmark.
Message-ID: <20141215115754.AA26F1C01C3@cobra.cs.uni-duesseldorf.de>

Author: Edd Barrett <vext01 at gmail.com>
Branch: min_5_secs
Changeset: r300:b7f0946c13a6
Date: 2014-12-15 11:58 +0000
http://bitbucket.org/pypy/benchmarks/changeset/b7f0946c13a6/

Log:	Fix a bug in the Richards benchmark.

	Spotted by Laurie, confirmed by me, fix from Armin.

diff --git a/unladen_swallow/performance/richards.py b/unladen_swallow/performance/richards.py
--- a/unladen_swallow/performance/richards.py
+++ b/unladen_swallow/performance/richards.py
@@ -363,8 +363,10 @@
 
     def run(self, iterations):
         for i in xrange(iterations):
-            taskWorkArea.holdCount = 0
-            taskWorkArea.qpktCount = 0
+            global taskWorkArea
+            # Must reinitiliase the taskWorkArea or benchmark will run slower
+            # upon subsequent runs.
+            taskWorkArea = TaskWorkArea()
 
             IdleTask(I_IDLE, 1, 10000, TaskState().running(), IdleTaskRec())
 

From noreply at buildbot.pypy.org  Mon Dec 15 13:08:47 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Dec 2014 13:08:47 +0100 (CET)
Subject: [pypy-commit] pypy optresult: Start porting the tests bit by bit,
 10% of optimizeopt passes
Message-ID: <20141215120847.8352B1C01C3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74937:5f24f3b47ef4
Date: 2014-12-14 21:38 +0200
http://bitbucket.org/pypy/pypy/changeset/5f24f3b47ef4/

Log:	Start porting the tests bit by bit, 10% of optimizeopt passes

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -94,15 +94,9 @@
         raise NotImplementedError
     getref._annspecialcase_ = 'specialize:arg(1)'
 
-    def clonebox(self):
-        raise NotImplementedError
-
     def constbox(self):
         raise NotImplementedError
 
-    def nonconstbox(self):
-        raise NotImplementedError
-
     def getaddr(self):
         "Only for raw addresses (BoxInt & ConstInt), not for GC addresses"
         raise NotImplementedError
@@ -235,11 +229,6 @@
                 assert isinstance(value, Symbolic)
         self.value = value
 
-    def clonebox(self):
-        return BoxInt(self.value)
-
-    nonconstbox = clonebox
-
     def getint(self):
         return self.value
 
@@ -277,11 +266,6 @@
         assert lltype.typeOf(valuestorage) is longlong.FLOATSTORAGE
         self.value = valuestorage
 
-    def clonebox(self):
-        return BoxFloat(self.value)
-
-    nonconstbox = clonebox
-
     def getfloatstorage(self):
         return self.value
 
@@ -324,11 +308,6 @@
         assert lltype.typeOf(value) == llmemory.GCREF
         self.value = value
 
-    def clonebox(self):
-        return BoxPtr(self.value)
-
-    nonconstbox = clonebox
-
     def getref_base(self):
         return self.value
 
@@ -435,9 +414,6 @@
     def forget_value(self):
         self.value = 0
 
-    def clonebox(self):
-        return BoxInt(self.value)
-
     def constbox(self):
         return ConstInt(self.value)
 
@@ -471,9 +447,6 @@
     def forget_value(self):
         self.value = longlong.ZEROF
 
-    def clonebox(self):
-        return BoxFloat(self.value)
-
     def constbox(self):
         return ConstFloat(self.value)
 
@@ -504,9 +477,6 @@
     def forget_value(self):
         self.value = lltype.nullptr(llmemory.GCREF.TO)
 
-    def clonebox(self):
-        return BoxPtr(self.value)
-
     def constbox(self):
         return ConstPtr(self.value)
 
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -145,12 +145,12 @@
             if structvalue in self._cached_fields:
                 if op.getopnum() == rop.SETFIELD_GC:
                     result = op.getarg(1)
+                    opnum = OpHelpers.getfield_for_descr(op.getdescr())
+                    getop = ResOperation(opnum, [op.getarg(0)],
+                                         op.getdescr())
+                    getop.is_source_op = True
                     if isinstance(result, Const):
-                        newresult = result.clonebox()
-                        optimizer.make_constant(newresult, result)
-                        result = newresult
-                    getop = ResOperation(rop.GETFIELD_GC, [op.getarg(0)],
-                                         result, op.getdescr())
+                        optimizer.make_constant(getop, result)
                     shortboxes.add_potential(getop, synthetic=True)
                 if op.getopnum() == rop.SETARRAYITEM_GC:
                     result = op.getarg(2)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -543,14 +543,14 @@
     def test_ooisnull_on_null_ptr_1(self):
         ops = """
         []
-        p0 = escape()
+        p0 = escape_r()
         guard_isnull(p0) []
         guard_isnull(p0) []
         jump()
         """
         expected = """
         []
-        p0 = escape()
+        p0 = escape_r()
         guard_isnull(p0) []
         jump()
         """
@@ -562,7 +562,7 @@
         pv = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(pv, p0, descr=valuedescr)
         guard_nonnull(p0) []
-        p1 = getfield_gc(pv, descr=valuedescr)
+        p1 = getfield_gc_r(pv, descr=valuedescr)
         guard_nonnull(p1) []
         jump(p0)
         """
@@ -641,13 +641,13 @@
         i3 = ptr_eq(NULL, p0)
         guard_false(i3) []
         guard_nonnull(p0) []
-        escape(p0)
+        escape_r(p0)
         jump()
         """
         expected = """
         []
         p0 = new_array(5, descr=arraydescr)
-        escape(p0)
+        escape_r(p0)
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -668,7 +668,7 @@
     def test_constptr_guard_value(self):
         ops = """
         []
-        p1 = escape()
+        p1 = escape_r()
         guard_value(p1, ConstPtr(myptr)) []
         jump()
         """
@@ -755,7 +755,7 @@
     def test_compare_with_itself(self):
         ops = """
         []
-        i0 = escape()
+        i0 = escape_i()
         i1 = int_lt(i0, i0)
         guard_false(i1) []
         i2 = int_le(i0, i0)
@@ -772,7 +772,7 @@
         """
         expected = """
         []
-        i0 = escape()
+        i0 = escape_i()
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -802,22 +802,22 @@
     def test_p123_simple(self):
         ops = """
         [i1, p2, p3]
-        i3 = getfield_gc(p3, descr=valuedescr)
-        escape(i3)
+        i3 = getfield_gc_i(p3, descr=valuedescr)
+        escape_n(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
         jump(i1, p1, p2)
         """
         preamble = """
         [i1, p2, p3]
-        i3 = getfield_gc(p3, descr=valuedescr)
-        escape(i3)
+        i3 = getfield_gc_i(p3, descr=valuedescr)
+        escape_n(i3)
         jump(i1, p2)
         """
         expected = """
         [i1, p2]
-        i3 = getfield_gc(p2, descr=valuedescr)
-        escape(i3)
+        i3 = getfield_gc_i(p2, descr=valuedescr)
+        escape_n(i3)
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i1, descr=valuedescr)
         jump(i1, p3)
@@ -828,8 +828,8 @@
     def test_p123_nested(self):
         ops = """
         [i1, p2, p3]
-        i3 = getfield_gc(p3, descr=valuedescr)
-        escape(i3)
+        i3 = getfield_gc_i(p3, descr=valuedescr)
+        escape_n(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
         p1sub = new_with_vtable(ConstClass(node_vtable2))
@@ -841,14 +841,14 @@
         # virtual.
         preamble = """
         [i1, p2, p3]
-        i3 = getfield_gc(p3, descr=valuedescr)
-        escape(i3)
+        i3 = getfield_gc_i(p3, descr=valuedescr)
+        escape_n(i3)
         jump(i1, p2)
         """
         expected = """
         [i1, p2]
-        i3 = getfield_gc(p2, descr=valuedescr)
-        escape(i3)
+        i3 = getfield_gc_i(p2, descr=valuedescr)
+        escape_n(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
         p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
@@ -861,9 +861,9 @@
     def test_p123_anti_nested(self):
         ops = """
         [i1, p2, p3]
-        p3sub = getfield_gc(p3, descr=nextdescr)
-        i3 = getfield_gc(p3sub, descr=valuedescr)
-        escape(i3)
+        p3sub = getfield_gc_r(p3, descr=nextdescr)
+        i3 = getfield_gc_i(p3sub, descr=valuedescr)
+        escape_n(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
@@ -874,9 +874,9 @@
         # a "young" virtual p2sub.  Make sure it is all forced.
         preamble = """
         [i1, p2, p3]
-        p3sub = getfield_gc(p3, descr=nextdescr)
-        i3 = getfield_gc(p3sub, descr=valuedescr)
-        escape(i3)
+        p3sub = getfield_gc_r(p3, descr=nextdescr)
+        i3 = getfield_gc_i(p3sub, descr=valuedescr)
+        escape_n(i3)
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
@@ -884,8 +884,8 @@
         """
         expected = """
         [i1, p2, p2sub]
-        i3 = getfield_gc(p2sub, descr=valuedescr)
-        escape(i3)
+        i3 = getfield_gc_i(p2sub, descr=valuedescr)
+        escape_n(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -385,7 +385,7 @@
         assert len(optimized.inputargs) == len(expected.inputargs)
         remap = {}
         for box1, box2 in zip(optimized.inputargs, expected.inputargs):
-            assert box1.__class__ == box2.__class__
+            assert box1.type == box2.type
             remap[box2] = box1
         assert equaloplists(optimized.operations,
                             expected.operations, False, remap, text_right)
@@ -419,7 +419,7 @@
 
         jump_args = jumpop.getarglist()[:]
         operations = operations[:-1]
-        memo = compile.Memo()
+        memo = compile.Memo(inputargs, jump_args)
         cloned_operations = [op.clone(memo) for op in operations]
         for op in cloned_operations:
             op.is_source_op = True
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -280,7 +280,7 @@
 
         # We dont need to inline the short preamble we are creating as we are conneting
         # the bridge to a different trace with a different short preamble
-        self.short_inliner = None
+        self.memo = None
 
         newoperations = self.optimizer.get_newoperations()
         self.boxes_created_this_iteration = {}
@@ -384,13 +384,14 @@
         assert isinstance(target_token, TargetToken)
 
         # Turn guards into conditional jumps to the preamble
-        for i in range(len(short)):
-            op = short[i]
-            if op.is_guard():
-                #op = op.clone() XXXX
-                op.setfailargs(None)
-                op.setdescr(None) # will be set to a proper descr when the preamble is used
-                short[i] = op
+        #for i in range(len(short)):
+        #    op = short[i]
+        #    if op.is_guard():
+                #op = op.clone(self.memo)
+                #op.is_source_op = True
+        #        op.setfailargs(None)
+        #        op.setdescr(None) # will be set to a proper descr when the preamble is used
+        #        short[i] = op
 
         # Clone ops and boxes to get private versions and
         short_inputargs = short[0].getarglist()
@@ -408,7 +409,10 @@
         for i in range(len(short)):
             op = short[i]
             newop = op.clone(memo)
-            if op.type != 'v' and op in self.short_boxes.assumed_classes:
+            if newop.is_guard():
+                newop.setfailargs(None)
+                newop.setdescr(None)
+            if op in self.short_boxes.assumed_classes:
                 target_token.assumed_classes[newop] = self.short_boxes.assumed_classes[op]
             short[i] = newop
 
@@ -443,8 +447,8 @@
         if op is None:
             return None
         if op is not None and op in self.short_seen:
-            if emit and self.short_inliner:
-                return self.short_inliner.inline_arg(op.result)
+            if emit and self.memo:
+                return self.memo.get(op, op)
             else:
                 return None
 
@@ -487,7 +491,7 @@
         short_op = self.short_boxes.producer(box)
         newresult = self.add_op_to_short(short_op)
 
-        short_jumpargs.append(short_op.result)
+        short_jumpargs.append(short_op)
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -161,7 +161,12 @@
                                              ['descr=%r' % descr]))
 
     def repr_short(self, memo):
-        return memo[self]
+        try:
+            return memo[self]
+        except KeyError:
+            name = self.type + str(len(memo))
+            memo[self] = name
+            return name
 
     def getopname(self):
         try:
@@ -327,6 +332,9 @@
     def nonnull(self):
         return self._resint != 0
 
+    def clone_input_arg(self):
+        return InputArgInt()
+
 class FloatOp(object):
     _mixin_ = True
 
@@ -349,6 +357,9 @@
     def nonnull(self):
         return bool(longlong.extract_bits(self._resfloat))
 
+    def clone_input_arg(self):
+        return InputArgFloat()
+
 class RefOp(object):
     _mixin_ = True
 
@@ -373,6 +384,9 @@
     def nonnull(self):
         return bool(self._resref)
 
+    def clone_input_arg(self):
+        return InputArgRef()
+
 class AbstractInputArg(AbstractValue):
     is_source_op = True
     source_op = None

From noreply at buildbot.pypy.org  Mon Dec 15 13:08:48 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 15 Dec 2014 13:08:48 +0100 (CET)
Subject: [pypy-commit] pypy optresult: Some progress on unrolling
Message-ID: <20141215120848.E64B51C01C3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74938:3cb5d15021b4
Date: 2014-12-15 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/3cb5d15021b4/

Log:	Some progress on unrolling

diff too long, truncating to 2000 out of 2397 lines

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -148,22 +148,21 @@
                     opnum = OpHelpers.getfield_for_descr(op.getdescr())
                     getop = ResOperation(opnum, [op.getarg(0)],
                                          op.getdescr())
-                    getop.is_source_op = True
+                    getop.source_op = result
                     if isinstance(result, Const):
                         optimizer.make_constant(getop, result)
-                    shortboxes.add_potential(getop, synthetic=True)
+                    shortboxes.add_potential(result, getop, synthetic=True)
                 if op.getopnum() == rop.SETARRAYITEM_GC:
                     result = op.getarg(2)
+                    opnum = OpHelpers.getarrayitem_for_descr(op.getdescr())
+                    getop = ResOperation(opnum, [op.getarg(0), op.getarg(1)],
+                                         op.getdescr())
+                    getop.source_op = result
                     if isinstance(result, Const):
-                        newresult = result.clonebox()
-                        optimizer.make_constant(newresult, result)
-                        result = newresult
-                    getop = ResOperation(rop.GETARRAYITEM_GC, [op.getarg(0), op.getarg(1)],
-                                         result, op.getdescr())
-                    shortboxes.add_potential(getop, synthetic=True)
+                        optimizer.make_constant(getop, result)
+                    shortboxes.add_potential(result, getop, synthetic=True)
                 elif op.type != 'v':
-                    xxxx
-                    shortboxes.add_potential(op)
+                    shortboxes.add_potential(op, op)
 
 class BogusPureField(JitException):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -124,7 +124,7 @@
 
     def produce_potential_short_preamble_ops(self, sb):
         for op in self.emitted_pure_operations:
-            sb.add_potential(op)
+            sb.add_potential(op, op)
 
 dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
                                       default=OptPure.optimize_default)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -23,7 +23,7 @@
 
     def produce_potential_short_preamble_ops(self, sb):
         for op in self.loop_invariant_producer.values():
-            sb.add_potential(op)
+            sb.add_potential(op, op)
 
     def propagate_forward(self, op):
         if op.boolinverse != -1 or op.boolreflex != -1:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -897,7 +897,7 @@
     def test_dont_delay_setfields(self):
         ops = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=nextdescr)
+        i1 = getfield_gc_i(p1, descr=nextdescr)
         i2 = int_sub(i1, 1)
         i2b = int_is_true(i2)
         guard_true(i2b) []
@@ -907,7 +907,7 @@
         """
         preamble = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=nextdescr)
+        i1 = getfield_gc_i(p1, descr=nextdescr)
         i2 = int_sub(i1, 1)
         i2b = int_is_true(i2)
         guard_true(i2b) []
@@ -932,27 +932,27 @@
         [i]
         guard_no_exception() []
         i1 = int_add(i, 3)
-        i2 = call(i1, descr=nonwritedescr)
+        i2 = call_i(i1, descr=nonwritedescr)
         guard_no_exception() [i1, i2]
-        i3 = call(i2, descr=nonwritedescr)
+        i3 = call_i(i2, descr=nonwritedescr)
         jump(i1)       # the exception is considered lost when we loop back
         """
         preamble = """
         [i]
         guard_no_exception() []    # occurs at the start of bridges, so keep it
         i1 = int_add(i, 3)
-        i2 = call(i1, descr=nonwritedescr)
+        i2 = call_i(i1, descr=nonwritedescr)
         guard_no_exception() [i1, i2]
-        i3 = call(i2, descr=nonwritedescr)
+        i3 = call_i(i2, descr=nonwritedescr)
         jump(i1)
         """
         expected = """
         [i]
         guard_no_exception() []    # occurs at the start of bridges, so keep it
         i1 = int_add(i, 3)
-        i2 = call(i1, descr=nonwritedescr)
+        i2 = call_i(i1, descr=nonwritedescr)
         guard_no_exception() [i1, i2]
-        i3 = call(i2, descr=nonwritedescr)
+        i3 = call_i(i2, descr=nonwritedescr)
         jump(i1)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -960,16 +960,16 @@
     def test_bug_guard_no_exception(self):
         ops = """
         []
-        i0 = call(123, descr=nonwritedescr)
-        p0 = call(0, "xy", descr=s2u_descr)      # string -> unicode
+        i0 = call_i(123, descr=nonwritedescr)
+        p0 = call_r(0, "xy", descr=s2u_descr)      # string -> unicode
         guard_no_exception() []
-        escape(p0)
+        escape_n(p0)
         jump()
         """
         expected = """
         []
-        i0 = call(123, descr=nonwritedescr)
-        escape(u"xy")
+        i0 = call_i(123, descr=nonwritedescr)
+        escape_n(u"xy")
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -979,20 +979,20 @@
     def test_call_loopinvariant(self):
         ops = """
         [i1]
-        i2 = call_loopinvariant(1, i1, descr=nonwritedescr)
+        i2 = call_loopinvariant_i(1, i1, descr=nonwritedescr)
         guard_no_exception() []
         guard_value(i2, 1) []
-        i3 = call_loopinvariant(1, i1, descr=nonwritedescr)
+        i3 = call_loopinvariant_i(1, i1, descr=nonwritedescr)
         guard_no_exception() []
         guard_value(i3, 1) []
-        i4 = call_loopinvariant(1, i1, descr=nonwritedescr)
+        i4 = call_loopinvariant_i(1, i1, descr=nonwritedescr)
         guard_no_exception() []
         guard_value(i4, 1) []
         jump(i1)
         """
         preamble = """
         [i1]
-        i2 = call(1, i1, descr=nonwritedescr)
+        i2 = call_i(1, i1, descr=nonwritedescr)
         guard_no_exception() []
         guard_value(i2, 1) []
         jump(i1)
@@ -1008,7 +1008,7 @@
     def test_virtual_1(self):
         ops = """
         [i, p0]
-        i0 = getfield_gc(p0, descr=valuedescr)
+        i0 = getfield_gc_i(p0, descr=valuedescr)
         i1 = int_add(i0, i)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
@@ -1016,7 +1016,7 @@
         """
         preamble = """
         [i, p0]
-        i0 = getfield_gc(p0, descr=valuedescr)
+        i0 = getfield_gc_i(p0, descr=valuedescr)
         i1 = int_add(i0, i)
         jump(i, i1)
         """
@@ -1030,7 +1030,7 @@
     def test_virtual_float(self):
         ops = """
         [f, p0]
-        f0 = getfield_gc(p0, descr=floatdescr)
+        f0 = getfield_gc_f(p0, descr=floatdescr)
         f1 = float_add(f0, f)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, f1, descr=floatdescr)
@@ -1038,7 +1038,7 @@
         """
         preamble = """
         [f, p0]
-        f2 = getfield_gc(p0, descr=floatdescr)
+        f2 = getfield_gc_f(p0, descr=floatdescr)
         f1 = float_add(f2, f)
         jump(f, f1)
         """
@@ -1100,7 +1100,7 @@
     def test_virtual_default_field(self):
         ops = """
         [p0]
-        i0 = getfield_gc(p0, descr=valuedescr)
+        i0 = getfield_gc_i(p0, descr=valuedescr)
         guard_value(i0, 0) []
         p1 = new_with_vtable(ConstClass(node_vtable))
         # the field 'value' has its default value of 0
@@ -1108,7 +1108,7 @@
         """
         preamble = """
         [p0]
-        i0 = getfield_gc(p0, descr=valuedescr)
+        i0 = getfield_gc_i(p0, descr=valuedescr)
         guard_value(i0, 0) []
         jump()
         """
@@ -1123,7 +1123,7 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        i0 = getfield_gc(p1, descr=valuedescr)
+        i0 = getfield_gc_i(p1, descr=valuedescr)
         i1 = int_add(i0, 1)
         jump(i1)
         """
@@ -1138,7 +1138,7 @@
         ops = """
         [i0, p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i1 = getfield_gc(p0, descr=valuedescr)
+        i1 = getfield_gc_i(p0, descr=valuedescr)
         i2 = int_sub(i1, 1)
         i3 = int_add(i0, i1)
         p1 = new_with_vtable(ConstClass(node_vtable))
@@ -1148,7 +1148,7 @@
         preamble = """
         [i0, p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i1 = getfield_gc(p0, descr=valuedescr)
+        i1 = getfield_gc_i(p0, descr=valuedescr)
         i2 = int_sub(i1, 1)
         i3 = int_add(i0, i1)
         jump(i3, i2)
@@ -1165,7 +1165,7 @@
         ops = """
         [i0, p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i1 = getfield_gc(p0, descr=valuedescr)
+        i1 = getfield_gc_i(p0, descr=valuedescr)
         i2 = int_sub(i1, 1)
         i3 = int_add(i0, i1)
         p2 = new_with_vtable(ConstClass(node_vtable2))
@@ -1178,10 +1178,10 @@
         preamble = """
         [i0, p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i1 = getfield_gc(p0, descr=valuedescr)
+        i1 = getfield_gc_i(p0, descr=valuedescr)
         i2 = int_sub(i1, 1)
         i3 = int_add(i0, i1)
-        i4 = same_as(i2) # This same_as should be killed by backend
+        #i4 = same_as_i(i2) # This same_as should be killed by backend
         jump(i3, i1, i2)
         """
         expected = """
@@ -1195,8 +1195,8 @@
     def test_virtual_recursive(self):
         ops = """
         [p0]
-        p41 = getfield_gc(p0, descr=nextdescr)
-        i0 = getfield_gc(p41, descr=valuedescr)
+        p41 = getfield_gc_r(p0, descr=nextdescr)
+        i0 = getfield_gc_i(p41, descr=valuedescr)
         p1 = new_with_vtable(ConstClass(node_vtable2))
         p2 = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2, p1, descr=nextdescr)
@@ -1207,8 +1207,8 @@
         """
         preamble = """
         [p0]
-        p41 = getfield_gc(p0, descr=nextdescr)
-        i0 = getfield_gc(p41, descr=valuedescr)
+        p41 = getfield_gc_r(p0, descr=nextdescr)
+        i0 = getfield_gc_i(p41, descr=valuedescr)
         i3 = int_add(i0, 1)
         jump(i3)
         """
@@ -1222,8 +1222,8 @@
     def test_virtual_recursive_forced(self):
         ops = """
         [p0]
-        p41 = getfield_gc(p0, descr=nextdescr)
-        i0 = getfield_gc(p41, descr=valuedescr)
+        p41 = getfield_gc_r(p0, descr=nextdescr)
+        i0 = getfield_gc_i(p41, descr=valuedescr)
         p1 = new_with_vtable(ConstClass(node_vtable2))
         p2 = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2, p1, descr=nextdescr)
@@ -1235,8 +1235,8 @@
         """
         preamble = """
         [p0]
-        p41 = getfield_gc(p0, descr=nextdescr)
-        i0 = getfield_gc(p41, descr=valuedescr)
+        p41 = getfield_gc_r(p0, descr=nextdescr)
+        i0 = getfield_gc_i(p41, descr=valuedescr)
         i1 = int_add(i0, 1)
         p1 = new_with_vtable(ConstClass(node_vtable2))
         p2 = new_with_vtable(ConstClass(node_vtable2))
@@ -1248,8 +1248,8 @@
         """
         loop = """
         [p0]
-        p41 = getfield_gc(p0, descr=nextdescr)
-        i0 = getfield_gc(p41, descr=valuedescr)
+        p41 = getfield_gc_r(p0, descr=nextdescr)
+        i0 = getfield_gc_i(p41, descr=valuedescr)
         i1 = int_add(i0, 1)
         p1 = new_with_vtable(ConstClass(node_vtable2))
         p2 = new_with_vtable(ConstClass(node_vtable2))
@@ -1266,7 +1266,7 @@
         [i0]
         p0 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p0, NULL, descr=nextdescr)
-        p2 = getfield_gc(p0, descr=nextdescr)
+        p2 = getfield_gc_r(p0, descr=nextdescr)
         i1 = ptr_eq(p2, NULL)
         jump(i1)
         """
@@ -1285,14 +1285,10 @@
         [i0]
         p0 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p0, ConstPtr(myptr), descr=nextdescr)
-        p2 = getfield_gc(p0, descr=nextdescr)
+        p2 = getfield_gc_r(p0, descr=nextdescr)
         i1 = ptr_eq(p2, NULL)
         jump(i1)
         """
-        preamble = """
-        [i0]
-        jump()
-        """
         expected = """
         []
         jump()
@@ -1306,7 +1302,7 @@
         p30 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p30, i28, descr=nextdescr)
         setfield_gc(p3, p30, descr=valuedescr)
-        p45 = getfield_gc(p3, descr=valuedescr)
+        p45 = getfield_gc_r(p3, descr=valuedescr)
         i29 = int_add(i28, 1)
         jump(i29, p45, p3)
         """
@@ -1317,7 +1313,7 @@
         p30 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p30, i28, descr=nextdescr)
         setfield_gc(p3, p30, descr=valuedescr)
-        p46 = same_as(p30) # This same_as should be killed by backend
+        #p46 = same_as(p30) # This same_as should be killed by backend
         jump(i29, p30, p3)
         """
         expected = """
@@ -1336,10 +1332,10 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        i0 = getfield_gc(p1, descr=valuedescr)
+        i0 = getfield_gc_i(p1, descr=valuedescr)
         i1 = int_add(i0, 1)
-        escape(p1)
-        escape(p1)
+        escape_n(p1)
+        escape_n(p1)
         jump(i1)
         """
         expected = """
@@ -1347,8 +1343,8 @@
         i1 = int_add(i, 1)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        escape(p1)
-        escape(p1)
+        escape_n(p1)
+        escape_n(p1)
         jump(i1)
         """
         self.optimize_loop(ops, expected)
@@ -1356,8 +1352,8 @@
     def test_nonvirtual_2(self):
         ops = """
         [i, p0]
-        i0 = getfield_gc(p0, descr=valuedescr)
-        escape(p0)
+        i0 = getfield_gc_i(p0, descr=valuedescr)
+        escape_n(p0)
         i1 = int_add(i0, i)
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
@@ -1365,8 +1361,8 @@
         """
         preamble = """
         [i, p0]
-        i0 = getfield_gc(p0, descr=valuedescr)
-        escape(p0)
+        i0 = getfield_gc_i(p0, descr=valuedescr)
+        escape_n(p0)
         i1 = int_add(i0, i)
         jump(i, i1)
         """
@@ -1374,7 +1370,7 @@
         [i, i1]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
-        escape(p1)
+        escape_n(p1)
         i2 = int_add(i1, i)
         jump(i, i2)
         """
@@ -1385,9 +1381,9 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        i1 = getfield_gc(p1, descr=valuedescr)
-        escape(p1)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        escape_n(p1)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         i3 = int_add(i1, i2)
         jump(i3)
         """
@@ -1395,8 +1391,8 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        escape(p1)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        escape_n(p1)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         i3 = int_add(i, i2)
         jump(i3)
         """
@@ -1407,18 +1403,18 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         setfield_gc(p1, 0, descr=valuedescr)
-        escape(p1)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        escape_n(p1)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         jump(i2)
         """
         expected = """
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, 0, descr=valuedescr)
-        escape(p1)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        escape_n(p1)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         jump(i2)
         """
         self.optimize_loop(ops, expected)
@@ -1428,7 +1424,7 @@
         [i]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i, descr=valuedescr)
-        i1 = getfield_gc_pure(p1, descr=valuedescr)
+        i1 = getfield_gc_pure_i(p1, descr=valuedescr)
         jump(i1)
         """
         expected = """
@@ -1440,7 +1436,7 @@
     def test_getfield_gc_pure_2(self):
         ops = """
         [i]
-        i1 = getfield_gc_pure(ConstPtr(myptr), descr=valuedescr)
+        i1 = getfield_gc_pure_i(ConstPtr(myptr), descr=valuedescr)
         jump(i1)
         """
         expected = """
@@ -1453,19 +1449,19 @@
     def test_getfield_gc_pure_3(self):
         ops = """
         []
-        p1 = escape()
-        p2 = getfield_gc_pure(p1, descr=nextdescr)
-        escape(p2)
-        p3 = getfield_gc_pure(p1, descr=nextdescr)
-        escape(p3)
+        p1 = escape_r()
+        p2 = getfield_gc_pure_r(p1, descr=nextdescr)
+        escape_n(p2)
+        p3 = getfield_gc_pure_r(p1, descr=nextdescr)
+        escape_n(p3)
         jump()
         """
         expected = """
         []
-        p1 = escape()
-        p2 = getfield_gc_pure(p1, descr=nextdescr)
-        escape(p2)
-        escape(p2)
+        p1 = escape_r()
+        p2 = getfield_gc_pure_r(p1, descr=nextdescr)
+        escape_n(p2)
+        escape_n(p2)
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -1473,19 +1469,19 @@
     def test_getfield_gc_nonpure_2(self):
         ops = """
         [i]
-        i1 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
-        call(i1, descr=nonwritedescr)
+        i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
+        call_n(i1, descr=nonwritedescr)
         jump(i)
         """
         preamble = """
         [i]
-        i1 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
-        call(i1, descr=nonwritedescr)
+        i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
+        call_n(i1, descr=nonwritedescr)
         jump(i, i1)
         """
         expected = """
         [i, i1]
-        call(i1, descr=nonwritedescr)
+        call_n(i1, descr=nonwritedescr)
         jump(i, i1)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -1493,12 +1489,12 @@
     def test_varray_boxed1(self):
         ops = """
         [p0, p8]
-        p11 = getfield_gc(p0, descr=otherdescr)
+        p11 = getfield_gc_r(p0, descr=otherdescr)
         guard_nonnull(p11) [p0, p8]
         guard_class(p11, ConstClass(node_vtable2)) [p0, p8]
-        p14 = getfield_gc(p11, descr=otherdescr)
+        p14 = getfield_gc_r(p11, descr=otherdescr)
         guard_isnull(p14) [p0, p8]
-        p18 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
+        p18 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
         setfield_gc(p31, 0, descr=adescr)
@@ -1517,7 +1513,7 @@
     def test_varray_boxed_simplified(self):
         ops = """
         [p0, p8]
-        p18 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
+        p18 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
         p35 = new_with_vtable(ConstClass(node_vtable))
@@ -1554,7 +1550,7 @@
         guard_value(i3, 3) []
         setarrayitem_gc(p1, 1, i1, descr=arraydescr)
         setarrayitem_gc(p1, 0, 25, descr=arraydescr)
-        i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         jump(i2)
         """
         expected = """
@@ -1566,9 +1562,9 @@
     def test_varray_clear_unroll_bug(self):
         ops = """
         [p0]
-        i0 = getarrayitem_gc(p0, 0, descr=arraydescr)
-        i1 = getarrayitem_gc(p0, 1, descr=arraydescr)
-        i2 = getarrayitem_gc(p0, 2, descr=arraydescr)
+        i0 = getarrayitem_gc_i(p0, 0, descr=arraydescr)
+        i1 = getarrayitem_gc_i(p0, 1, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p0, 2, descr=arraydescr)
         i3 = int_add(i0, i1)
         i4 = int_add(i3, i2)
         p1 = new_array_clear(3, descr=arraydescr)
@@ -1588,7 +1584,7 @@
         [i1]
         p1 = new_array(2, descr=arraydescr)
         setarrayitem_gc(p1, 0, 25, descr=arraydescr)
-        i2 = getarrayitem_gc(p1, 0, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p1, 0, descr=arraydescr)
         jump(i2)
         """
         preamble = """
@@ -1609,7 +1605,7 @@
         guard_value(i3, 3) []
         setarrayitem_gc(p1, 1, f1, descr=floatarraydescr)
         setarrayitem_gc(p1, 0, 3.5, descr=floatarraydescr)
-        f2 = getarrayitem_gc(p1, 1, descr=floatarraydescr)
+        f2 = getarrayitem_gc_f(p1, 1, descr=floatarraydescr)
         jump(f2)
         """
         expected = """
@@ -1640,7 +1636,7 @@
         p1 = new_array(5, descr=arraydescr)
         setarrayitem_gc(p1, 0, i1, descr=arraydescr)
         setarrayitem_gc(p1, 1, 0, descr=arraydescr)
-        escape(p1)
+        escape_n(p1)
         jump(i1)
         """
         expected = """
@@ -1648,7 +1644,7 @@
         p1 = new_array(5, descr=arraydescr)
         setarrayitem_gc(p1, 0, i1, descr=arraydescr)
         setarrayitem_gc(p1, 1, 0, descr=arraydescr)
-        escape(p1)
+        escape_n(p1)
         jump(i1)
         """
         self.optimize_loop(ops, expected)
@@ -1656,8 +1652,8 @@
     def test_varray_2(self):
         ops = """
         [i0, p1]
-        i1 = getarrayitem_gc(p1, 0, descr=arraydescr)
-        i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         i3 = int_sub(i1, i2)
         guard_value(i3, 15) []
         p2 = new_array(2, descr=arraydescr)
@@ -1667,8 +1663,8 @@
         """
         preamble = """
         [i0, p1]
-        i1 = getarrayitem_gc(p1, 0, descr=arraydescr)
-        i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         i3 = int_sub(i1, i2)
         guard_value(i3, 15) []
         jump(i0)
@@ -1684,22 +1680,22 @@
     def test_p123_array(self):
         ops = """
         [i1, p2, p3]
-        i3 = getarrayitem_gc(p3, 0, descr=arraydescr)
-        escape(i3)
+        i3 = getarrayitem_gc_i(p3, 0, descr=arraydescr)
+        escape_n(i3)
         p1 = new_array(1, descr=arraydescr)
         setarrayitem_gc(p1, 0, i1, descr=arraydescr)
         jump(i1, p1, p2)
         """
         preamble = """
         [i1, p2, p3]
-        i3 = getarrayitem_gc(p3, 0, descr=arraydescr)
-        escape(i3)
+        i3 = getarrayitem_gc_i(p3, 0, descr=arraydescr)
+        escape_n(i3)
         jump(i1, p2)
         """
         expected = """
         [i1, p2]
-        i3 = getarrayitem_gc(p2, 0, descr=arraydescr)
-        escape(i3)
+        i3 = getarrayitem_gc_i(p2, 0, descr=arraydescr)
+        escape_n(i3)
         p1 = new_array(1, descr=arraydescr)
         setarrayitem_gc(p1, 0, i1, descr=arraydescr)
         jump(i1, p1)
@@ -1712,19 +1708,19 @@
         []
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, 3, descr=valuedescr)
-        i1 = getfield_gc(p2, descr=valuedescr)    # i1 = const 3
+        i1 = getfield_gc_i(p2, descr=valuedescr)    # i1 = const 3
         p1 = new_array(i1, descr=arraydescr)
-        escape(p1)
+        escape_n(p1)
         i2 = arraylen_gc(p1)
-        escape(i2)
+        escape_n(i2)
         jump()
         """
         expected = """
         []
         p1 = new_array(3, descr=arraydescr)
-        escape(p1)
+        escape_n(p1)
         i2 = arraylen_gc(p1)
-        escape(i2)
+        escape_n(i2)
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -1732,21 +1728,21 @@
     def test_vstruct_1(self):
         ops = """
         [i1, p2]
-        i2 = getfield_gc(p2, descr=adescr)
-        escape(i2)
+        i2 = getfield_gc_i(p2, descr=adescr)
+        escape_n(i2)
         p3 = new(descr=ssize)
         setfield_gc(p3, i1, descr=adescr)
         jump(i1, p3)
         """
         preamble = """
         [i1, p2]
-        i2 = getfield_gc(p2, descr=adescr)
-        escape(i2)
+        i2 = getfield_gc_i(p2, descr=adescr)
+        escape_n(i2)
         jump(i1)
         """
         expected = """
         [i1]
-        escape(i1)
+        escape_n(i1)
         jump(i1)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -1754,22 +1750,22 @@
     def test_p123_vstruct(self):
         ops = """
         [i1, p2, p3]
-        i3 = getfield_gc(p3, descr=adescr)
-        escape(i3)
+        i3 = getfield_gc_i(p3, descr=adescr)
+        escape_n(i3)
         p1 = new(descr=ssize)
         setfield_gc(p1, i1, descr=adescr)
         jump(i1, p1, p2)
         """
         preamble = """
         [i1, p2, p3]
-        i3 = getfield_gc(p3, descr=adescr)
-        escape(i3)
+        i3 = getfield_gc_i(p3, descr=adescr)
+        escape_n(i3)
         jump(i1, p2)
         """
         expected = """
         [i1, p2]
-        i3 = getfield_gc(p2, descr=adescr)
-        escape(i3)
+        i3 = getfield_gc_i(p2, descr=adescr)
+        escape_n(i3)
         p1 = new(descr=ssize)
         setfield_gc(p1, i1, descr=adescr)
         jump(i1, p1)
@@ -1780,11 +1776,11 @@
     def test_virtual_raw_malloc_basic(self):
         ops = """
         [i1]
-        i2 = call('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
-        i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr)
-        call('free', i2, descr=raw_free_descr)
+        i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
+        call_n('free', i2, descr=raw_free_descr)
         jump(i3)
         """
         expected = """
@@ -1797,11 +1793,11 @@
         ops = """
         [i1]
         i5 = int_mul(10, 1)
-        i2 = call('malloc', i5, descr=raw_malloc_descr)
+        i2 = call_i('malloc', i5, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
-        i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr)
-        call('free', i2, descr=raw_free_descr)
+        i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr)
+        call_n('free', i2, descr=raw_free_descr)
         jump(i3)
         """
         expected = """
@@ -1820,7 +1816,7 @@
         setarrayitem_raw(i2, 1, 123, descr=rawarraydescr_char)
         setarrayitem_raw(i2, 1, 789, descr=rawarraydescr_float)
         label('foo') # we expect the buffer to be forced *after* the label
-        escape(i2)
+        escape_n(i2)
         call('free', i2, descr=raw_free_descr)
         jump(i1)
         """
@@ -1833,7 +1829,7 @@
         raw_store(i2, 1, 123, descr=rawarraydescr_char)
         raw_store(i2, 2, 456, descr=rawarraydescr_char)
         raw_store(i2, 8, 789, descr=rawarraydescr_float)
-        escape(i2)
+        escape_n(i2)
         call('free', i2, descr=raw_free_descr)
         jump(i1)
         """
@@ -1932,7 +1928,7 @@
         i3 = int_add(i2, 1) # get a slice of the original buffer
         setarrayitem_raw(i3, 4, 4242, descr=rawarraydescr_char) # write to the slice
         label('foo')
-        escape(i3)
+        escape_n(i3)
         jump(i0, i1)
         """
         expected = """
@@ -1945,7 +1941,7 @@
         raw_store(i2, 5, 4242, descr=rawarraydescr_char)
         # this is generated by VirtualRawSliceValue._really_force
         i4 = int_add(i2, 1)
-        escape(i4)
+        escape_n(i4)
         jump(i0, i1)
         """
         self.optimize_loop(ops, expected)
@@ -2035,7 +2031,7 @@
         i0 = call('malloc', 16, descr=raw_malloc_descr)
         guard_no_exception() []
         i1 = int_add(i0, 8)
-        escape(i0)
+        escape_n(i0)
         setarrayitem_raw(i1, 0, f1, descr=rawarraydescr_float)
         jump(f1)
         """
@@ -2043,7 +2039,7 @@
         [f1]
         i0 = call('malloc', 16, descr=raw_malloc_descr)
         #guard_no_exception() []  # XXX should appear
-        escape(i0)
+        escape_n(i0)
         i1 = int_add(i0, 8)
         setarrayitem_raw(i1, 0, f1, descr=rawarraydescr_float)
         jump(f1)
@@ -2053,24 +2049,24 @@
     def test_duplicate_getfield_1(self):
         ops = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
-        i2 = getfield_gc(p2, descr=valuedescr)
-        i3 = getfield_gc(p1, descr=valuedescr)
-        i4 = getfield_gc(p2, descr=valuedescr)
-        escape(i1)
-        escape(i2)
-        escape(i3)
-        escape(i4)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p2, descr=valuedescr)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
+        i4 = getfield_gc_i(p2, descr=valuedescr)
+        escape_n(i1)
+        escape_n(i2)
+        escape_n(i3)
+        escape_n(i4)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
-        i2 = getfield_gc(p2, descr=valuedescr)
-        escape(i1)
-        escape(i2)
-        escape(i1)
-        escape(i2)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p2, descr=valuedescr)
+        escape_n(i1)
+        escape_n(i2)
+        escape_n(i1)
+        escape_n(i2)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -2078,10 +2074,10 @@
     def test_duplicate_getfield_2(self):
         ops = """
         [p1, p2, i0]
-        i1 = getfield_gc(p1, descr=valuedescr)
-        i2 = getfield_gc(p2, descr=valuedescr)
-        i3 = getfield_gc(p1, descr=valuedescr)
-        i4 = getfield_gc(p2, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p2, descr=valuedescr)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
+        i4 = getfield_gc_i(p2, descr=valuedescr)
         i5 = int_add(i3, i4)
         i6 = int_add(i0, i5)
         jump(p1, p2, i6)
@@ -2097,14 +2093,14 @@
         ops = """
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
-        i2 = getfield_gc(p1, descr=valuedescr)
-        escape(i2)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
+        escape_n(i2)
         jump(p1, i1)
         """
         expected = """
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
-        escape(i1)
+        escape_n(i1)
         jump(p1, i1)
         """
         self.optimize_loop(ops, expected)
@@ -2114,15 +2110,15 @@
         [p1, p2, i1]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, p1, descr=nextdescr)
-        i2 = getfield_gc(p1, descr=valuedescr)
-        escape(i2)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
+        escape_n(i2)
         jump(p1, p2, i1)
         """
         expected = """
         [p1, p2, i1]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, p1, descr=nextdescr)
-        escape(i1)
+        escape_n(i1)
         jump(p1, p2, i1)
         """
         self.optimize_loop(ops, expected)
@@ -2132,8 +2128,8 @@
         [p1, p2, i1, i2]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, i2, descr=valuedescr)
-        i3 = getfield_gc(p1, descr=valuedescr)
-        escape(i3)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
+        escape_n(i3)
         jump(p1, p2, i1, i3)
         """
         self.optimize_loop(ops, ops)
@@ -2141,19 +2137,19 @@
     def test_duplicate_getfield_mergepoint_has_no_side_effects(self):
         ops = """
         [p1]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         debug_merge_point(15, 0)
-        i2 = getfield_gc(p1, descr=valuedescr)
-        escape(i1)
-        escape(i2)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
+        escape_n(i1)
+        escape_n(i2)
         jump(p1)
         """
         expected = """
         [p1]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         debug_merge_point(15, 0)
-        escape(i1)
-        escape(i1)
+        escape_n(i1)
+        escape_n(i1)
         jump(p1)
         """
         self.optimize_loop(ops, expected)
@@ -2161,21 +2157,21 @@
     def test_duplicate_getfield_ovf_op_does_not_clear(self):
         ops = """
         [p1]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = int_add_ovf(i1, 14)
         guard_no_overflow() []
-        i3 = getfield_gc(p1, descr=valuedescr)
-        escape(i2)
-        escape(i3)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
+        escape_n(i2)
+        escape_n(i3)
         jump(p1)
         """
         expected = """
         [p1]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         i2 = int_add_ovf(i1, 14)
         guard_no_overflow() []
-        escape(i2)
-        escape(i1)
+        escape_n(i2)
+        escape_n(i1)
         jump(p1)
         """
         self.optimize_loop(ops, expected)
@@ -2183,19 +2179,19 @@
     def test_duplicate_getfield_setarrayitem_does_not_clear(self):
         ops = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         setarrayitem_gc(p2, 0, p1, descr=arraydescr2)
-        i3 = getfield_gc(p1, descr=valuedescr)
-        escape(i1)
-        escape(i3)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
+        escape_n(i1)
+        escape_n(i3)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
         setarrayitem_gc(p2, 0, p1, descr=arraydescr2)
-        escape(i1)
-        escape(i1)
+        escape_n(i1)
+        escape_n(i1)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -2203,17 +2199,17 @@
     def test_duplicate_getfield_constant(self):
         ops = """
         []
-        i1 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
-        i2 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
-        escape(i1)
-        escape(i2)
+        i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
+        i2 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
+        escape_n(i1)
+        escape_n(i2)
         jump()
         """
         expected = """
         []
-        i1 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
-        escape(i1)
-        escape(i1)
+        i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
+        escape_n(i1)
+        escape_n(i1)
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -2222,17 +2218,17 @@
         ops = """
         [p1]
         guard_value(p1, ConstPtr(myptr)) []
-        i1 = getfield_gc(p1, descr=valuedescr)
-        i2 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
-        escape(i1)
-        escape(i2)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        i2 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
+        escape_n(i1)
+        escape_n(i2)
         jump(p1)
         """
         expected = """
         []
-        i1 = getfield_gc(ConstPtr(myptr), descr=valuedescr)
-        escape(i1)
-        escape(i1)
+        i1 = getfield_gc_i(ConstPtr(myptr), descr=valuedescr)
+        escape_n(i1)
+        escape_n(i1)
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -2240,11 +2236,11 @@
     def test_duplicate_getfield_sideeffects_1(self):
         ops = """
         [p1]
-        i1 = getfield_gc(p1, descr=valuedescr)
-        escape()
-        i2 = getfield_gc(p1, descr=valuedescr)
-        escape(i1)
-        escape(i2)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        escape_n()
+        i2 = getfield_gc_i(p1, descr=valuedescr)
+        escape_n(i1)
+        escape_n(i2)
         jump(p1)
         """
         self.optimize_loop(ops, ops)
@@ -2253,9 +2249,9 @@
         ops = """
         [p1, i1]
         setfield_gc(p1, i1, descr=valuedescr)
-        escape()
-        i2 = getfield_gc(p1, descr=valuedescr)
-        escape(i2)
+        escape_n()
+        i2 = getfield_gc_i(p1, descr=valuedescr)
+        escape_n(i2)
         jump(p1, i1)
         """
         self.optimize_loop(ops, ops)
@@ -2279,16 +2275,16 @@
     def test_duplicate_setfield_1(self):
         ops = """
         [p1]
-        i1 = escape()
-        i2 = escape()
+        i1 = escape_i()
+        i2 = escape_i()
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p1, i2, descr=valuedescr)
         jump(p1)
         """
         expected = """
         [p1]
-        i1 = escape()
-        i2 = escape()
+        i1 = escape_i()
+        i2 = escape_i()
         setfield_gc(p1, i2, descr=valuedescr)
         jump(p1)
         """
@@ -2298,15 +2294,15 @@
         ops = """
         [p1, i1, i3]
         setfield_gc(p1, i1, descr=valuedescr)
-        i2 = getfield_gc(p1, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=valuedescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        escape(i2)
+        escape_n(i2)
         jump(p1, i1, i3)
         """
         expected = """
         [p1, i1, i3]
         setfield_gc(p1, i3, descr=valuedescr)
-        escape(i1)
+        escape_n(i1)
         jump(p1, i1, i3)
         """
         self.optimize_loop(ops, expected)
@@ -2315,9 +2311,9 @@
         ops = """
         [p1, p2, i1, i3]
         setfield_gc(p1, i1, descr=valuedescr)
-        i2 = getfield_gc(p2, descr=valuedescr)
+        i2 = getfield_gc_i(p2, descr=valuedescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        escape(i2)
+        escape_n(i2)
         jump(p1, p2, i1, i3)
         """
         # potential aliasing of p1 and p2 means that we cannot kill the
@@ -2331,38 +2327,38 @@
         #
         # some operations on which the above setfield_gc cannot have effect
         i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr)
-        i4 = getarrayitem_gc(p3, i3, descr=arraydescr)
+        i4 = getarrayitem_gc_i(p3, i3, descr=arraydescr)
         i5 = int_add(i3, i4)
         setarrayitem_gc(p3, 0, i5, descr=arraydescr)
         setfield_gc(p1, i4, descr=nextdescr)
         #
         setfield_gc(p1, i2, descr=valuedescr)
-        escape()
+        escape_n()
         jump(p1, i1, i2, p3)
         """
         preamble = """
         [p1, i1, i2, p3]
         #
         i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr)
-        i4 = getarrayitem_gc(p3, i3, descr=arraydescr)
+        i4 = getarrayitem_gc_i(p3, i3, descr=arraydescr)
         i5 = int_add(i3, i4)
         #
         setfield_gc(p1, i2, descr=valuedescr)
         setarrayitem_gc(p3, 0, i5, descr=arraydescr)
         setfield_gc(p1, i4, descr=nextdescr)
-        escape()
+        escape_n()
         jump(p1, i1, i2, p3, i3)
         """
         expected = """
         [p1, i1, i2, p3, i3]
         #
-        i4 = getarrayitem_gc(p3, i3, descr=arraydescr)
+        i4 = getarrayitem_gc_i(p3, i3, descr=arraydescr)
         i5 = int_add(i3, i4)
         #
         setfield_gc(p1, i2, descr=valuedescr)
         setarrayitem_gc(p3, 0, i5, descr=arraydescr)
         setfield_gc(p1, i4, descr=nextdescr)
-        escape()
+        escape_n()
         jump(p1, i1, i2, p3, i3)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -2375,16 +2371,16 @@
         setfield_gc(p0, p1, descr=nextdescr)
         setfield_raw(i1, i1, descr=valuedescr)    # random op with side-effects
         p2 = getfield_gc(p0, descr=nextdescr)
-        i2 = getfield_gc(p2, descr=valuedescr)
+        i2 = getfield_gc_i(p2, descr=valuedescr)
         setfield_gc(p0, NULL, descr=nextdescr)
-        escape(i2)
+        escape_n(i2)
         jump(p0, i1)
         """
         expected = """
         [p0, i1]
         setfield_raw(i1, i1, descr=valuedescr)
         setfield_gc(p0, NULL, descr=nextdescr)
-        escape(i1)
+        escape_n(i1)
         jump(p0, i1)
         """
         self.optimize_loop(ops, expected)
@@ -2393,7 +2389,7 @@
         ops = """
         [p1, i1, i2]
         setfield_gc(p1, i1, descr=valuedescr)
-        escape()
+        escape_n()
         setfield_gc(p1, i2, descr=valuedescr)
         jump(p1, i1, i2)
         """
@@ -2437,7 +2433,7 @@
         guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
-        escape()
+        escape_n()
         jump(p1, i2, i4)
         """
         preamble = """
@@ -2445,7 +2441,7 @@
         guard_true(i3) [p1]
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
-        escape()
+        escape_n()
         i5 = same_as(i4)
         jump(p1, i2, i4, i5)
         """
@@ -2453,7 +2449,7 @@
         [p1, i2, i4, i5]
         guard_true(i4) [p1]
         setfield_gc(p1, NULL, descr=nextdescr)
-        escape()
+        escape_n()
         jump(p1, i2, i5, i5)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -2467,7 +2463,7 @@
         guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
-        escape()
+        escape_n()
         jump(p1, i2, i4)
         """
         preamble = """
@@ -2475,7 +2471,7 @@
         guard_true(i3) [i2, p1]
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
-        escape()
+        escape_n()
         i5 = same_as(i4)
         jump(p1, i2, i4, i5)
         """
@@ -2483,7 +2479,7 @@
         [p1, i2, i4, i5]
         guard_true(i4) [i2, p1]
         setfield_gc(p1, NULL, descr=nextdescr)
-        escape()
+        escape_n()
         jump(p1, i2, i5, i5)
         """
         self.optimize_loop(ops, expected)
@@ -2539,13 +2535,13 @@
         guard_value(p1, ConstPtr(myptr)) []
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(ConstPtr(myptr), i2, descr=valuedescr)
-        escape()
+        escape_n()
         jump(p1, i1, i2)
         """
         expected = """
         [i1, i2]
         setfield_gc(ConstPtr(myptr), i2, descr=valuedescr)
-        escape()
+        escape_n()
         jump(i1, i2)
         """
         self.optimize_loop(ops, expected)
@@ -2555,14 +2551,14 @@
         [p0, i0, p1, i1, i2]
         setfield_gc(p0, i1, descr=valuedescr)
         copystrcontent(p0, p1, i0, i1, i2)
-        escape()
+        escape_n()
         jump(p0, i0, p1, i1, i2)
         """
         expected = """
         [p0, i0, p1, i1, i2]
         copystrcontent(p0, p1, i0, i1, i2)
         setfield_gc(p0, i1, descr=valuedescr)
-        escape()
+        escape_n()
         jump(p0, i0, p1, i1, i2)
         """
         self.optimize_loop(ops, expected)
@@ -2570,24 +2566,24 @@
     def test_duplicate_getarrayitem_1(self):
         ops = """
         [p1]
-        p2 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p3 = getarrayitem_gc(p1, 1, descr=arraydescr2)
-        p4 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p5 = getarrayitem_gc(p1, 1, descr=arraydescr2)
-        escape(p2)
-        escape(p3)
-        escape(p4)
-        escape(p5)
+        p2 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p5 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
+        escape_n(p2)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(p5)
         jump(p1)
         """
         expected = """
         [p1]
-        p2 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p3 = getarrayitem_gc(p1, 1, descr=arraydescr2)
-        escape(p2)
-        escape(p3)
-        escape(p2)
-        escape(p3)
+        p2 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
+        escape_n(p2)
+        escape_n(p3)
+        escape_n(p2)
+        escape_n(p3)
         jump(p1)
         """
         self.optimize_loop(ops, expected)
@@ -2595,10 +2591,10 @@
     def test_duplicate_getarrayitem_2(self):
         ops = """
         [p1, i0]
-        i2 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        i3 = getarrayitem_gc(p1, 1, descr=arraydescr2)
-        i4 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        i5 = getarrayitem_gc(p1, 1, descr=arraydescr2)
+        i2 = getarrayitem_gc_i(p1, 0, descr=arraydescr2)
+        i3 = getarrayitem_gc_i(p1, 1, descr=arraydescr2)
+        i4 = getarrayitem_gc_i(p1, 0, descr=arraydescr2)
+        i5 = getarrayitem_gc_i(p1, 1, descr=arraydescr2)
         i6 = int_add(i3, i4)
         i7 = int_add(i0, i6)
         jump(p1, i7)
@@ -2615,13 +2611,13 @@
         [p1, p2]
         setarrayitem_gc(p1, 0, p2, descr=arraydescr2)
         p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        escape(p3)
+        escape_n(p3)
         jump(p1, p3)
         """
         expected = """
         [p1, p2]
         setarrayitem_gc(p1, 0, p2, descr=arraydescr2)
-        escape(p2)
+        escape_n(p2)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -2634,8 +2630,8 @@
         setarrayitem_gc(p1, i1, p3, descr=arraydescr2)
         p4 = getarrayitem_gc(p1, 0, descr=arraydescr2)
         p5 = getarrayitem_gc(p1, i1, descr=arraydescr2)
-        escape(p4)
-        escape(p5)
+        escape_n(p4)
+        escape_n(p5)
         jump(p1, p2, p3, i1)
         """
         expected = """
@@ -2643,8 +2639,8 @@
         setarrayitem_gc(p1, 0, p2, descr=arraydescr2)
         setarrayitem_gc(p1, i1, p3, descr=arraydescr2)
         p4 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        escape(p4)
-        escape(p3)
+        escape_n(p4)
+        escape_n(p3)
         jump(p1, p2, p3, i1)
         """
         self.optimize_loop(ops, expected)
@@ -2658,9 +2654,9 @@
         p5 = getarrayitem_gc(p1, i1, descr=arraydescr2)
         p6 = getarrayitem_gc(p1, 0, descr=arraydescr2)
         p7 = getarrayitem_gc(p1, 1, descr=arraydescr2)
-        escape(p5)
-        escape(p6)
-        escape(p7)
+        escape_n(p5)
+        escape_n(p6)
+        escape_n(p7)
         jump(p1, p2, p3, p4, i1)
         """
         expected = """
@@ -2669,9 +2665,9 @@
         setarrayitem_gc(p1, 0, p3, descr=arraydescr2)
         setarrayitem_gc(p1, 1, p4, descr=arraydescr2)
         p5 = getarrayitem_gc(p1, i1, descr=arraydescr2)
-        escape(p5)
-        escape(p3)
-        escape(p4)
+        escape_n(p5)
+        escape_n(p3)
+        escape_n(p4)
         jump(p1, p2, p3, p4, i1)
         """
         self.optimize_loop(ops, expected)
@@ -2682,17 +2678,17 @@
         p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
         i4 = getfield_gc_pure(ConstPtr(myptr), descr=valuedescr)
         p5 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        escape(p3)
-        escape(i4)
-        escape(p5)
+        escape_n(p3)
+        escape_n(i4)
+        escape_n(p5)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
         p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        escape(p3)
-        escape(5)
-        escape(p3)
+        escape_n(p3)
+        escape_n(5)
+        escape_n(p3)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -2704,16 +2700,16 @@
         setarrayitem_gc(p2, 1, p4, descr=arraydescr2)
         p5 = getarrayitem_gc(p1, 0, descr=arraydescr2)
         p6 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        escape(p5)
-        escape(p6)
+        escape_n(p5)
+        escape_n(p6)
         jump(p1, p2, p3, p4, i1)
         """
         expected = """
         [p1, p2, p3, p4, i1]
         setarrayitem_gc(p1, 0, p3, descr=arraydescr2)
         setarrayitem_gc(p2, 1, p4, descr=arraydescr2)
-        escape(p3)
-        escape(p4)
+        escape_n(p3)
+        escape_n(p4)
         jump(p1, p2, p3, p4, i1)
         """
         self.optimize_loop(ops, expected)
@@ -2753,19 +2749,19 @@
         [i0, p1]
         p4 = getfield_gc(p1, descr=nextdescr)
         guard_nonnull(p4) []
-        escape(p4)
+        escape_n(p4)
         #
         p2 = new_with_vtable(ConstClass(node_vtable))
-        p3 = escape()
+        p3 = escape_r()
         setfield_gc(p2, p3, descr=nextdescr)
         jump(i0, p2)
         """
         expected = """
         [i0, p4]
         guard_nonnull(p4) []
-        escape(p4)
+        escape_n(p4)
         #
-        p3 = escape()
+        p3 = escape_r()
         jump(i0, p3)
         """
         self.optimize_loop(ops, expected)
@@ -2775,19 +2771,19 @@
         [i0, p1]
         p4 = getarrayitem_gc(p1, 0, descr=arraydescr2)
         guard_nonnull(p4) []
-        escape(p4)
+        escape_n(p4)
         #
         p2 = new_array(1, descr=arraydescr2)
-        p3 = escape()
+        p3 = escape_r()
         setarrayitem_gc(p2, 0, p3, descr=arraydescr2)
         jump(i0, p2)
         """
         expected = """
         [i0, p4]
         guard_nonnull(p4) []
-        escape(p4)
+        escape_n(p4)
         #
-        p3 = escape()
+        p3 = escape_r()
         jump(i0, p3)
         """
         self.optimize_loop(ops, expected)
@@ -2807,7 +2803,7 @@
         p1a = new_with_vtable(ConstClass(node_vtable2))
         p2a = new_with_vtable(ConstClass(node_vtable))
         p3a = new_with_vtable(ConstClass(node_vtable))
-        escape(p3a)
+        escape_n(p3a)
         setfield_gc(p1a, p2a, descr=nextdescr)
         setfield_gc(p1a, p3a, descr=otherdescr)
         jump(p1a)
@@ -2821,7 +2817,7 @@
         guard_class(p3, ConstClass(node_vtable)) []
         p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        escape(p3a)
+        escape_n(p3a)
         jump(p3a)
         """
         expected = """
@@ -2835,7 +2831,7 @@
         p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        escape(p3anew)
+        escape_n(p3anew)
         jump(p3anew)
         """
         #self.optimize_loop(ops, expected) # XXX Virtual(node_vtable2, nextdescr=Not, otherdescr=Not)
@@ -2856,7 +2852,7 @@
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
         p3a = new_with_vtable(ConstClass(node_vtable))
-        escape(p3a)
+        escape_n(p3a)
         setfield_gc(p1a, p2a, descr=nextdescr)
         setfield_gc(p1a, p3a, descr=otherdescr)
         jump(p1a)
@@ -2872,7 +2868,7 @@
         p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        escape(p3a)
+        escape_n(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
         jump(p2a, p3a)
@@ -2882,7 +2878,7 @@
         p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        escape(p3a)
+        escape_n(p3a)
         jump(p2a, p3a)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -2910,7 +2906,7 @@
     def test_bug_5(self):
         ops = """
         [p0]
-        i0 = escape()
+        i0 = escape_i()
         i2 = getfield_gc(p0, descr=valuedescr)
         i4 = int_add(i2, 1)
         setfield_gc(p0, i4, descr=valuedescr)
@@ -2918,18 +2914,18 @@
         i6 = getfield_gc(p0, descr=valuedescr)
         i8 = int_sub(i6, 1)
         setfield_gc(p0, i8, descr=valuedescr)
-        escape()
+        escape_n()
         jump(p0)
         """
         expected = """
         [p0]
-        i0 = escape()
+        i0 = escape_i()
         i2 = getfield_gc(p0, descr=valuedescr)
         i4 = int_add(i2, 1)
         setfield_gc(p0, i4, descr=valuedescr)
         guard_true(i0) []
         setfield_gc(p0, i2, descr=valuedescr)
-        escape()
+        escape_n()
         jump(p0)
         """
         self.optimize_loop(ops, expected)
@@ -2950,7 +2946,7 @@
         guard_class(p1, ConstClass(node_vtable2)) []
         #
         p2 = new_with_vtable(ConstClass(node_vtable))
-        escape(p2)      # prevent it from staying Virtual
+        escape_n(p2)      # prevent it from staying Virtual
         jump(p2)
         """
         self.raises(InvalidLoop, self.optimize_loop, ops, "crash!")
@@ -3126,8 +3122,8 @@
         i4 = int_add(i2, 1)
         i4b = int_is_true(i4)
         guard_true(i4b) []
-        escape(i3)
-        escape(i4)
+        escape_n(i3)
+        escape_n(i4)
         guard_true(i1) []
         guard_true(i2) []
         jump(p1, p2)
@@ -3138,15 +3134,15 @@
         i3 = int_add(i1, 1)
         i3b = int_is_true(i3)
         guard_true(i3b) []
-        escape(i3)
-        escape(i3)
+        escape_n(i3)
+        escape_n(i3)
         guard_true(i1) []
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
-        escape(2)
-        escape(2)
+        escape_n(2)
+        escape_n(2)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -3186,8 +3182,8 @@
         guard_no_overflow() []
         i4b = int_is_true(i4)
         guard_true(i4b) []
-        escape(i3)
-        escape(i4)
+        escape_n(i3)
+        escape_n(i4)
         jump(i1)
         """
         preamble = """
@@ -3196,14 +3192,14 @@
         guard_no_overflow() []
         i3b = int_is_true(i3)
         guard_true(i3b) []
-        escape(i3)
-        escape(i3)
+        escape_n(i3)
+        escape_n(i3)
         jump(i1, i3)
         """
         expected = """
         [i1, i3]
-        escape(i3)
-        escape(i3)
+        escape_n(i3)
+        escape_n(i3)
         jump(i1, i3)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -3220,8 +3216,8 @@
         guard_no_overflow() []
         i4b = int_is_true(i4)
         guard_true(i4b) []
-        escape(i3)
-        escape(i4)
+        escape_n(i3)
+        escape_n(i4)
         jump(i1, p1)
         """
         preamble = """
@@ -3231,15 +3227,15 @@
         i3b = int_is_true(i3)
         guard_true(i3b) []
         setfield_gc(p1, i1, descr=valuedescr)
-        escape(i3)
-        escape(i3)
+        escape_n(i3)
+        escape_n(i3)
         jump(i1, p1, i3)
         """
         expected = """
         [i1, p1, i3]
         setfield_gc(p1, i1, descr=valuedescr)
-        escape(i3)
-        escape(i3)
+        escape_n(i3)
+        escape_n(i3)
         jump(i1, p1, i3)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -3392,12 +3388,12 @@
         [i0]
         i1 = int_sub_ovf(i0, i0)
         guard_no_overflow() []
-        escape(i1)
+        escape_n(i1)
         jump(i1)
         """
         expected = """
         []
-        escape(0)
+        escape_n(0)
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -3451,16 +3447,16 @@
         i1 = getfield_gc(p1, descr=valuedescr)
         i2 = call(i1, descr=nonwritedescr)
         i3 = getfield_gc(p1, descr=valuedescr)
-        escape(i1)
-        escape(i3)
+        escape_n(i1)
+        escape_n(i3)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
         i1 = getfield_gc(p1, descr=valuedescr)
         i2 = call(i1, descr=nonwritedescr)
-        escape(i1)
-        escape(i1)
+        escape_n(i1)
+        escape_n(i1)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -3473,10 +3469,10 @@
         i3 = call(i1, descr=writeadescr)
         i4 = getfield_gc(p1, descr=adescr)
         i5 = getfield_gc(p1, descr=bdescr)
-        escape(i1)
-        escape(i2)
-        escape(i4)
-        escape(i5)
+        escape_n(i1)
+        escape_n(i2)
+        escape_n(i4)
+        escape_n(i5)
         jump(p1, p2)
         """
         expected = """
@@ -3485,10 +3481,10 @@
         i2 = getfield_gc(p1, descr=bdescr)
         i3 = call(i1, descr=writeadescr)
         i4 = getfield_gc(p1, descr=adescr)
-        escape(i1)
-        escape(i2)
-        escape(i4)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
+        escape_n(i4)
+        escape_n(i2)
         jump(p1, p2)
         """
         self.optimize_loop(ops, expected)
@@ -3501,10 +3497,10 @@
         i3 = call(i1, descr=writeadescr)
         p5 = getarrayitem_gc(p1, 0, descr=arraydescr2)
         p6 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        escape(p3)
-        escape(p4)
-        escape(p5)
-        escape(p6)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(p5)
+        escape_n(p6)
         jump(p1, p2, i1)
         """
         expected = """
@@ -3512,10 +3508,10 @@
         p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
         p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
         i3 = call(i1, descr=writeadescr)
-        escape(p3)
-        escape(p4)
-        escape(p3)
-        escape(p4)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(p3)
+        escape_n(p4)
         jump(p1, p2, i1)
         """
         self.optimize_loop(ops, expected)
@@ -3525,32 +3521,32 @@
         [p1, p2, i1]
         p3 = getarrayitem_gc(p2, 0, descr=arraydescr2)
         p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         i3 = call(i1, descr=writearraydescr)
         p5 = getarrayitem_gc(p2, 0, descr=arraydescr2)
         p6 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        i4 = getarrayitem_gc(p1, 1, descr=arraydescr)
-        escape(p3)
-        escape(p4)
-        escape(p5)
-        escape(p6)
-        escape(i2)
-        escape(i4)
+        i4 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(p5)
+        escape_n(p6)
+        escape_n(i2)
+        escape_n(i4)
         jump(p1, p2, i1)
         """
         expected = """
         [p1, p2, i1]
         p3 = getarrayitem_gc(p2, 0, descr=arraydescr2)
         p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
+        i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         i3 = call(i1, descr=writearraydescr)
-        i4 = getarrayitem_gc(p1, 1, descr=arraydescr)
-        escape(p3)
-        escape(p4)
-        escape(p3)
-        escape(p4)
-        escape(i2)
-        escape(i4)
+        i4 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(p3)
+        escape_n(p4)
+        escape_n(i2)
+        escape_n(i4)
         jump(p1, p2, i1)
         """
         self.optimize_loop(ops, expected)
@@ -3689,24 +3685,24 @@
         call_pure_results = {tuple(arg_consts): ConstInt(42)}
         ops = '''
         [i0, i1, i2]
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
         i3 = call_pure(123456, 4, 5, 6, descr=plaincalldescr)
         i4 = call_pure(123456, 4, i0, 6, descr=plaincalldescr)
         jump(i0, i3, i4)
         '''
         preamble = '''
         [i0, i1, i2]
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
         i153 = same_as(i4)
         jump(i0, i4, i153)
         '''
         expected = '''
         [i0, i4, i5]
-        escape(42)
-        escape(i4)
+        escape_n(42)
+        escape_n(i4)
         jump(i0, i5, i5)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
@@ -3717,8 +3713,8 @@
         call_pure_results = {tuple(arg_consts): ConstInt(42)}
         ops = '''
         [i0, i1, i2]
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
         i3 = call_pure(123456, 4, 5, 6, descr=plaincalldescr)
         guard_no_exception() []
         i4 = call_pure(123456, 4, i0, 6, descr=plaincalldescr)
@@ -3727,8 +3723,8 @@
         '''
         preamble = '''
         [i0, i1, i2]
-        escape(i1)
-        escape(i2)
+        escape_n(i1)
+        escape_n(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
         guard_no_exception() []
         i155 = same_as(i4)
@@ -3736,8 +3732,8 @@
         '''
         expected = '''
         [i0, i2, i3]
-        escape(42)
-        escape(i2)
+        escape_n(42)
+        escape_n(i2)
         jump(i0, i3, i3)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
@@ -3748,7 +3744,7 @@
         ops = '''
         [p1, i1, i2]
         p2 = call_pure(0, p1, i1, i2, descr=strslicedescr)
-        escape(p2)
+        escape_n(p2)
         jump(p1, i1, i2)
         '''
         preamble = '''
@@ -3756,14 +3752,14 @@
         i6 = int_sub(i2, i1)
         p2 = newstr(i6)
         copystrcontent(p1, p2, i1, 0, i6)
-        escape(p2)
+        escape_n(p2)
         jump(p1, i1, i2, i6)
         '''
         expected = '''
         [p1, i1, i2, i6]
         p2 = newstr(i6)
         copystrcontent(p1, p2, i1, 0, i6)
-        escape(p2)
+        escape_n(p2)
         jump(p1, i1, i2, i6)
         '''
         self.optimize_loop(ops, expected, preamble)
@@ -3789,7 +3785,7 @@
         ops = """
         [p1]
         p2 = virtual_ref(p1, 5)
-        escape(p2)
+        escape_n(p2)
         virtual_ref_finish(p2, p1)
         jump(p1)
         """
@@ -3799,7 +3795,7 @@
         p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable))
         setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p2, p0, descr=virtualtokendescr)
-        escape(p2)
+        escape_n(p2)
         setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         jump(p1)
@@ -3909,7 +3905,7 @@
         guard_no_exception(descr=fdescr) [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=refdescr)
-        escape()
+        escape_n()
         jump(p0, i1)
         """
         preamble = """
@@ -3918,7 +3914,7 @@
         call(i1, descr=nonwritedescr)
         guard_no_exception(descr=fdescr) [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
-        escape()
+        escape_n()
         jump(p0, i1)
         """
         expected = """
@@ -3927,7 +3923,7 @@
         call(i1, descr=nonwritedescr)
         guard_no_exception(descr=fdescr2) [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
-        escape()
+        escape_n()
         jump(p0, i1)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -3948,7 +3944,7 @@
         [i1]
         p1 = new_with_vtable(ConstClass(node_vtable))
         p2 = virtual_ref(p1, 7)
-        escape(p2)
+        escape_n(p2)
         virtual_ref_finish(p2, p1)
         call_may_force(i1, descr=mayforcevirtdescr)

From noreply at buildbot.pypy.org  Mon Dec 15 16:36:14 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Mon, 15 Dec 2014 16:36:14 +0100 (CET)
Subject: [pypy-commit] pypy default: (arigo,
 ltratt) Avoid allocating memory when ordereddicts have spare
 capacity.
Message-ID: <20141215153614.52C891D28C3@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r74939:7ff0e531c521
Date: 2014-12-15 15:35 +0000
http://bitbucket.org/pypy/pypy/changeset/7ff0e531c521/

Log:	(arigo, ltratt) Avoid allocating memory when ordereddicts have spare
	capacity.

	Previously, dicts could yo-yo in size if someone continually added
	and removed elements, but the total number of live elements remained
	more-or-less constant. This patch changes things so that compaction
	of entries happens more often than shrinking of memory. Put another
	way, we only shrink the size allocated to an ordereddict when it is
	extremely sparsely populated. This reduces GC pressure quite a bit.

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -575,7 +575,11 @@
 
 @jit.dont_look_inside
 def ll_dict_grow(d):
-    if d.num_items < d.num_used_items // 4:
+    if d.num_items < d.num_used_items // 2:
+        # At least 50% of the allocated entries are dead, so perform a
+        # compaction. If ll_dict_remove_deleted_items detects that over
+        # 75% of allocated entries are dead, then it will also shrink the
+        # memory allocated at the same time as doing a compaction.
         ll_dict_remove_deleted_items(d)
         return True
 
@@ -594,8 +598,10 @@
     return False
 
 def ll_dict_remove_deleted_items(d):
-    new_allocated = _overallocate_entries_len(d.num_items)
-    if new_allocated < len(d.entries) // 2:
+    if d.num_items < len(d.entries) // 4:
+        # At least 75% of the allocated entries are dead, so shrink the memory
+        # allocated as well as doing a compaction.
+        new_allocated = _overallocate_entries_len(d.num_items)
         newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
     else:
         newitems = d.entries

From noreply at buildbot.pypy.org  Mon Dec 15 16:37:24 2014
From: noreply at buildbot.pypy.org (vext01)
Date: Mon, 15 Dec 2014 16:37:24 +0100 (CET)
Subject: [pypy-commit] benchmarks min_5_secs: Backed out changeset
	b7f0946c13a6
Message-ID: <20141215153724.CA41C1D288A@cobra.cs.uni-duesseldorf.de>

Author: Edd Barrett <vext01 at gmail.com>
Branch: min_5_secs
Changeset: r301:8575f60bfd42
Date: 2014-12-15 15:32 +0000
http://bitbucket.org/pypy/benchmarks/changeset/8575f60bfd42/

Log:	Backed out changeset b7f0946c13a6

diff --git a/unladen_swallow/performance/richards.py b/unladen_swallow/performance/richards.py
--- a/unladen_swallow/performance/richards.py
+++ b/unladen_swallow/performance/richards.py
@@ -363,10 +363,8 @@
 
     def run(self, iterations):
         for i in xrange(iterations):
-            global taskWorkArea
-            # Must reinitiliase the taskWorkArea or benchmark will run slower
-            # upon subsequent runs.
-            taskWorkArea = TaskWorkArea()
+            taskWorkArea.holdCount = 0
+            taskWorkArea.qpktCount = 0
 
             IdleTask(I_IDLE, 1, 10000, TaskState().running(), IdleTaskRec())
 

From noreply at buildbot.pypy.org  Mon Dec 15 16:37:26 2014
From: noreply at buildbot.pypy.org (vext01)
Date: Mon, 15 Dec 2014 16:37:26 +0100 (CET)
Subject: [pypy-commit] benchmarks min_5_secs: Reset the TaskWorkArea without
 allocating a new instance.
Message-ID: <20141215153726.6F7ED1D288A@cobra.cs.uni-duesseldorf.de>

Author: Edd Barrett <vext01 at gmail.com>
Branch: min_5_secs
Changeset: r302:e8a9a5a3a102
Date: 2014-12-15 15:38 +0000
http://bitbucket.org/pypy/benchmarks/changeset/e8a9a5a3a102/

Log:	Reset the TaskWorkArea without allocating a new instance.

	From Stefan Marr. Thanks.

diff --git a/unladen_swallow/performance/richards.py b/unladen_swallow/performance/richards.py
--- a/unladen_swallow/performance/richards.py
+++ b/unladen_swallow/performance/richards.py
@@ -144,6 +144,9 @@
 
 class TaskWorkArea(object):
     def __init__(self):
+        self.reset()
+
+    def reset(self):
         self.taskTab = [None] * TASKTABSIZE
 
         self.taskList = None
@@ -363,8 +366,7 @@
 
     def run(self, iterations):
         for i in xrange(iterations):
-            taskWorkArea.holdCount = 0
-            taskWorkArea.qpktCount = 0
+            taskWorkArea.reset()
 
             IdleTask(I_IDLE, 1, 10000, TaskState().running(), IdleTaskRec())
 

From noreply at buildbot.pypy.org  Mon Dec 15 16:46:37 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Dec 2014 16:46:37 +0100 (CET)
Subject: [pypy-commit] pypy default: Add a performance hack:
	llop.gc_writebarrier
Message-ID: <20141215154637.87B631C050D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74940:5fc98f5c2d94
Date: 2014-12-15 15:46 +0000
http://bitbucket.org/pypy/pypy/changeset/5fc98f5c2d94/

Log:	Add a performance hack: llop.gc_writebarrier

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -605,6 +605,10 @@
         newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
     else:
         newitems = d.entries
+        # The loop below does a lot of writes into 'newitems'.  It's a better
+        # idea to do a single gc_writebarrier rather than activating the
+        # card-by-card logic (worth 11% in microbenchmarks).
+        llop.gc_writebarrier(lltype.Void, newitems)
     #
     ENTRIES = lltype.typeOf(d).TO.entries.TO
     ENTRY = ENTRIES.OF

From noreply at buildbot.pypy.org  Mon Dec 15 17:27:17 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Dec 2014 17:27:17 +0100 (CET)
Subject: [pypy-commit] pypy default: oups
Message-ID: <20141215162717.B0B4E1C1347@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74941:ad419d66acec
Date: 2014-12-15 16:26 +0000
http://bitbucket.org/pypy/pypy/changeset/ad419d66acec/

Log:	oups

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -608,6 +608,7 @@
         # The loop below does a lot of writes into 'newitems'.  It's a better
         # idea to do a single gc_writebarrier rather than activating the
         # card-by-card logic (worth 11% in microbenchmarks).
+        from rpython.rtyper.lltypesystem.lloperation import llop
         llop.gc_writebarrier(lltype.Void, newitems)
     #
     ENTRIES = lltype.typeOf(d).TO.entries.TO

From noreply at buildbot.pypy.org  Mon Dec 15 17:37:16 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Dec 2014 17:37:16 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix this test
Message-ID: <20141215163716.5C4541C01C3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74942:db87f8d3abb8
Date: 2014-12-15 16:37 +0000
http://bitbucket.org/pypy/pypy/changeset/db87f8d3abb8/

Log:	Fix this test

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -72,6 +72,8 @@
         'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr)
                              and DICTVALUE._needsgc()),
         }
+    if getattr(ll_eq_function, 'no_direct_compare', False):
+        entrymeths['no_direct_compare'] = True
 
     # * the key
     entryfields.append(("key", DICTKEY))
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -292,9 +292,6 @@
         res = self.interpret(func, [5])
         assert res == 6
 
-    def test_dict_with_SHORT_keys(self):
-        py.test.skip("I don't want to edit this file on two branches")
-
     def test_memoryerror_should_not_insert(self):
         py.test.skip("I don't want to edit this file on two branches")
 

From noreply at buildbot.pypy.org  Mon Dec 15 18:19:45 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 15 Dec 2014 18:19:45 +0100 (CET)
Subject: [pypy-commit] pypy default: Do a ll_clear_indexes() instead of
 allocating a new array for d.indexes
Message-ID: <20141215171945.AFA821C050D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74943:e6ad4389d70b
Date: 2014-12-15 17:19 +0000
http://bitbucket.org/pypy/pypy/changeset/e6ad4389d70b/

Log:	Do a ll_clear_indexes() instead of allocating a new array for
	d.indexes when we can

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -330,6 +330,20 @@
     keepalive_until_here(newp)
     return newp
 
+ at jit.dont_look_inside
+ at specialize.ll()
+def ll_arrayclear(p):
+    # Equivalent to memset(array, 0).  Only for GcArray(primitive-type) for now.
+    from rpython.rlib.objectmodel import keepalive_until_here
+
+    length = len(p)
+    ARRAY = lltype.typeOf(p).TO
+    offset = llmemory.itemoffsetof(ARRAY, 0)
+    dest_addr = llmemory.cast_ptr_to_adr(p) + offset
+    llmemory.raw_memclear(dest_addr, llmemory.sizeof(ARRAY.OF) * length)
+    keepalive_until_here(p)
+
+
 def no_release_gil(func):
     func._dont_inline_ = True
     func._no_release_gil_ = True
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -158,6 +158,16 @@
     assert a2[2].x == 3
     assert a2[2].y == 15
 
+def test_ll_arrayclear():
+    TYPE = lltype.GcArray(lltype.Signed)
+    a1 = lltype.malloc(TYPE, 10)
+    for i in range(10):
+        a1[i] = 100 + i
+    rgc.ll_arrayclear(a1)
+    assert len(a1) == 10
+    for i in range(10):
+        assert a1[i] == 0
+
 def test__contains_gcptr():
     assert not rgc._contains_gcptr(lltype.Signed)
     assert not rgc._contains_gcptr(
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -418,6 +418,7 @@
 TYPE_LONG  = lltype.Unsigned
 
 def ll_malloc_indexes_and_choose_lookup(d, n):
+    # keep in sync with ll_clear_indexes() below
     if n <= 256:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_BYTE.TO, n,
@@ -439,6 +440,16 @@
                                                          zero=True))
         d.lookup_function_no = FUNC_LONG
 
+def ll_clear_indexes(d, n):
+    if n <= 256:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_BYTE, d.indexes))
+    elif n <= 65536:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_SHORT, d.indexes))
+    elif IS_64BIT and n <= 2 ** 32:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_INT, d.indexes))
+    else:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_LONG, d.indexes))
+
 def ll_call_insert_clean_function(d, hash, i):
     DICT = lltype.typeOf(d).TO
     if d.lookup_function_no == FUNC_BYTE:
@@ -704,13 +715,17 @@
         ll_dict_reindex(d, new_size)
 
 def ll_dict_reindex(d, new_size):
-    ll_malloc_indexes_and_choose_lookup(d, new_size)
+    if bool(d.indexes) and _ll_len_of_d_indexes(d) == new_size:
+        ll_clear_indexes(d, new_size)   # hack: we can reuse the same array
+    else:
+        ll_malloc_indexes_and_choose_lookup(d, new_size)
     d.resize_counter = new_size * 2 - d.num_items * 3
     assert d.resize_counter > 0
     #
     entries = d.entries
     i = 0
-    while i < d.num_used_items:
+    ibound = d.num_used_items
+    while i < ibound:
         if entries.valid(i):
             hash = entries.hash(i)
             ll_call_insert_clean_function(d, hash, i)

From noreply at buildbot.pypy.org  Mon Dec 15 19:06:10 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Mon, 15 Dec 2014 19:06:10 +0100 (CET)
Subject: [pypy-commit] pypy default: Rename ordereddict attributes to be
	less confusing.
Message-ID: <20141215180610.E6F6C1C1347@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r74944:7e75a782be2c
Date: 2014-12-15 16:33 +0000
http://bitbucket.org/pypy/pypy/changeset/7e75a782be2c/

Log:	Rename ordereddict attributes to be less confusing.

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -28,8 +28,8 @@
 #    }
 #
 #    struct dicttable {
-#        int num_items;
-#        int num_used_items;
+#        int num_live_items;
+#        int num_ever_used_items;
 #        int resize_counter;
 #        {byte, short, int, long} *indexes;
 #        dictentry *entries;
@@ -113,8 +113,8 @@
     DICTENTRY = lltype.Struct("odictentry", *entryfields)
     DICTENTRYARRAY = lltype.GcArray(DICTENTRY,
                                     adtmeths=entrymeths)
-    fields =          [ ("num_items", lltype.Signed),
-                        ("num_used_items", lltype.Signed),
+    fields =          [ ("num_live_items", lltype.Signed),
+                        ("num_ever_used_items", lltype.Signed),
                         ("resize_counter", lltype.Signed),
                         ("indexes", llmemory.GCREF),
                         ("lookup_function_no", lltype.Signed),
@@ -492,11 +492,11 @@
     return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
-    return d.num_items
+    return d.num_live_items
 
 def ll_dict_bool(d):
     # check if a dict is True, allowing for None
-    return bool(d) and d.num_items != 0
+    return bool(d) and d.num_live_items != 0
 
 def ll_dict_getitem(d, key):
     index = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
@@ -519,18 +519,18 @@
         entry = d.entries[i]
         entry.value = value
     else:
-        if len(d.entries) == d.num_used_items:
+        if len(d.entries) == d.num_ever_used_items:
             if ll_dict_grow(d):
-                ll_call_insert_clean_function(d, hash, d.num_used_items)
-        entry = d.entries[d.num_used_items]
+                ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
+        entry = d.entries[d.num_ever_used_items]
         entry.key = key
         entry.value = value
         if hasattr(ENTRY, 'f_hash'):
             entry.f_hash = hash
         if hasattr(ENTRY, 'f_valid'):
             entry.f_valid = True
-        d.num_used_items += 1
-        d.num_items += 1
+        d.num_ever_used_items += 1
+        d.num_live_items += 1
         rc = d.resize_counter - 3
         if rc <= 0:
             ll_dict_resize(d)
@@ -540,16 +540,16 @@
 
 def _ll_dict_insertclean(d, key, value, hash):
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    ll_call_insert_clean_function(d, hash, d.num_used_items)
-    entry = d.entries[d.num_used_items]
+    ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
+    entry = d.entries[d.num_ever_used_items]
     entry.key = key
     entry.value = value
     if hasattr(ENTRY, 'f_hash'):
         entry.f_hash = hash
     if hasattr(ENTRY, 'f_valid'):
         entry.f_valid = True
-    d.num_used_items += 1
-    d.num_items += 1
+    d.num_ever_used_items += 1
+    d.num_live_items += 1
     rc = d.resize_counter - 3
     d.resize_counter = rc
 
@@ -575,7 +575,7 @@
 
 @jit.dont_look_inside
 def ll_dict_grow(d):
-    if d.num_items < d.num_used_items // 2:
+    if d.num_live_items < d.num_ever_used_items // 2:
         # At least 50% of the allocated entries are dead, so perform a
         # compaction. If ll_dict_remove_deleted_items detects that over
         # 75% of allocated entries are dead, then it will also shrink the
@@ -598,10 +598,10 @@
     return False
 
 def ll_dict_remove_deleted_items(d):
-    if d.num_items < len(d.entries) // 4:
+    if d.num_live_items < len(d.entries) // 4:
         # At least 75% of the allocated entries are dead, so shrink the memory
         # allocated as well as doing a compaction.
-        new_allocated = _overallocate_entries_len(d.num_items)
+        new_allocated = _overallocate_entries_len(d.num_live_items)
         newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
     else:
         newitems = d.entries
@@ -610,7 +610,7 @@
     ENTRY = ENTRIES.OF
     isrc = 0
     idst = 0
-    isrclimit = d.num_used_items
+    isrclimit = d.num_ever_used_items
     while isrc < isrclimit:
         if d.entries.valid(isrc):
             src = d.entries[isrc]
@@ -624,8 +624,8 @@
                 dst.f_valid = True
             idst += 1
         isrc += 1
-    assert d.num_items == idst
-    d.num_used_items = idst
+    assert d.num_live_items == idst
+    d.num_ever_used_items = idst
     if ((ENTRIES.must_clear_key or ENTRIES.must_clear_value) and
             d.entries == newitems):
         # must clear the extra entries: they may contain valid pointers
@@ -652,7 +652,7 @@
 @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
 def _ll_dict_del(d, index):
     d.entries.mark_deleted(index)
-    d.num_items -= 1
+    d.num_live_items -= 1
     # clear the key and the value if they are GC pointers
     ENTRIES = lltype.typeOf(d.entries).TO
     ENTRY = ENTRIES.OF
@@ -679,14 +679,14 @@
     # make a 'new_size' estimate and shrink it if there are many
     # deleted entry markers.  See CPython for why it is a good idea to
     # quadruple the dictionary size as long as it's not too big.
-    # (Quadrupling comes from '(d.num_items + d.num_items + 1) * 2'
-    # as long as num_items is not too large.)
-    num_extra = min(d.num_items + 1, 30000)
+    # (Quadrupling comes from '(d.num_live_items + d.num_live_items + 1) * 2'
+    # as long as num_live_items is not too large.)
+    num_extra = min(d.num_live_items + 1, 30000)
     _ll_dict_resize_to(d, num_extra)
 ll_dict_resize.oopspec = 'odict.resize(d)'
 
 def _ll_dict_resize_to(d, num_extra):
-    new_estimate = (d.num_items + num_extra) * 2
+    new_estimate = (d.num_live_items + num_extra) * 2
     new_size = DICT_INITSIZE
     while new_size <= new_estimate:
         new_size *= 2
@@ -698,12 +698,12 @@
 
 def ll_dict_reindex(d, new_size):
     ll_malloc_indexes_and_choose_lookup(d, new_size)
-    d.resize_counter = new_size * 2 - d.num_items * 3
+    d.resize_counter = new_size * 2 - d.num_live_items * 3
     assert d.resize_counter > 0
     #
     entries = d.entries
     i = 0
-    while i < d.num_used_items:
+    while i < d.num_ever_used_items:
         if entries.valid(i):
             hash = entries.hash(i)
             ll_call_insert_clean_function(d, hash, i)
@@ -778,7 +778,7 @@
     else:
         # pristine entry -- lookup failed
         if store_flag == FLAG_STORE:
-            indexes[i] = rffi.cast(T, d.num_used_items + VALID_OFFSET)
+            indexes[i] = rffi.cast(T, d.num_ever_used_items + VALID_OFFSET)
         elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
             return ll_kill_something(d, T)
         return -1
@@ -795,7 +795,7 @@
             if store_flag == FLAG_STORE:
                 if deletedslot == -1:
                     deletedslot = intmask(i)
-                indexes[deletedslot] = rffi.cast(T, d.num_used_items +
+                indexes[deletedslot] = rffi.cast(T, d.num_ever_used_items +
                                                  VALID_OFFSET)
             elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
                 return ll_kill_something(d, T)
@@ -855,8 +855,8 @@
     d = DICT.allocate()
     d.entries = _ll_empty_array(DICT)
     ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
-    d.num_items = 0
-    d.num_used_items = 0
+    d.num_live_items = 0
+    d.num_ever_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
 OrderedDictRepr.ll_newdict = staticmethod(ll_newdict)
@@ -869,8 +869,8 @@
     d = DICT.allocate()
     d.entries = DICT.entries.TO.allocate(orig_length_estimate)
     ll_malloc_indexes_and_choose_lookup(d, n)
-    d.num_items = 0
-    d.num_used_items = 0
+    d.num_live_items = 0
+    d.num_ever_used_items = 0
     d.resize_counter = n * 2
     return d
 
@@ -920,7 +920,7 @@
         entries = dict.entries
         index = iter.index
         assert index >= 0
-        entries_len = dict.num_used_items
+        entries_len = dict.num_ever_used_items
         while index < entries_len:
             nextindex = index + 1
             if entries.valid(index):
@@ -955,15 +955,15 @@
     newdict = DICT.allocate()
     newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
 
-    newdict.num_items = dict.num_items
-    newdict.num_used_items = dict.num_used_items
+    newdict.num_live_items = dict.num_live_items
+    newdict.num_ever_used_items = dict.num_ever_used_items
     if hasattr(DICT, 'fnkeyeq'):
         newdict.fnkeyeq = dict.fnkeyeq
     if hasattr(DICT, 'fnkeyhash'):
         newdict.fnkeyhash = dict.fnkeyhash
 
     i = 0
-    while i < newdict.num_used_items:
+    while i < newdict.num_ever_used_items:
         d_entry = newdict.entries[i]
         entry = dict.entries[i]
         ENTRY = lltype.typeOf(newdict.entries).TO.OF
@@ -980,14 +980,14 @@
 ll_dict_copy.oopspec = 'odict.copy(dict)'
 
 def ll_dict_clear(d):
-    if d.num_used_items == 0:
+    if d.num_ever_used_items == 0:
         return
     DICT = lltype.typeOf(d).TO
     old_entries = d.entries
     d.entries = _ll_empty_array(DICT)
     ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
-    d.num_items = 0
-    d.num_used_items = 0
+    d.num_live_items = 0
+    d.num_ever_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     # old_entries.delete() XXX
 ll_dict_clear.oopspec = 'odict.clear(d)'
@@ -995,9 +995,9 @@
 def ll_dict_update(dic1, dic2):
     if dic1 == dic2:
         return
-    ll_prepare_dict_update(dic1, dic2.num_items)
+    ll_prepare_dict_update(dic1, dic2.num_live_items)
     i = 0
-    while i < dic2.num_used_items:
+    while i < dic2.num_ever_used_items:
         entries = dic2.entries
         if entries.valid(i):
             entry = entries[i]
@@ -1016,12 +1016,12 @@
     #      (d.resize_counter - 1) // 3 = room left in d
     #  so, if num_extra == 1, we need d.resize_counter > 3
     #      if num_extra == 2, we need d.resize_counter > 6  etc.
-    # Note however a further hack: if num_extra <= d.num_items,
+    # Note however a further hack: if num_extra <= d.num_live_items,
     # we avoid calling _ll_dict_resize_to here.  This is to handle
     # the case where dict.update() actually has a lot of collisions.
-    # If num_extra is much greater than d.num_items the conditional_call
+    # If num_extra is much greater than d.num_live_items the conditional_call
     # will trigger anyway, which is really the goal.
-    x = num_extra - d.num_items
+    x = num_extra - d.num_live_items
     jit.conditional_call(d.resize_counter <= x * 3,
                          _ll_dict_resize_to, d, num_extra)
 
@@ -1038,9 +1038,9 @@
 
 def _make_ll_keys_values_items(kind):
     def ll_kvi(LIST, dic):
-        res = LIST.ll_newlist(dic.num_items)
+        res = LIST.ll_newlist(dic.num_live_items)
         entries = dic.entries
-        dlen = dic.num_used_items
+        dlen = dic.num_ever_used_items
         items = res.ll_items()
         i = 0
         p = 0
@@ -1074,24 +1074,24 @@
     return i != -1
 
 def _ll_getnextitem(dic):
-    if dic.num_items == 0:
+    if dic.num_live_items == 0:
         raise KeyError
 
     entries = dic.entries
 
     while True:
-        i = dic.num_used_items - 1
+        i = dic.num_ever_used_items - 1
         if entries.valid(i):
             break
-        dic.num_used_items -= 1
+        dic.num_ever_used_items -= 1
 
     key = entries[i].key
     index = dic.lookup_function(dic, key, entries.hash(i),
                                 FLAG_DELETE_TRY_HARD)
     # if the lookup function returned me a random strange thing,
     # don't care about deleting the item
-    if index == dic.num_used_items - 1:
-        dic.num_used_items -= 1
+    if index == dic.num_ever_used_items - 1:
+        dic.num_ever_used_items -= 1
     else:
         assert index != -1
     return index
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -71,7 +71,7 @@
             for j in range(i):
                 assert rordereddict.ll_dict_getitem(ll_d, llstr(str(j))) == j
             rordereddict.ll_dict_setitem(ll_d, llstr(str(i)), i)
-        assert ll_d.num_items == 20
+        assert ll_d.num_live_items == 20
         for i in range(20):
             assert rordereddict.ll_dict_getitem(ll_d, llstr(str(i))) == i
 
@@ -84,7 +84,7 @@
             rordereddict.ll_dict_setitem(ll_d, llstr(str(i)), i)
             if i % 2 != 0:
                 rordereddict.ll_dict_delitem(ll_d, llstr(str(i)))
-        assert ll_d.num_items == 10
+        assert ll_d.num_live_items == 10
         for i in range(0, 20, 2):
             assert rordereddict.ll_dict_getitem(ll_d, llstr(str(i))) == i
 
@@ -129,7 +129,7 @@
         for i in range(40):
             rordereddict.ll_dict_setitem(ll_d, lls, i)
             rordereddict.ll_dict_delitem(ll_d, lls)
-        assert ll_d.num_used_items <= 10
+        assert ll_d.num_ever_used_items <= 10
 
     def test_dict_iteration(self):
         DICT = self._get_str_dict()
@@ -190,7 +190,7 @@
         rordereddict.ll_dict_setitem(ll_d, llstr("j"), 1)
         rordereddict.ll_dict_setitem(ll_d, llstr("l"), 1)
         rordereddict.ll_dict_clear(ll_d)
-        assert ll_d.num_items == 0
+        assert ll_d.num_live_items == 0
 
     def test_get(self):
         DICT = self._get_str_dict()
@@ -349,7 +349,7 @@
             if 1.38 <= x <= 1.39:
                 complete_check()
                 print 'current dict length:', referencelength
-            assert l_dict.num_items == referencelength
+            assert l_dict.num_live_items == referencelength
         complete_check()
 
     def test_stress_2(self):
@@ -418,5 +418,5 @@
             if 1.38 <= x <= 1.39:
                 complete_check()
                 print 'current dict length:', referencelength
-            assert l_dict.num_items == referencelength
+            assert l_dict.num_live_items == referencelength
         complete_check()

From noreply at buildbot.pypy.org  Mon Dec 15 19:06:12 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Mon, 15 Dec 2014 19:06:12 +0100 (CET)
Subject: [pypy-commit] pypy default: Compact dictionaries that have very few
 live items left in them.
Message-ID: <20141215180612.56C7D1C1347@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r74945:22ee57670e0d
Date: 2014-12-15 16:37 +0000
http://bitbucket.org/pypy/pypy/changeset/22ee57670e0d/

Log:	Compact dictionaries that have very few live items left in them.

	For some reason, we copied CPython's hard-to-defend behaviour here.
	Replace it with a (very) conservative check that will only shrink
	the memory allocated for a dictionary if it has very few items left.

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -661,19 +661,11 @@
         entry.key = lltype.nullptr(ENTRY.key.TO)
     if ENTRIES.must_clear_value:
         entry.value = lltype.nullptr(ENTRY.value.TO)
-    #
-    # The rest is commented out: like CPython we no longer shrink the
-    # dictionary here.  It may shrink later if we try to append a number
-    # of new items to it.  Unsure if this behavior was designed in
-    # CPython or is accidental.  A design reason would be that if you
-    # delete all items in a dictionary (e.g. with a series of
-    # popitem()), then CPython avoids shrinking the table several times.
-    #num_entries = len(d.entries)
-    #if num_entries > DICT_INITSIZE and d.num_items <= num_entries / 4:
-    #    ll_dict_resize(d)
-    # A previous xxx: move the size checking and resize into a single
-    # call which is opaque to the JIT when the dict isn't virtual, to
-    # avoid extra branches.
+
+    # If the dictionary is at least 87.5% dead items, then consider shrinking
+    # it.
+    if d.num_live_items + DICT_INITSIZE <= len(d.entries) / 8:
+        ll_dict_resize(d)
 
 def ll_dict_resize(d):
     # make a 'new_size' estimate and shrink it if there are many

From noreply at buildbot.pypy.org  Mon Dec 15 19:06:13 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Mon, 15 Dec 2014 19:06:13 +0100 (CET)
Subject: [pypy-commit] pypy default: Reuse deleted items at the end of an
	orderedarray.
Message-ID: <20141215180613.8460B1C1347@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r74946:16ce1a0def4d
Date: 2014-12-15 17:03 +0000
http://bitbucket.org/pypy/pypy/changeset/16ce1a0def4d/

Log:	Reuse deleted items at the end of an orderedarray.

	If a user deletes item(s) at the end of an orderedarray, they can be
	immediately reused rather than marked as dead and compacted later.

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -662,6 +662,19 @@
     if ENTRIES.must_clear_value:
         entry.value = lltype.nullptr(ENTRY.value.TO)
 
+    if index == d.num_ever_used_items - 1:
+        # The last element of the ordereddict has been deleted. Instead of
+        # simply marking the item as dead, we can safely reuse it. Since it's
+        # also possible that there are more dead items immediately behind the
+        # last one, we reclaim all the dead items at the end of the ordereditem
+        # at the same point.
+        i = d.num_ever_used_items - 2
+        while i >= 0 and not d.entries.valid(i):
+            i -= 1
+        j = i + 1
+        assert j >= 0
+        d.num_ever_used_items = j
+
     # If the dictionary is at least 87.5% dead items, then consider shrinking
     # it.
     if d.num_live_items + DICT_INITSIZE <= len(d.entries) / 8:

From noreply at buildbot.pypy.org  Mon Dec 15 19:06:14 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Mon, 15 Dec 2014 19:06:14 +0100 (CET)
Subject: [pypy-commit] pypy default: Merge.
Message-ID: <20141215180614.B1FDD1C1347@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r74947:acbf6e9532c4
Date: 2014-12-15 17:52 +0000
http://bitbucket.org/pypy/pypy/changeset/acbf6e9532c4/

Log:	Merge.

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -330,6 +330,20 @@
     keepalive_until_here(newp)
     return newp
 
+ at jit.dont_look_inside
+ at specialize.ll()
+def ll_arrayclear(p):
+    # Equivalent to memset(array, 0).  Only for GcArray(primitive-type) for now.
+    from rpython.rlib.objectmodel import keepalive_until_here
+
+    length = len(p)
+    ARRAY = lltype.typeOf(p).TO
+    offset = llmemory.itemoffsetof(ARRAY, 0)
+    dest_addr = llmemory.cast_ptr_to_adr(p) + offset
+    llmemory.raw_memclear(dest_addr, llmemory.sizeof(ARRAY.OF) * length)
+    keepalive_until_here(p)
+
+
 def no_release_gil(func):
     func._dont_inline_ = True
     func._no_release_gil_ = True
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -158,6 +158,16 @@
     assert a2[2].x == 3
     assert a2[2].y == 15
 
+def test_ll_arrayclear():
+    TYPE = lltype.GcArray(lltype.Signed)
+    a1 = lltype.malloc(TYPE, 10)
+    for i in range(10):
+        a1[i] = 100 + i
+    rgc.ll_arrayclear(a1)
+    assert len(a1) == 10
+    for i in range(10):
+        assert a1[i] == 0
+
 def test__contains_gcptr():
     assert not rgc._contains_gcptr(lltype.Signed)
     assert not rgc._contains_gcptr(
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -72,6 +72,8 @@
         'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr)
                              and DICTVALUE._needsgc()),
         }
+    if getattr(ll_eq_function, 'no_direct_compare', False):
+        entrymeths['no_direct_compare'] = True
 
     # * the key
     entryfields.append(("key", DICTKEY))
@@ -416,6 +418,7 @@
 TYPE_LONG  = lltype.Unsigned
 
 def ll_malloc_indexes_and_choose_lookup(d, n):
+    # keep in sync with ll_clear_indexes() below
     if n <= 256:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_BYTE.TO, n,
@@ -437,6 +440,16 @@
                                                          zero=True))
         d.lookup_function_no = FUNC_LONG
 
+def ll_clear_indexes(d, n):
+    if n <= 256:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_BYTE, d.indexes))
+    elif n <= 65536:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_SHORT, d.indexes))
+    elif IS_64BIT and n <= 2 ** 32:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_INT, d.indexes))
+    else:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_LONG, d.indexes))
+
 def ll_call_insert_clean_function(d, hash, i):
     DICT = lltype.typeOf(d).TO
     if d.lookup_function_no == FUNC_BYTE:
@@ -605,6 +618,11 @@
         newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
     else:
         newitems = d.entries
+        # The loop below does a lot of writes into 'newitems'.  It's a better
+        # idea to do a single gc_writebarrier rather than activating the
+        # card-by-card logic (worth 11% in microbenchmarks).
+        from rpython.rtyper.lltypesystem.lloperation import llop
+        llop.gc_writebarrier(lltype.Void, newitems)
     #
     ENTRIES = lltype.typeOf(d).TO.entries.TO
     ENTRY = ENTRIES.OF
@@ -702,13 +720,17 @@
         ll_dict_reindex(d, new_size)
 
 def ll_dict_reindex(d, new_size):
-    ll_malloc_indexes_and_choose_lookup(d, new_size)
+    if bool(d.indexes) and _ll_len_of_d_indexes(d) == new_size:
+        ll_clear_indexes(d, new_size)   # hack: we can reuse the same array
+    else:
+        ll_malloc_indexes_and_choose_lookup(d, new_size)
     d.resize_counter = new_size * 2 - d.num_live_items * 3
     assert d.resize_counter > 0
     #
     entries = d.entries
     i = 0
-    while i < d.num_ever_used_items:
+    ibound = d.num_ever_used_items
+    while i < ibound:
         if entries.valid(i):
             hash = entries.hash(i)
             ll_call_insert_clean_function(d, hash, i)
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -292,9 +292,6 @@
         res = self.interpret(func, [5])
         assert res == 6
 
-    def test_dict_with_SHORT_keys(self):
-        py.test.skip("I don't want to edit this file on two branches")
-
     def test_memoryerror_should_not_insert(self):
         py.test.skip("I don't want to edit this file on two branches")
 

From noreply at buildbot.pypy.org  Mon Dec 15 20:42:05 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 15 Dec 2014 20:42:05 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: raise ValueError on null chars in
	_sqlite3
Message-ID: <20141215194205.35C0F1C050D@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74948:6855ba726fb0
Date: 2014-12-15 14:41 -0500
http://bitbucket.org/pypy/pypy/changeset/6855ba726fb0/

Log:	raise ValueError on null chars in _sqlite3

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1202,6 +1202,8 @@
 
         if not isinstance(sql, basestring):
             raise Warning("SQL is of wrong type. Must be string or unicode.")
+        if '\0' in sql:
+            raise ValueError("the query contains a null character")
 
         first_word = sql.lstrip().split(" ")[0].upper()
         if first_word == "":
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -263,6 +263,19 @@
         finally:
             del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
 
+    def test_null_character(self, con):
+        if not hasattr(_sqlite3, '_ffi') and sys.version_info < (2, 7, 9):
+            pytest.skip("_sqlite3 too old")
+        exc = raises(ValueError, con, "\0select 1")
+        assert str(exc.value) == "the query contains a null character"
+        exc = raises(ValueError, con, "select 1\0")
+        assert str(exc.value) == "the query contains a null character"
+        cur = con.cursor()
+        exc = raises(ValueError, cur.execute, "\0select 2")
+        assert str(exc.value) == "the query contains a null character"
+        exc = raises(ValueError, cur.execute, "select 2\0")
+        assert str(exc.value) == "the query contains a null character"
+
 
 class TestSQLiteHost(BaseTestSQLite):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Mon Dec 15 21:07:52 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 15 Dec 2014 21:07:52 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: test/fix _socket.socket repr
Message-ID: <20141215200752.3FDBC1C1347@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74949:ef0308d0f344
Date: 2014-12-15 15:06 -0500
http://bitbucket.org/pypy/pypy/changeset/ef0308d0f344/

Log:	test/fix _socket.socket repr

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -142,6 +142,12 @@
     def get_family_w(self, space):
         return space.wrap(self.sock.family)
 
+    def descr_repr(self, space):
+        return space.wrap("<socket object, fd=%d, family=%d,"
+                          " type=%d, protocol=%d>" %
+                          (self.sock.fd, self.sock.family,
+                           self.sock.type, self.sock.proto))
+
     def accept_w(self, space):
         """accept() -> (socket object, address info)
 
@@ -661,6 +667,7 @@
  [*] not available on all platforms!""",
     __new__ = descr_socket_new,
     __weakref__ = make_weakref_descr(W_Socket),
+    __repr__ = interp2app(W_Socket.descr_repr),
     type = GetSetProperty(W_Socket.get_type_w),
     proto = GetSetProperty(W_Socket.get_proto_w),
     family = GetSetProperty(W_Socket.get_family_w),
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -372,6 +372,19 @@
         assert s.fileno() > -1
         assert isinstance(s.fileno(), int)
 
+    def test_socket_repr(self):
+        import _socket
+        s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM)
+        try:
+            expected = ('<socket object, fd=%s, family=%s, type=%s, protocol=%s>'
+                        % (s.fileno(), s.family, s.type, s.proto))
+            assert repr(s) == expected
+        finally:
+            s.close()
+        expected = ('<socket object, fd=-1, family=%s, type=%s, protocol=%s>'
+                    % (s.family, s.type, s.proto))
+        assert repr(s) == expected
+
     def test_socket_close(self):
         import _socket, os
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)

From noreply at buildbot.pypy.org  Mon Dec 15 23:50:23 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 15 Dec 2014 23:50:23 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: test/fix re.match.group accepting
	long
Message-ID: <20141215225023.B64141C1347@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74950:ab9dcc40fcf4
Date: 2014-12-15 17:49 -0500
http://bitbucket.org/pypy/pypy/changeset/ab9dcc40fcf4/

Log:	test/fix re.match.group accepting long

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -415,7 +415,8 @@
         try:
             groupnum = space.int_w(w_arg)
         except OperationError, e:
-            if not e.match(space, space.w_TypeError):
+            if not e.match(space, space.w_TypeError) and \
+                    not e.match(space, space.w_OverflowError):
                 raise
             try:
                 w_groupnum = space.getitem(self.srepat.w_groupindex, w_arg)
diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -191,6 +191,13 @@
         raises(IndexError, m.group, 'foobarbaz')
         raises(IndexError, m.group, 'first', 'foobarbaz')
 
+    def test_group_takes_long(self):
+        import re
+        import sys
+        assert re.match("(foo)", "foo").group(1L) == "foo"
+        exc = raises(IndexError, re.match("", "").group, sys.maxint + 1)
+        assert str(exc.value) == "no such group"
+
     def test_expand(self):
         import re
         m = re.search("a(..)(?P<name>..)", "ab1bc")

From noreply at buildbot.pypy.org  Tue Dec 16 01:21:21 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 01:21:21 +0100 (CET)
Subject: [pypy-commit] pypy default: Minimal change to fix
 test_memoryerror_should_not_insert.
Message-ID: <20141216002121.645691C050D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74951:bad1c782ed83
Date: 2014-12-16 00:14 +0000
http://bitbucket.org/pypy/pypy/changeset/bad1c782ed83/

Log:	Minimal change to fix test_memoryerror_should_not_insert.

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -532,9 +532,27 @@
         entry = d.entries[i]
         entry.value = value
     else:
+        reindexed = False
         if len(d.entries) == d.num_ever_used_items:
-            if ll_dict_grow(d):
-                ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
+            try:
+                reindexed = ll_dict_grow(d)
+            except MemoryError:
+                _ll_dict_rescue(d)
+                raise
+        rc = d.resize_counter - 3
+        if rc <= 0:
+            try:
+                ll_dict_resize(d)
+                reindexed = True
+            except MemoryError:
+                _ll_dict_rescue(d)
+                raise
+            rc = d.resize_counter - 3
+            ll_assert(rc > 0, "ll_dict_resize failed?")
+        if reindexed:
+            ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
+        #
+        d.resize_counter = rc
         entry = d.entries[d.num_ever_used_items]
         entry.key = key
         entry.value = value
@@ -544,12 +562,13 @@
             entry.f_valid = True
         d.num_ever_used_items += 1
         d.num_live_items += 1
-        rc = d.resize_counter - 3
-        if rc <= 0:
-            ll_dict_resize(d)
-            rc = d.resize_counter - 3
-            ll_assert(rc > 0, "ll_dict_resize failed?")
-        d.resize_counter = rc
+
+def _ll_dict_rescue(d):
+    # MemoryError situation!  The 'indexes' contains an invalid entry
+    # at this point.  But we can call ll_dict_reindex() with the
+    # following arguments, ensuring no further malloc occurs.
+    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+_ll_dict_rescue._dont_inline_ = True
 
 def _ll_dict_insertclean(d, key, value, hash):
     ENTRY = lltype.typeOf(d.entries).TO.OF
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -292,9 +292,6 @@
         res = self.interpret(func, [5])
         assert res == 6
 
-    def test_memoryerror_should_not_insert(self):
-        py.test.skip("I don't want to edit this file on two branches")
-
 
 class TestStress:
 

From noreply at buildbot.pypy.org  Tue Dec 16 17:10:40 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 17:10:40 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix a CPython test
Message-ID: <20141216161040.2193A1D239C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74953:09728db49463
Date: 2014-12-16 15:31 +0000
http://bitbucket.org/pypy/pypy/changeset/09728db49463/

Log:	Fix a CPython test

diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -1179,7 +1179,11 @@
         od = OrderedDict(pairs)
         # yaml.dump(od) -->
         # '!!python/object/apply:__main__.OrderedDict\n- - [a, 1]\n  - [b, 2]\n'
-        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1]))
+
+        # PyPy bug fix: added [0] at the end of this line, because the
+        # test is really about the 2-tuples that need to be 2-lists
+        # inside the list of 6 of them
+        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1][0]))
 
     def test_reduce_not_too_fat(self):
         # do not save instance dictionary if not needed

From noreply at buildbot.pypy.org  Tue Dec 16 17:10:41 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 17:10:41 +0100 (CET)
Subject: [pypy-commit] pypy default: Add a test for
	'OrderedDict.popitem(last=False)'
Message-ID: <20141216161041.4DA3C1D239C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74954:702aaf7c9043
Date: 2014-12-16 16:10 +0000
http://bitbucket.org/pypy/pypy/changeset/702aaf7c9043/

Log:	Add a test for 'OrderedDict.popitem(last=False)'

diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -1108,6 +1108,16 @@
             od.popitem()
         self.assertEqual(len(od), 0)
 
+    def test_popitem_first(self):
+        pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
+        shuffle(pairs)
+        od = OrderedDict(pairs)
+        while pairs:
+            self.assertEqual(od.popitem(last=False), pairs.pop(0))
+        with self.assertRaises(KeyError):
+            od.popitem(last=False)
+        self.assertEqual(len(od), 0)
+
     def test_pop(self):
         pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
         shuffle(pairs)

From noreply at buildbot.pypy.org  Tue Dec 16 17:19:07 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 17:19:07 +0100 (CET)
Subject: [pypy-commit] pypy default: (ltratt, arigo)
Message-ID: <20141216161907.868E91C01C3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74955:b823dea08125
Date: 2014-12-16 16:18 +0000
http://bitbucket.org/pypy/pypy/changeset/b823dea08125/

Log:	(ltratt, arigo)

	Add a test for the exact output of __reduce__

diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -1203,6 +1203,16 @@
         od.x = 10
         self.assertEqual(len(od.__reduce__()), 3)
 
+    def test_reduce_exact_output(self):
+        # PyPy: test that __reduce__() produces the exact same answer as
+        # CPython does, even though in the 'all_ordered_dicts' branch we
+        # have to emulate it.
+        pairs = [['c', 1], ['b', 2], ['d', 4]]
+        od = OrderedDict(pairs)
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,)))
+        od.x = 10
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,), {'x': 10}))
+
     def test_repr(self):
         od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)])
         self.assertEqual(repr(od),

From noreply at buildbot.pypy.org  Tue Dec 16 17:28:49 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Tue, 16 Dec 2014 17:28:49 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: Make all dictionaries be
	ordered by default.
Message-ID: <20141216162849.CF29E1D3554@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: all_ordered_dicts
Changeset: r74956:91b77ba66c70
Date: 2014-12-16 14:48 +0000
http://bitbucket.org/pypy/pypy/changeset/91b77ba66c70/

Log:	Make all dictionaries be ordered by default.

	This could be done in a less ugly way in the long term.

diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -389,6 +389,8 @@
         assert isinstance(dct2, SomeOrderedDict), "OrderedDict.update(dict) not allowed"
         dct1.dictdef.union(dct2.dictdef)
 
+SomeDict=SomeOrderedDict
+
 
 class SomeIterator(SomeObject):
     "Stands for an iterator returning objects from a given container."

From noreply at buildbot.pypy.org  Tue Dec 16 17:28:51 2014
From: noreply at buildbot.pypy.org (ltratt)
Date: Tue, 16 Dec 2014 17:28:51 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: Merge default.
Message-ID: <20141216162851.2FA311D3554@cobra.cs.uni-duesseldorf.de>

Author: Laurence Tratt <laurie at tratt.net>
Branch: all_ordered_dicts
Changeset: r74957:a8e941c88899
Date: 2014-12-16 16:21 +0000
http://bitbucket.org/pypy/pypy/changeset/a8e941c88899/

Log:	Merge default.

diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -1108,6 +1108,16 @@
             od.popitem()
         self.assertEqual(len(od), 0)
 
+    def test_popitem_first(self):
+        pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
+        shuffle(pairs)
+        od = OrderedDict(pairs)
+        while pairs:
+            self.assertEqual(od.popitem(last=False), pairs.pop(0))
+        with self.assertRaises(KeyError):
+            od.popitem(last=False)
+        self.assertEqual(len(od), 0)
+
     def test_pop(self):
         pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
         shuffle(pairs)
@@ -1179,7 +1189,11 @@
         od = OrderedDict(pairs)
         # yaml.dump(od) -->
         # '!!python/object/apply:__main__.OrderedDict\n- - [a, 1]\n  - [b, 2]\n'
-        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1]))
+
+        # PyPy bug fix: added [0] at the end of this line, because the
+        # test is really about the 2-tuples that need to be 2-lists
+        # inside the list of 6 of them
+        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1][0]))
 
     def test_reduce_not_too_fat(self):
         # do not save instance dictionary if not needed
@@ -1189,6 +1203,16 @@
         od.x = 10
         self.assertEqual(len(od.__reduce__()), 3)
 
+    def test_reduce_exact_output(self):
+        # PyPy: test that __reduce__() produces the exact same answer as
+        # CPython does, even though in the 'all_ordered_dicts' branch we
+        # have to emulate it.
+        pairs = [['c', 1], ['b', 2], ['d', 4]]
+        od = OrderedDict(pairs)
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,)))
+        od.x = 10
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,), {'x': 10}))
+
     def test_repr(self):
         od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)])
         self.assertEqual(repr(od),

From noreply at buildbot.pypy.org  Tue Dec 16 17:42:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 17:42:47 +0100 (CET)
Subject: [pypy-commit] pypy default: fix
Message-ID: <20141216164247.424461D239C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74958:badc6fe2b088
Date: 2014-12-16 17:39 +0100
http://bitbucket.org/pypy/pypy/changeset/badc6fe2b088/

Log:	fix

diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -389,6 +389,8 @@
         assert isinstance(dct2, SomeOrderedDict), "OrderedDict.update(dict) not allowed"
         dct1.dictdef.union(dct2.dictdef)
 
+SomeDict = SomeOrderedDict
+
 
 class SomeIterator(SomeObject):
     "Stands for an iterator returning objects from a given container."
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -563,6 +563,7 @@
         d.num_ever_used_items += 1
         d.num_live_items += 1
 
+ at jit.dont_look_inside
 def _ll_dict_rescue(d):
     # MemoryError situation!  The 'indexes' contains an invalid entry
     # at this point.  But we can call ll_dict_reindex() with the
@@ -588,6 +589,7 @@
 def _ll_len_of_d_indexes(d):
     # xxx Haaaack: returns len(d.indexes).  Works independently of
     # the exact type pointed to by d, using a forced cast...
+    # Must only be called by @jit.dont_look_inside functions.
     return len(rffi.cast(DICTINDEX_BYTE, d.indexes))
 
 def _overallocate_entries_len(baselen):

From noreply at buildbot.pypy.org  Tue Dec 16 17:42:48 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 17:42:48 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: hg merge default
Message-ID: <20141216164248.6804D1D239C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r74959:5cefc7e504a8
Date: 2014-12-16 17:42 +0100
http://bitbucket.org/pypy/pypy/changeset/5cefc7e504a8/

Log:	hg merge default

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -563,6 +563,7 @@
         d.num_ever_used_items += 1
         d.num_live_items += 1
 
+ at jit.dont_look_inside
 def _ll_dict_rescue(d):
     # MemoryError situation!  The 'indexes' contains an invalid entry
     # at this point.  But we can call ll_dict_reindex() with the
@@ -588,6 +589,7 @@
 def _ll_len_of_d_indexes(d):
     # xxx Haaaack: returns len(d.indexes).  Works independently of
     # the exact type pointed to by d, using a forced cast...
+    # Must only be called by @jit.dont_look_inside functions.
     return len(rffi.cast(DICTINDEX_BYTE, d.indexes))
 
 def _overallocate_entries_len(baselen):

From noreply at buildbot.pypy.org  Tue Dec 16 17:44:07 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 17:44:07 +0100 (CET)
Subject: [pypy-commit] pypy default: oups
Message-ID: <20141216164407.13D041D239C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74960:ec2b02ba1cc9
Date: 2014-12-16 17:43 +0100
http://bitbucket.org/pypy/pypy/changeset/ec2b02ba1cc9/

Log:	oups

diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -389,8 +389,6 @@
         assert isinstance(dct2, SomeOrderedDict), "OrderedDict.update(dict) not allowed"
         dct1.dictdef.union(dct2.dictdef)
 
-SomeDict = SomeOrderedDict
-
 
 class SomeIterator(SomeObject):
     "Stands for an iterator returning objects from a given container."

From noreply at buildbot.pypy.org  Tue Dec 16 18:53:21 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 18:53:21 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: (ltratt, arigo)
Message-ID: <20141216175321.860481C01C3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r74961:7ff22ce8993e
Date: 2014-12-16 17:51 +0000
http://bitbucket.org/pypy/pypy/changeset/7ff22ce8993e/

Log:	(ltratt, arigo)

	Attempt to fix popitem() bug (maybe?)

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -463,6 +463,19 @@
     else:
         assert False
 
+def ll_call_delete_by_entry_index(d, hash, i):
+    DICT = lltype.typeOf(d).TO
+    if d.lookup_function_no == FUNC_BYTE:
+        ll_dict_delete_by_entry_index(d, hash, i, TYPE_BYTE)
+    elif d.lookup_function_no == FUNC_SHORT:
+        ll_dict_delete_by_entry_index(d, hash, i, TYPE_SHORT)
+    elif IS_64BIT and d.lookup_function_no == FUNC_INT:
+        ll_dict_delete_by_entry_index(d, hash, i, TYPE_INT)
+    elif d.lookup_function_no == FUNC_LONG:
+        ll_dict_delete_by_entry_index(d, hash, i, TYPE_LONG)
+    else:
+        assert False
+
 def ll_valid_from_flag(entries, i):
     return entries[i].f_valid
 
@@ -769,23 +782,11 @@
 FLAG_LOOKUP = 0
 FLAG_STORE = 1
 FLAG_DELETE = 2
-FLAG_DELETE_TRY_HARD = 3
 
 @specialize.memo()
 def _ll_ptr_to_array_of(T):
     return lltype.Ptr(lltype.GcArray(T))
 
-def ll_kill_something(d, T):
-    INDEXES = _ll_ptr_to_array_of(T)
-    i = 0
-    indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
-    while True:
-        index = rffi.cast(lltype.Signed, indexes[i])
-        if index >= VALID_OFFSET:
-            indexes[i] = rffi.cast(T, DELETED)
-            return index
-        i += 1
-
 @jit.look_inside_iff(lambda d, key, hash, store_flag, T:
                      jit.isvirtual(d) and jit.isconstant(key))
 @jit.oopspec('ordereddict.lookup(d, key, hash, store_flag, T)')
@@ -827,8 +828,6 @@
         # pristine entry -- lookup failed
         if store_flag == FLAG_STORE:
             indexes[i] = rffi.cast(T, d.num_ever_used_items + VALID_OFFSET)
-        elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
-            return ll_kill_something(d, T)
         return -1
 
     # In the loop, a deleted entry (everused and not valid) is by far
@@ -845,8 +844,6 @@
                     deletedslot = intmask(i)
                 indexes[deletedslot] = rffi.cast(T, d.num_ever_used_items +
                                                  VALID_OFFSET)
-            elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
-                return ll_kill_something(d, T)
             return -1
         elif index >= VALID_OFFSET:
             checkingkey = entries[index - VALID_OFFSET].key
@@ -881,12 +878,31 @@
     mask = len(indexes) - 1
     i = r_uint(hash & mask)
     perturb = r_uint(hash)
-    while rffi.cast(lltype.Signed, indexes[i]) != 0:
+    while rffi.cast(lltype.Signed, indexes[i]) != FREE:
         i = (i << 2) + i + perturb + 1
         i = i & mask
         perturb >>= PERTURB_SHIFT
     indexes[i] = rffi.cast(T, index + VALID_OFFSET)
 
+def ll_dict_delete_by_entry_index(d, hash, locate_index, T):
+    # Another simplified version of ll_dict_lookup() which locates a
+    # hashtable entry with the given 'index' stored in it, and deletes it.
+    # This *should* be safe against evil user-level __eq__/__hash__
+    # functions because the 'hash' argument here should be the one stored
+    # into the directory, which is correct.
+    INDEXES = _ll_ptr_to_array_of(T)
+    indexes = lltype.cast_opaque_ptr(INDEXES, d.indexes)
+    mask = len(indexes) - 1
+    i = r_uint(hash & mask)
+    perturb = r_uint(hash)
+    locate_value = locate_index + VALID_OFFSET
+    while rffi.cast(lltype.Signed, indexes[i]) != locate_value:
+        assert rffi.cast(lltype.Signed, indexes[i]) != FREE
+        i = (i << 2) + i + perturb + 1
+        i = i & mask
+        perturb >>= PERTURB_SHIFT
+    indexes[i] = rffi.cast(T, DELETED)
+
 # ____________________________________________________________
 #
 #  Irregular operations.
@@ -1133,16 +1149,9 @@
             break
         dic.num_ever_used_items -= 1
 
-    key = entries[i].key
-    index = dic.lookup_function(dic, key, entries.hash(i),
-                                FLAG_DELETE_TRY_HARD)
-    # if the lookup function returned me a random strange thing,
-    # don't care about deleting the item
-    if index == dic.num_ever_used_items - 1:
-        dic.num_ever_used_items -= 1
-    else:
-        assert index != -1
-    return index
+    # we must remove the precise entry in the hashtable that points to 'i'
+    ll_call_delete_by_entry_index(dic, entries.hash(i), i)
+    return i
 
 def ll_dict_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)

From noreply at buildbot.pypy.org  Tue Dec 16 19:09:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 19:09:15 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: Add a comment
Message-ID: <20141216180915.AE4561D239C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r74962:317f9db81f78
Date: 2014-12-16 18:09 +0000
http://bitbucket.org/pypy/pypy/changeset/317f9db81f78/

Log:	Add a comment

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -1143,6 +1143,10 @@
 
     entries = dic.entries
 
+    # find the last entry.  It's unclear if the loop below is still
+    # needed nowadays, because 'num_ever_used_items - 1' should always
+    # point to the last active item (we decrease it as needed in
+    # _ll_dict_del).  Better safe than sorry.
     while True:
         i = dic.num_ever_used_items - 1
         if entries.valid(i):

From noreply at buildbot.pypy.org  Tue Dec 16 21:03:23 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 21:03:23 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: Fix another CPython test
Message-ID: <20141216200323.CAA741D28DF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r74963:e283fc272bc7
Date: 2014-12-16 20:03 +0000
http://bitbucket.org/pypy/pypy/changeset/e283fc272bc7/

Log:	Fix another CPython test

diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -578,7 +578,12 @@
             def __repr__(self):
                 return "MySet(%s)" % repr(list(self))
         s = MySet([5,43,2,1])
-        self.assertEqual(s.pop(), 1)
+        # changed from CPython 2.7: it was "s.pop() == 1" but I see
+        # nothing that guarantees a particular order here.  In the
+        # 'all_ordered_dicts' branch of PyPy (or with OrderedDict
+        # instead of sets), it consistently returns 5, but this test
+        # should not rely on this or any other order.
+        self.assert_(s.pop() in [5,43,2,1])
 
     def test_issue8750(self):
         empty = WithSet()

From noreply at buildbot.pypy.org  Tue Dec 16 21:16:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 16 Dec 2014 21:16:47 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Pick a directory name in /tmp
 that doesn't conflict with different
Message-ID: <20141216201647.4E68A1D27F5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r303:23b8edbb7fae
Date: 2014-12-16 20:17 +0000
http://bitbucket.org/pypy/benchmarks/changeset/23b8edbb7fae/

Log:	Pick a directory name in /tmp that doesn't conflict with different
	users, because then we get permission denied

diff --git a/own/icbd/icbd/type_analyzer/analyze_all.py b/own/icbd/icbd/type_analyzer/analyze_all.py
--- a/own/icbd/icbd/type_analyzer/analyze_all.py
+++ b/own/icbd/icbd/type_analyzer/analyze_all.py
@@ -73,7 +73,7 @@
         prune_amount = 0
 
     if output_dir is None:
-        output_dir = os.path.join("/tmp", "icbd_%s" % proj_name)
+        output_dir = os.path.join("/tmp", "icbd_%s_%d" % (proj_name, os.getuid()))
 
     static_dir = os.path.join(output_dir, "static")
 

From noreply at buildbot.pypy.org  Wed Dec 17 10:26:18 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 17 Dec 2014 10:26:18 +0100 (CET)
Subject: [pypy-commit] pypy default: minor simplification - only export
 state when necessary (that is when we're
Message-ID: <20141217092618.DBD651C04B4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74964:f2d7e0e3f156
Date: 2014-12-17 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/f2d7e0e3f156/

Log:	minor simplification - only export state when necessary (that is
	when we're optimizing preamble or base for retrace)

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -128,7 +128,8 @@
                       [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
 
     try:
-        start_state = optimize_trace(metainterp_sd, part, enable_opts)
+        start_state = optimize_trace(metainterp_sd, part, enable_opts,
+                                     export_state=True)
     except InvalidLoop:
         return None
     target_token = part.operations[0].getdescr()
@@ -156,7 +157,7 @@
 
         try:
             optimize_trace(metainterp_sd, part, enable_opts,
-                           start_state=start_state)
+                           start_state=start_state, export_state=False)
         except InvalidLoop:
             return None
 
@@ -209,7 +210,7 @@
     assert label.getopnum() == rop.LABEL
     try:
         optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
-                       start_state=start_state)
+                       start_state=start_state, export_state=False)
     except InvalidLoop:
         # Fall back on jumping to preamble
         target_token = label.getdescr()
@@ -220,7 +221,8 @@
         try:
             optimize_trace(metainterp_sd, part,
                            jitdriver_sd.warmstate.enable_opts,
-                           inline_short_preamble=False, start_state=start_state)
+                           inline_short_preamble=False, start_state=start_state,
+                           export_state=False)
         except InvalidLoop:
             return None
     assert part.operations[-1].getopnum() != rop.LABEL
@@ -789,7 +791,8 @@
     else:
         inline_short_preamble = True
     try:
-        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts, inline_short_preamble)
+        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts,
+                               inline_short_preamble, export_state=True)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -48,7 +48,8 @@
     return optimizations, unroll
 
 def optimize_trace(metainterp_sd, loop, enable_opts,
-                   inline_short_preamble=True, start_state=None):
+                   inline_short_preamble=True, start_state=None,
+                   export_state=True):
     """Optimize loop.operations to remove internal overheadish operations.
     """
 
@@ -59,7 +60,8 @@
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
             return optimize_unroll(metainterp_sd, loop, optimizations,
-                                   inline_short_preamble, start_state)
+                                   inline_short_preamble, start_state,
+                                   export_state)
         else:
             optimizer = Optimizer(metainterp_sd, loop, optimizations)
             optimizer.propagate_all_forward()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -45,7 +45,8 @@
             part.operations = operations
 
             self.add_guard_future_condition(part)
-            state = self._do_optimize_loop(part, None, state)
+            state = self._do_optimize_loop(part, None, state,
+                                           export_state=True)
             if part.operations[-1].getopnum() == rop.LABEL:
                 last_label = [part.operations.pop()]
             else:
@@ -497,7 +498,8 @@
 
 class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel):
 
-    def _do_optimize_loop(self, loop, call_pure_results, state):
+    def _do_optimize_loop(self, loop, call_pure_results, state,
+                          export_state=False):
         from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
         from rpython.jit.metainterp.optimizeopt.util import args_dict
         from rpython.jit.metainterp.optimizeopt.pure import OptPure
@@ -505,7 +507,7 @@
         self.loop = loop
         loop.call_pure_results = args_dict()
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        return optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True, state)
+        return optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True, state, export_state)
 
     def test_optimizer_renaming_boxes1(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -104,7 +104,7 @@
         if loop.operations[-1].getopnum() == rop.JUMP:
             loop.operations[-1].setdescr(token)
         expected = convert_old_style_to_targets(self.parse(optops), jump=True)
-        self._do_optimize_loop(loop, call_pure_results)
+        self._do_optimize_loop(loop, call_pure_results, export_state=False)
         print '\n'.join([str(o) for o in loop.operations])
         self.assert_equal(loop, expected)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -390,7 +390,8 @@
         assert equaloplists(optimized.operations,
                             expected.operations, False, remap, text_right)
 
-    def _do_optimize_loop(self, loop, call_pure_results, start_state=None):
+    def _do_optimize_loop(self, loop, call_pure_results, start_state=None,
+                          export_state=False):
         from rpython.jit.metainterp.optimizeopt import optimize_trace
         from rpython.jit.metainterp.optimizeopt.util import args_dict
 
@@ -406,7 +407,8 @@
             metainterp_sd.callinfocollection = self.callinfocollection
         #
         return optimize_trace(metainterp_sd, loop, self.enable_opts,
-                              start_state=start_state)
+                              start_state=start_state,
+                              export_state=export_state)
 
     def unroll_and_optimize(self, loop, call_pure_results=None):
         self.add_guard_future_condition(loop)
@@ -426,7 +428,8 @@
         preamble.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(token))] + \
                               operations +  \
                               [ResOperation(rop.LABEL, jump_args, None, descr=token)]
-        start_state = self._do_optimize_loop(preamble, call_pure_results)
+        start_state = self._do_optimize_loop(preamble, call_pure_results,
+                                             export_state=True)
 
         assert preamble.operations[-1].getopnum() == rop.LABEL
 
@@ -440,7 +443,8 @@
         assert loop.operations[0].getopnum() == rop.LABEL
         loop.inputargs = loop.operations[0].getarglist()
 
-        self._do_optimize_loop(loop, call_pure_results, start_state)
+        self._do_optimize_loop(loop, call_pure_results, start_state,
+                               export_state=False)
         extra_same_as = []
         while loop.operations[0].getopnum() != rop.LABEL:
             extra_same_as.append(loop.operations[0])
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -16,10 +16,11 @@
 # FIXME: Introduce some VirtualOptimizer super class instead
 
 def optimize_unroll(metainterp_sd, loop, optimizations,
-                    inline_short_preamble=True, start_state=None):
+                    inline_short_preamble=True, start_state=None,
+                    export_state=True):
     opt = UnrollOptimizer(metainterp_sd, loop, optimizations)
     opt.inline_short_preamble = inline_short_preamble
-    return opt.propagate_all_forward(start_state)
+    return opt.propagate_all_forward(start_state, export_state)
 
 
 class UnrollableOptimizer(Optimizer):
@@ -70,7 +71,7 @@
         prev = self.fix_snapshot(jump_args, snapshot.prev)
         return Snapshot(prev, new_snapshot_args)
 
-    def propagate_all_forward(self, starting_state):
+    def propagate_all_forward(self, starting_state, export_state=True):
         loop = self.optimizer.loop
         self.optimizer.clear_newoperations()
 
@@ -148,7 +149,10 @@
         KillHugeIntBounds(self.optimizer).apply()
 
         loop.operations = self.optimizer.get_newoperations()
-        final_state = self.export_state(stop_label)
+        if export_state:
+            final_state = self.export_state(stop_label)
+        else:
+            final_state = None
         loop.operations.append(stop_label)
         return final_state
 

From noreply at buildbot.pypy.org  Wed Dec 17 10:29:41 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 17 Dec 2014 10:29:41 +0100 (CET)
Subject: [pypy-commit] pypy default: we don't need to kill huge int bounds
 if we're just finishing the optimizer pass
Message-ID: <20141217092941.AE1891C01C3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74965:da4d4e229a6d
Date: 2014-12-17 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/da4d4e229a6d/

Log:	we don't need to kill huge int bounds if we're just finishing the
	optimizer pass

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -146,7 +146,8 @@
             self.close_bridge(start_label)
 
         self.optimizer.flush()
-        KillHugeIntBounds(self.optimizer).apply()
+        if export_state:
+            KillHugeIntBounds(self.optimizer).apply()
 
         loop.operations = self.optimizer.get_newoperations()
         if export_state:

From noreply at buildbot.pypy.org  Wed Dec 17 10:37:36 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 17 Dec 2014 10:37:36 +0100 (CET)
Subject: [pypy-commit] pypy default: * rename test_disable_optimizations to
 test_zdisable_opts, so it runs last
Message-ID: <20141217093736.AC2E31C01C3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74966:62241619b8b5
Date: 2014-12-17 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/62241619b8b5/

Log:	* rename test_disable_optimizations to test_zdisable_opts, so it
	runs last
	* disable test_multilabel - it's a feature we don't use outside of
	this test, we might want to reconsider if it's even necessary
	* kill available_boxes, I don't see how they can be useful outside of
	multilabel

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -545,8 +545,8 @@
         self.optimize_loop(ops, expected)
 
 
-class TestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin):
+class XxxTestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin):
     pass
 
-class TestOptimizerRenamingBoxesLLtype(BaseTestOptimizerRenamingBoxes, LLtypeMixin):
+class XxxTestOptimizerRenamingBoxesLLtype(BaseTestOptimizerRenamingBoxes, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_disable_optimizations.py b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
rename from rpython/jit/metainterp/optimizeopt/test/test_disable_optimizations.py
rename to rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -182,8 +182,7 @@
             for box in self.inputargs:
                 self.boxes_created_this_iteration[box] = None
 
-        short_boxes = ShortBoxes(self.optimizer, inputargs,
-                                 self.boxes_created_this_iteration)
+        short_boxes = ShortBoxes(self.optimizer, inputargs)
 
         self.optimizer.clear_newoperations()
         for i in range(len(original_jump_args)):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -585,13 +585,12 @@
 
 
 class ShortBoxes(object):
-    def __init__(self, optimizer, surviving_boxes, available_boxes=None):
+    def __init__(self, optimizer, surviving_boxes):
         self.potential_ops = {}
         self.alternatives = {}
         self.synthetic = {}
         self.rename = {}
         self.optimizer = optimizer
-        self.available_boxes = available_boxes
         self.assumed_classes = {}
 
         if surviving_boxes is not None:
@@ -663,8 +662,6 @@
             return
         if box in self.short_boxes_in_production:
             raise BoxNotProducable
-        if self.available_boxes is not None and box not in self.available_boxes:
-            raise BoxNotProducable
         self.short_boxes_in_production[box] = None
 
         if box in self.potential_ops:

From noreply at buildbot.pypy.org  Wed Dec 17 13:36:36 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Dec 2014 13:36:36 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: (ltratt, arigo)
Message-ID: <20141217123636.904671D28E8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r74967:1a8e8c05fb54
Date: 2014-12-17 12:35 +0000
http://bitbucket.org/pypy/pypy/changeset/1a8e8c05fb54/

Log:	(ltratt, arigo)

	Add some jit.look_inside_iff(): so far the ll_dict_grow() call ends
	up as a residual call

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -620,8 +620,12 @@
     some += newsize >> 3
     return newsize + some
 
- at jit.dont_look_inside
+ at jit.look_inside_iff(lambda d: jit.isvirtual(d))
 def ll_dict_grow(d):
+    # note: this @jit.look_inside_iff is here to inline the three lines
+    # at the end of this function.  It's important because dicts start
+    # with a length-zero 'd.entries' which must be grown as soon as we
+    # insert an element.
     if d.num_live_items < d.num_ever_used_items // 2:
         # At least 50% of the allocated entries are dead, so perform a
         # compaction. If ll_dict_remove_deleted_items detects that over
@@ -644,6 +648,7 @@
     d.entries = newitems
     return False
 
+ at jit.dont_look_inside
 def ll_dict_remove_deleted_items(d):
     if d.num_live_items < len(d.entries) // 4:
         # At least 75% of the allocated entries are dead, so shrink the memory

From noreply at buildbot.pypy.org  Wed Dec 17 14:02:10 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 17 Dec 2014 14:02:10 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: merge default
Message-ID: <20141217130210.9BBB61C08E7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: vmprof
Changeset: r74968:060eabccb5b8
Date: 2014-12-17 14:01 +0100
http://bitbucket.org/pypy/pypy/changeset/060eabccb5b8/

Log:	merge default

diff too long, truncating to 2000 out of 2100 lines

diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -478,6 +478,29 @@
         except TypeError:
             pass
 
+    def CheckCurDescription(self):
+        self.cu.execute("select * from test")
+
+        actual = self.cu.description
+        expected = [
+            ('id', None, None, None, None, None, None),
+            ('name', None, None, None, None, None, None),
+            ('income', None, None, None, None, None, None),
+        ]
+        self.assertEqual(expected, actual)
+
+    def CheckCurDescriptionVoidStatement(self):
+        self.cu.execute("insert into test(name) values (?)", ("foo",))
+        self.assertIsNone(self.cu.description)
+
+    def CheckCurDescriptionWithoutStatement(self):
+        cu = self.cx.cursor()
+        try:
+            self.assertIsNone(cu.description)
+        finally:
+            cu.close()
+
+
 @unittest.skipUnless(threading, 'This test requires threading.')
 class ThreadTests(unittest.TestCase):
     def setUp(self):
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1175,8 +1175,9 @@
         try:
             return self.__description
         except AttributeError:
-            self.__description = self.__statement._get_description()
-            return self.__description
+            if self.__statement:
+                self.__description = self.__statement._get_description()
+                return self.__description
     description = property(__get_description)
 
     def __get_lastrowid(self):
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -125,6 +125,8 @@
         else:
             return self.space.builtin
 
+    _NO_CELLS = []
+
     @jit.unroll_safe
     def initialize_frame_scopes(self, outer_func, code):
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
@@ -143,7 +145,7 @@
         nfreevars = len(code.co_freevars)
         if not nfreevars:
             if not ncellvars:
-                self.cells = []
+                self.cells = self._NO_CELLS
                 return            # no self.cells needed - fast path
         elif outer_func is None:
             space = self.space
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -58,7 +58,7 @@
     w_globals = from_ref(space, py_frame.c_f_globals)
 
     frame = space.FrameClass(space, code, w_globals, outer_func=None)
-    frame.f_lineno = py_frame.c_f_lineno
+    frame.f_lineno = rffi.getintfield(py_frame, 'c_f_lineno')
     w_obj = space.wrap(frame)
     track_reference(space, py_obj, w_obj)
     return w_obj
diff --git a/pypy/module/micronumpy/flagsobj.py b/pypy/module/micronumpy/flagsobj.py
--- a/pypy/module/micronumpy/flagsobj.py
+++ b/pypy/module/micronumpy/flagsobj.py
@@ -5,46 +5,26 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import constants as NPY
-
+from pypy.module.micronumpy.strides import is_c_contiguous, is_f_contiguous
 
 def enable_flags(arr, flags):
     arr.flags |= flags
 
-
 def clear_flags(arr, flags):
     arr.flags &= ~flags
 
-
- at jit.unroll_safe
 def _update_contiguous_flags(arr):
-    shape = arr.shape
-    strides = arr.strides
-
-    is_c_contig = True
-    sd = arr.dtype.elsize
-    for i in range(len(shape) - 1, -1, -1):
-        dim = shape[i]
-        if strides[i] != sd:
-            is_c_contig = False
-            break
-        if dim == 0:
-            break
-        sd *= dim
+    is_c_contig = is_c_contiguous(arr)
     if is_c_contig:
         enable_flags(arr, NPY.ARRAY_C_CONTIGUOUS)
     else:
         clear_flags(arr, NPY.ARRAY_C_CONTIGUOUS)
 
-    sd = arr.dtype.elsize
-    for i in range(len(shape)):
-        dim = shape[i]
-        if strides[i] != sd:
-            clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
-            return
-        if dim == 0:
-            break
-        sd *= dim
-    enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
+    is_f_contig = is_f_contiguous(arr)
+    if is_f_contig:
+        enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
+    else:
+        clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
 
 
 class W_FlagsObject(W_Root):
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -19,7 +19,7 @@
     order_converter, shape_converter, searchside_converter
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.strides import get_shape_from_iterable, \
-    shape_agreement, shape_agreement_multiple
+    shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
 
 
 def _match_dot_shapes(space, left, right):
@@ -837,7 +837,15 @@
                 raise OperationError(space.w_ValueError, space.wrap(
                     "new type not compatible with array."))
         else:
-            if dims == 1 or impl.get_strides()[0] < impl.get_strides()[-1]:
+            if not is_c_contiguous(impl) and not is_f_contiguous(impl):
+                if old_itemsize != new_itemsize:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array."))
+                # Strides, shape does not change
+                v = impl.astype(space, dtype)
+                return wrap_impl(space, w_type, self, v) 
+            strides = impl.get_strides()
+            if dims == 1 or strides[0] <strides[-1]:
                 # Column-major, resize first dimension
                 if new_shape[0] * old_itemsize % new_itemsize != 0:
                     raise OperationError(space.w_ValueError, space.wrap(
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -429,3 +429,35 @@
                     n_old_elems_to_use *= old_shape[oldI]
     assert len(new_strides) == len(new_shape)
     return new_strides[:]
+
+ at jit.unroll_safe
+def is_c_contiguous(arr):
+    shape = arr.get_shape()
+    strides = arr.get_strides()
+    ret = True
+    sd = arr.dtype.elsize
+    for i in range(len(shape) - 1, -1, -1):
+        dim = shape[i]
+        if strides[i] != sd:
+            ret = False
+            break
+        if dim == 0:
+            break
+        sd *= dim
+    return ret
+
+ at jit.unroll_safe
+def is_f_contiguous(arr):
+    shape = arr.get_shape()
+    strides = arr.get_strides()
+    ret = True
+    sd = arr.dtype.elsize
+    for i in range(len(shape)):
+        dim = shape[i]
+        if strides[i] != sd:
+            ret = False
+            break
+        if dim == 0:
+            break
+        sd *= dim
+    return ret
diff --git a/pypy/module/micronumpy/test/test_iterators.py b/pypy/module/micronumpy/test/test_iterators.py
--- a/pypy/module/micronumpy/test/test_iterators.py
+++ b/pypy/module/micronumpy/test/test_iterators.py
@@ -13,6 +13,11 @@
         self.strides = strides
         self.start = start
 
+    def get_shape(self):
+        return self.shape
+
+    def get_strides(self):
+        return self.strides
 
 class TestIterDirect(object):
     def test_iterator_basic(self):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1807,6 +1807,17 @@
         x = array([], dtype=[('a', 'int8'), ('b', 'int8')])
         y = x.view(dtype='int16')
 
+    def test_view_of_slice(self):
+        from numpy import empty
+        x = empty([6], 'uint32')
+        x.fill(0xdeadbeef)
+        s = x[::3]
+        exc = raises(ValueError, s.view, 'uint8')
+        assert exc.value[0] == 'new type not compatible with array.'
+        s[...] = 2
+        v = s.view(x.__class__)
+        assert (v == 2).all()
+    
     def test_tolist_scalar(self):
         from numpy import dtype
         int32 = dtype('int32').type
@@ -2912,11 +2923,7 @@
         a = empty(10, dtype=[(_, int) for _ in 'abcde'])
         a.fill(123)
         for i in a:
-            import sys
-            if '__pypy__' in sys.builtin_module_names:
-                assert tuple(i) == (123,) + (0,) * 4
-            else:
-                assert tuple(i) == (123,) * 5
+            assert tuple(i) == (123,) * 5
 
         a = zeros(3, dtype=dtype(complex).newbyteorder())
         a.fill(1.5+2.5j)
@@ -3846,7 +3853,7 @@
         a = np.array([b, b, b], dtype=dt)
         assert a.shape == (3, 2)
         for i in a.flat:
-            assert tuple(i) == (True, False)
+            assert tuple(i) == (True, True)
 
         dt = np.dtype([('A', '<i8'), ('B', '<f8'), ('C', '<c16')])
         b = np.array((999999, 1e+20, 1e+20+0j), dtype=dt)
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -311,26 +311,26 @@
     BoxType = boxes.W_BoolBox
     format_code = "?"
 
-    True = BoxType(True)
-    False = BoxType(False)
+    _True = BoxType(True)
+    _False = BoxType(False)
 
     @specialize.argtype(1)
     def box(self, value):
         box = Primitive.box(self, value)
         if box.value:
-            return self.True
+            return self._True
         else:
-            return self.False
+            return self._False
 
     @specialize.argtype(1, 2)
     def box_complex(self, real, imag):
         box = Primitive.box(self, real)
         if box.value:
-            return self.True
+            return self._True
         box = Primitive.box(self, imag)
         if box.value:
-            return self.True
-        return self.False
+            return self._True
+        return self._False
 
     def coerce_subtype(self, space, w_subtype, w_item):
         # Doesn't return subclasses so it can return the constants.
@@ -1869,7 +1869,7 @@
                 items_w = space.fixedview(w_item.get_scalar_value())
             else:
                 # XXX support initializing from readable buffers
-                items_w = [w_item]
+                items_w = [w_item] * len(dtype.fields)
         else:
             items_w = [None] * len(dtype.fields)
         arr = VoidBoxStorage(dtype.elsize, dtype)
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -415,9 +415,12 @@
         self.match_descr(op.descr, exp_descr)
 
 
-    def _next_op(self, iter_ops, assert_raises=False):
+    def _next_op(self, iter_ops, assert_raises=False, ignore_ops=set()):
         try:
-            op = iter_ops.next()
+            while True:
+                op = iter_ops.next()
+                if op.name not in ignore_ops:
+                    break
         except StopIteration:
             self._assert(assert_raises, "not enough operations")
             return
@@ -454,9 +457,7 @@
         else:
             assert 0, "'{{{' not followed by '}}}'"
         while exp_ops:
-            op = self._next_op(iter_ops)
-            if op.name in ignore_ops:
-                continue
+            op = self._next_op(iter_ops, ignore_ops=ignore_ops)
             # match 'op' against any of the exp_ops; the first successful
             # match is kept, and the exp_op gets removed from the list
             for i, exp_op in enumerate(exp_ops):
@@ -491,10 +492,7 @@
                     self.match_any_order(iter_exp_ops, iter_ops, ignore_ops)
                     continue
                 else:
-                    while True:
-                        op = self._next_op(iter_ops)
-                        if op.name not in ignore_ops:
-                            break
+                    op = self._next_op(iter_ops, ignore_ops=ignore_ops)
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
                 if type(exp_op) is not str and exp_op[4] is False:    # optional operation
@@ -504,7 +502,7 @@
                 raise
         #
         # make sure we exhausted iter_ops
-        self._next_op(iter_ops, assert_raises=True)
+        self._next_op(iter_ops, assert_raises=True, ignore_ops=ignore_ops)
 
     def match(self, expected_src, ignore_ops=[]):
         def format(src, opindex=None):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -318,6 +318,16 @@
             jump(i4, descr=...)
         """
         assert self.match(loop, expected, ignore_ops=['force_token'])
+        #
+        loop = """
+            [i0]
+            i1 = int_add(i0, 1)
+            i4 = force_token()
+        """
+        expected = """
+            i1 = int_add(i0, 1)
+        """
+        assert self.match(loop, expected, ignore_ops=['force_token'])
 
     def test_match_dots_in_arguments(self):
         loop = """
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -342,9 +342,9 @@
         guard_false(i114, descr=...)
         --TICK--
         i119 = call(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=4 OS=110>)
-        raw_store(i119, 0, i112, descr=<ArrayS 2>)
-        raw_store(i119, 2, i112, descr=<ArrayS 2>)
-        raw_store(i119, 4, i112, descr=<ArrayS 2>)
+        raw_store(i119, 0, i160, descr=<ArrayS 2>)
+        raw_store(i119, 2, i160, descr=<ArrayS 2>)
+        raw_store(i119, 4, i160, descr=<ArrayS 2>)
         setfield_gc(p167, i119, descr=<FieldU pypy.module._cffi_backend.cdataobj.W_CData.inst__cdata .+>)
         i123 = arraylen_gc(p67, descr=<ArrayP .>)
         jump(..., descr=...)
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -3,6 +3,7 @@
 from __future__ import absolute_import
 import py
 
+
 class BaseTestDatetime:
     def test_repr(self):
         print datetime
@@ -210,11 +211,13 @@
             naive == aware
         assert str(e.value) == "can't compare offset-naive and offset-aware times"
 
-class TestDatetimeCPython(BaseTestDatetime):
+
+class TestDatetimeHost(BaseTestDatetime):
     def setup_class(cls):
         global datetime
         import datetime
 
+
 class TestDatetimePyPy(BaseTestDatetime):
     def setup_class(cls):
         global datetime
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -1,267 +1,285 @@
 # -*- coding: utf-8 -*-
 """Tests for _sqlite3.py"""
 
-import pytest, sys
+from __future__ import absolute_import
+import pytest
+import sys
 
-if sys.version_info < (2, 7):
-    pytest.skip("_sqlite3 requires Python 2.7")
-try:
-    import _cffi_backend
-except ImportError:
-    # On CPython, "pip install cffi".  On old PyPy's, no chance
-    pytest.skip("_sqlite3 requires _cffi_backend to be installed")
-
-from lib_pypy import _sqlite3
 
 def pytest_funcarg__con(request):
     con = _sqlite3.connect(':memory:')
     request.addfinalizer(lambda: con.close())
     return con
 
-def test_list_ddl(con):
-    """From issue996.  Mostly just looking for lack of exceptions."""
-    cursor = con.cursor()
-    cursor.execute('CREATE TABLE foo (bar INTEGER)')
-    result = list(cursor)
-    assert result == []
-    cursor.execute('INSERT INTO foo (bar) VALUES (42)')
-    result = list(cursor)
-    assert result == []
-    cursor.execute('SELECT * FROM foo')
-    result = list(cursor)
-    assert result == [(42,)]
 
-def test_connect_takes_same_positional_args_as_Connection(con):
-    from inspect import getargspec
-    clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
-    conargs = getargspec(_sqlite3.connect).args
-    assert clsargs == conargs
+class BaseTestSQLite:
+    def test_list_ddl(self, con):
+        """From issue996.  Mostly just looking for lack of exceptions."""
+        cursor = con.cursor()
+        cursor.execute('CREATE TABLE foo (bar INTEGER)')
+        result = list(cursor)
+        assert result == []
+        cursor.execute('INSERT INTO foo (bar) VALUES (42)')
+        result = list(cursor)
+        assert result == []
+        cursor.execute('SELECT * FROM foo')
+        result = list(cursor)
+        assert result == [(42,)]
 
-def test_total_changes_after_close(con):
-    con.close()
-    pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
+    def test_connect_takes_same_positional_args_as_Connection(self, con):
+        if not hasattr(_sqlite3, '_ffi'):
+            pytest.skip("only works for lib_pypy _sqlite3")
+        from inspect import getargspec
+        clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
+        conargs = getargspec(_sqlite3.connect).args
+        assert clsargs == conargs
 
-def test_connection_check_init():
-    class Connection(_sqlite3.Connection):
-        def __init__(self, name):
+    def test_total_changes_after_close(self, con):
+        con.close()
+        pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
+
+    def test_connection_check_init(self):
+        class Connection(_sqlite3.Connection):
+            def __init__(self, name):
+                pass
+
+        con = Connection(":memory:")
+        e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()")
+        assert '__init__' in e.value.message
+
+    def test_cursor_check_init(self, con):
+        class Cursor(_sqlite3.Cursor):
+            def __init__(self, name):
+                pass
+
+        cur = Cursor(con)
+        e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')")
+        assert '__init__' in e.value.message
+
+    def test_connection_after_close(self, con):
+        pytest.raises(TypeError, "con()")
+        con.close()
+        # raises ProgrammingError because should check closed before check args
+        pytest.raises(_sqlite3.ProgrammingError, "con()")
+
+    def test_cursor_iter(self, con):
+        cur = con.cursor()
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        next(cur)
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        con.commit()
+        next(cur)
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        with pytest.raises(_sqlite3.ProgrammingError):
+            cur.executemany('select 1', [])
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        cur.execute('create table test(ing)')
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        cur.execute('insert into test values(1)')
+        con.commit()
+        with pytest.raises(StopIteration):
+            next(cur)
+
+    def test_cursor_after_close(self, con):
+        cur = con.execute('select 1')
+        cur.close()
+        con.close()
+        pytest.raises(_sqlite3.ProgrammingError, "cur.close()")
+        # raises ProgrammingError because should check closed before check args
+        pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)")
+        pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)")
+
+    @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')")
+    def test_connection_del(self, tmpdir):
+        """For issue1325."""
+        import os
+        import gc
+        try:
+            import resource
+        except ImportError:
+            pytest.skip("needs resource module")
+
+        limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+        try:
+            fds = 0
+            while True:
+                fds += 1
+                resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
+                try:
+                    for p in os.pipe(): os.close(p)
+                except OSError:
+                    assert fds < 100
+                else:
+                    break
+
+            def open_many(cleanup):
+                con = []
+                for i in range(3):
+                    con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
+                    if cleanup:
+                        con[i] = None
+                        gc.collect(); gc.collect()
+
+            pytest.raises(_sqlite3.OperationalError, open_many, False)
+            gc.collect(); gc.collect()
+            open_many(True)
+        finally:
+            resource.setrlimit(resource.RLIMIT_NOFILE, limit)
+
+    def test_on_conflict_rollback_executemany(self, con):
+        major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+        if (int(major), int(minor), int(micro)) < (3, 2, 2):
+            pytest.skip("requires sqlite3 version >= 3.2.2")
+        con.execute("create table foo(x, unique(x) on conflict rollback)")
+        con.execute("insert into foo(x) values (1)")
+        try:
+            con.executemany("insert into foo(x) values (?)", [[1]])
+        except _sqlite3.DatabaseError:
             pass
+        con.execute("insert into foo(x) values (2)")
+        try:
+            con.commit()
+        except _sqlite3.OperationalError:
+            pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
 
-    con = Connection(":memory:")
-    e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()")
-    assert '__init__' in e.value.message
+    def test_statement_arg_checking(self, con):
+        with pytest.raises(_sqlite3.Warning) as e:
+            con(123)
+        assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.'
+        with pytest.raises(ValueError) as e:
+            con.execute(123)
+        assert str(e.value) == 'operation parameter must be str or unicode'
+        with pytest.raises(ValueError) as e:
+            con.executemany(123, 123)
+        assert str(e.value) == 'operation parameter must be str or unicode'
+        with pytest.raises(ValueError) as e:
+            con.executescript(123)
+        assert str(e.value) == 'script argument must be unicode or string.'
 
-def test_cursor_check_init(con):
-    class Cursor(_sqlite3.Cursor):
-        def __init__(self, name):
-            pass
+    def test_statement_param_checking(self, con):
+        con.execute('create table foo(x)')
+        con.execute('insert into foo(x) values (?)', [2])
+        con.execute('insert into foo(x) values (?)', (2,))
+        class seq(object):
+            def __len__(self):
+                return 1
+            def __getitem__(self, key):
+                return 2
+        con.execute('insert into foo(x) values (?)', seq())
+        del seq.__len__
+        with pytest.raises(_sqlite3.ProgrammingError):
+            con.execute('insert into foo(x) values (?)', seq())
+        with pytest.raises(_sqlite3.ProgrammingError):
+            con.execute('insert into foo(x) values (?)', {2:2})
+        with pytest.raises(ValueError) as e:
+            con.execute('insert into foo(x) values (?)', 2)
+        assert str(e.value) == 'parameters are of unsupported type'
 
-    cur = Cursor(con)
-    e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')")
-    assert '__init__' in e.value.message
+    def test_explicit_begin(self, con):
+        con.execute('BEGIN')
+        con.execute('BEGIN ')
+        con.execute('BEGIN')
+        con.commit()
+        con.execute('BEGIN')
+        con.commit()
 
-def test_connection_after_close(con):
-    pytest.raises(TypeError, "con()")
-    con.close()
-    # raises ProgrammingError because should check closed before check args
-    pytest.raises(_sqlite3.ProgrammingError, "con()")
+    def test_row_factory_use(self, con):
+        con.row_factory = 42
+        con.execute('select 1')
 
-def test_cursor_iter(con):
-    cur = con.cursor()
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_returning_blob_must_own_memory(self, con):
+        import gc
+        con.create_function("returnblob", 0, lambda: buffer("blob"))
+        cur = con.execute("select returnblob()")
+        val = cur.fetchone()[0]
+        for i in range(5):
+            gc.collect()
+            got = (val[0], val[1], val[2], val[3])
+            assert got == ('b', 'l', 'o', 'b')
+        # in theory 'val' should be a read-write buffer
+        # but it's not right now
+        if not hasattr(_sqlite3, '_ffi'):
+            val[1] = 'X'
+            got = (val[0], val[1], val[2], val[3])
+            assert got == ('b', 'X', 'o', 'b')
 
-    cur.execute('select 1')
-    next(cur)
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_description_after_fetchall(self, con):
+        cur = con.cursor()
+        assert cur.description is None
+        cur.execute("select 42").fetchall()
+        assert cur.description is not None
 
-    cur.execute('select 1')
-    con.commit()
-    next(cur)
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_executemany_lastrowid(self, con):
+        cur = con.cursor()
+        cur.execute("create table test(a)")
+        cur.executemany("insert into test values (?)", [[1], [2], [3]])
+        assert cur.lastrowid is None
 
-    with pytest.raises(_sqlite3.ProgrammingError):
-        cur.executemany('select 1', [])
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_authorizer_bad_value(self, con):
+        def authorizer_cb(action, arg1, arg2, dbname, source):
+            return 42
+        con.set_authorizer(authorizer_cb)
+        with pytest.raises(_sqlite3.OperationalError) as e:
+            con.execute('select 123')
+        major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+        if (int(major), int(minor), int(micro)) >= (3, 6, 14):
+            assert str(e.value) == 'authorizer malfunction'
+        else:
+            assert str(e.value) == \
+                ("illegal return value (1) from the authorization function - "
+                 "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
 
-    cur.execute('select 1')
-    cur.execute('create table test(ing)')
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_issue1573(self, con):
+        cur = con.cursor()
+        cur.execute(u'SELECT 1 as m&#233;il')
+        assert cur.description[0][0] == u"m&#233;il".encode('utf-8')
 
-    cur.execute('select 1')
-    cur.execute('insert into test values(1)')
-    con.commit()
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_adapter_exception(self, con):
+        def cast(obj):
+            raise ZeroDivisionError
 
-def test_cursor_after_close(con):
-    cur = con.execute('select 1')
-    cur.close()
-    con.close()
-    pytest.raises(_sqlite3.ProgrammingError, "cur.close()")
-    # raises ProgrammingError because should check closed before check args
-    pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)")
-    pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)")
+        _sqlite3.register_adapter(int, cast)
+        try:
+            cur = con.cursor()
+            cur.execute("select ?", (4,))
+            val = cur.fetchone()[0]
+            # Adapter error is ignored, and parameter is passed as is.
+            assert val == 4
+            assert type(val) is int
+        finally:
+            del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
 
- at pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')")
-def test_connection_del(tmpdir):
-    """For issue1325."""
-    import os
-    import gc
-    try:
-        import resource
-    except ImportError:
-        pytest.skip("needs resource module")
 
-    limit = resource.getrlimit(resource.RLIMIT_NOFILE)
-    try:
-        fds = 0
-        while True:
-            fds += 1
-            resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
-            try:
-                for p in os.pipe(): os.close(p)
-            except OSError:
-                assert fds < 100
-            else:
-                break
-        def open_many(cleanup):
-            con = []
-            for i in range(3):
-                con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
-                if cleanup:
-                    con[i] = None
-                    gc.collect(); gc.collect()
+class TestSQLiteHost(BaseTestSQLite):
+    def setup_class(cls):
+        global _sqlite3
+        import _sqlite3
 
-        pytest.raises(_sqlite3.OperationalError, open_many, False)
-        gc.collect(); gc.collect()
-        open_many(True)
-    finally:
-        resource.setrlimit(resource.RLIMIT_NOFILE, limit)
 
-def test_on_conflict_rollback_executemany(con):
-    major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
-    if (int(major), int(minor), int(micro)) < (3, 2, 2):
-        pytest.skip("requires sqlite3 version >= 3.2.2")
-    con.execute("create table foo(x, unique(x) on conflict rollback)")
-    con.execute("insert into foo(x) values (1)")
-    try:
-        con.executemany("insert into foo(x) values (?)", [[1]])
-    except _sqlite3.DatabaseError:
-        pass
-    con.execute("insert into foo(x) values (2)")
-    try:
-        con.commit()
-    except _sqlite3.OperationalError:
-        pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
+class TestSQLitePyPy(BaseTestSQLite):
+    def setup_class(cls):
+        if sys.version_info < (2, 7):
+            pytest.skip("_sqlite3 requires Python 2.7")
 
-def test_statement_arg_checking(con):
-    with pytest.raises(_sqlite3.Warning) as e:
-        con(123)
-    assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.'
-    with pytest.raises(ValueError) as e:
-        con.execute(123)
-    assert str(e.value) == 'operation parameter must be str or unicode'
-    with pytest.raises(ValueError) as e:
-        con.executemany(123, 123)
-    assert str(e.value) == 'operation parameter must be str or unicode'
-    with pytest.raises(ValueError) as e:
-        con.executescript(123)
-    assert str(e.value) == 'script argument must be unicode or string.'
+        try:
+            import _cffi_backend
+        except ImportError:
+            # On CPython, "pip install cffi".  On old PyPy's, no chance
+            pytest.skip("_sqlite3 requires _cffi_backend to be installed")
 
-def test_statement_param_checking(con):
-    con.execute('create table foo(x)')
-    con.execute('insert into foo(x) values (?)', [2])
-    con.execute('insert into foo(x) values (?)', (2,))
-    class seq(object):
-        def __len__(self):
-            return 1
-        def __getitem__(self, key):
-            return 2
-    con.execute('insert into foo(x) values (?)', seq())
-    del seq.__len__
-    with pytest.raises(_sqlite3.ProgrammingError):
-        con.execute('insert into foo(x) values (?)', seq())
-    with pytest.raises(_sqlite3.ProgrammingError):
-        con.execute('insert into foo(x) values (?)', {2:2})
-    with pytest.raises(ValueError) as e:
-        con.execute('insert into foo(x) values (?)', 2)
-    assert str(e.value) == 'parameters are of unsupported type'
-
-def test_explicit_begin(con):
-    con.execute('BEGIN')
-    con.execute('BEGIN ')
-    con.execute('BEGIN')
-    con.commit()
-    con.execute('BEGIN')
-    con.commit()
-
-def test_row_factory_use(con):
-    con.row_factory = 42
-    con.execute('select 1')
-
-def test_returning_blob_must_own_memory(con):
-    import gc
-    con.create_function("returnblob", 0, lambda: buffer("blob"))
-    cur = con.execute("select returnblob()")
-    val = cur.fetchone()[0]
-    for i in range(5):
-        gc.collect()
-        got = (val[0], val[1], val[2], val[3])
-        assert got == ('b', 'l', 'o', 'b')
-    # in theory 'val' should be a read-write buffer
-    # but it's not right now
-    pytest.skip("in theory 'val' should be a read-write buffer")
-    val[1] = 'X'
-    got = (val[0], val[1], val[2], val[3])
-    assert got == ('b', 'X', 'o', 'b')
-
-def test_description_after_fetchall(con):
-    cur = con.cursor()
-    cur.execute("select 42").fetchall()
-    assert cur.description is not None
-
-def test_executemany_lastrowid(con):
-    cur = con.cursor()
-    cur.execute("create table test(a)")
-    cur.executemany("insert into test values (?)", [[1], [2], [3]])
-    assert cur.lastrowid is None
-
-
-def test_authorizer_bad_value(con):
-    def authorizer_cb(action, arg1, arg2, dbname, source):
-        return 42
-    con.set_authorizer(authorizer_cb)
-    with pytest.raises(_sqlite3.OperationalError) as e:
-        con.execute('select 123')
-    major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
-    if (int(major), int(minor), int(micro)) >= (3, 6, 14):
-        assert str(e.value) == 'authorizer malfunction'
-    else:
-        assert str(e.value) == \
-            ("illegal return value (1) from the authorization function - "
-             "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
-
-
-def test_issue1573(con):
-    cur = con.cursor()
-    cur.execute(u'SELECT 1 as m&#233;il')
-    assert cur.description[0][0] == u"m&#233;il".encode('utf-8')
-
-def test_adapter_exception(con):
-    def cast(obj):
-        raise ZeroDivisionError
-
-    _sqlite3.register_adapter(int, cast)
-    try:
-        cur = con.cursor()
-        cur.execute("select ?", (4,))
-        val = cur.fetchone()[0]
-        # Adapter error is ignored, and parameter is passed as is.
-        assert val == 4
-        assert type(val) is int
-    finally:
-        del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
+        global _sqlite3
+        from lib_pypy import _sqlite3
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -366,8 +366,12 @@
             tup = w_self._lookup_where(name)
             return tup
         name = promote_string(name)
-        w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
-        return w_class, unwrap_cell(space, w_value)
+        tup_w = w_self._pure_lookup_where_with_method_cache(name, version_tag)
+        w_class, w_value = tup_w
+        if (space.config.objspace.std.withtypeversion and
+                isinstance(w_value, TypeCell)):
+            return w_class, w_value.w_value
+        return tup_w   # don't make a new tuple, reuse the old one
 
     def _pure_lookup_where_possibly_with_method_cache(w_self, name, version_tag):
         if w_self.space.config.objspace.std.withmethodcache:
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -19,7 +19,7 @@
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
 from rpython.annotator.argument import simple_args
-from rpython.rlib.objectmodel import r_dict, Symbolic
+from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper import extregistry
 
@@ -260,21 +260,23 @@
                     result.listdef.generalize(self.immutablevalue(e))
                 result.const_box = key
                 return result
-        elif tp is dict or tp is r_dict or tp is SomeOrderedDict.knowntype:
-            if tp is SomeOrderedDict.knowntype:
-                cls = SomeOrderedDict
-            else:
-                cls = SomeDict
+        elif (tp is dict or tp is r_dict or
+              tp is SomeOrderedDict.knowntype or tp is r_ordereddict):
             key = Constant(x)
             try:
                 return self.immutable_cache[key]
             except KeyError:
+                if tp is SomeOrderedDict.knowntype or tp is r_ordereddict:
+                    cls = SomeOrderedDict
+                else:
+                    cls = SomeDict
+                is_r_dict = issubclass(tp, r_dict)
                 result = cls(DictDef(self,
                                         s_ImpossibleValue,
                                         s_ImpossibleValue,
-                                        is_r_dict = tp is r_dict))
+                                        is_r_dict = is_r_dict))
                 self.immutable_cache[key] = result
-                if tp is r_dict:
+                if is_r_dict:
                     s_eqfn = self.immutablevalue(x.key_eq)
                     s_hashfn = self.immutablevalue(x.key_hash)
                     result.dictdef.dictkey.update_rdict_annotations(s_eqfn,
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -74,7 +74,6 @@
             # for a CALL_ASSEMBLER: record it as a potential jump.
             if descr is not original_jitcell_token:
                 original_jitcell_token.record_jump_to(descr)
-            descr.exported_state = None
             op.cleardescr()    # clear reference, mostly for tests
         elif isinstance(descr, TargetToken):
             # for a JUMP: record it as a potential jump.
@@ -84,10 +83,6 @@
             if descr.original_jitcell_token is not original_jitcell_token:
                 assert descr.original_jitcell_token is not None
                 original_jitcell_token.record_jump_to(descr.original_jitcell_token)
-            # exported_state is clear by optimizeopt when the short preamble is
-            # constrcucted. if that did not happen the label should not show up
-            # in a trace that will be used
-            assert descr.exported_state is None
             if not we_are_translated():
                 op._descr_wref = weakref.ref(op._descr)
             op.cleardescr()    # clear reference to prevent the history.Stats
@@ -133,7 +128,7 @@
                       [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
 
     try:
-        optimize_trace(metainterp_sd, part, enable_opts)
+        start_state = optimize_trace(metainterp_sd, part, enable_opts)
     except InvalidLoop:
         return None
     target_token = part.operations[0].getdescr()
@@ -146,7 +141,7 @@
     loop.quasi_immutable_deps = {}
     if part.quasi_immutable_deps:
         loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
-    while part.operations[-1].getopnum() == rop.LABEL:
+    if part.operations[-1].getopnum() == rop.LABEL:
         inliner = Inliner(inputargs, jumpargs)
         part.quasi_immutable_deps = None
         part.operations = [part.operations[-1]] + \
@@ -160,13 +155,15 @@
         jumpargs = part.operations[-1].getarglist()
 
         try:
-            optimize_trace(metainterp_sd, part, enable_opts)
+            optimize_trace(metainterp_sd, part, enable_opts,
+                           start_state=start_state)
         except InvalidLoop:
             return None
 
         loop.operations = loop.operations[:-1] + part.operations
         if part.quasi_immutable_deps:
             loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
+    assert part.operations[-1].getopnum() != rop.LABEL
 
     if not loop.quasi_immutable_deps:
         loop.quasi_immutable_deps = None
@@ -186,7 +183,7 @@
 
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,
-                    partial_trace, resumekey):
+                    partial_trace, resumekey, start_state):
     """Try to compile a new procedure by closing the current history back
     to the first operation.
     """
@@ -211,18 +208,19 @@
     orignial_label = label.clone()
     assert label.getopnum() == rop.LABEL
     try:
-        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts)
+        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
+                       start_state=start_state)
     except InvalidLoop:
         # Fall back on jumping to preamble
         target_token = label.getdescr()
         assert isinstance(target_token, TargetToken)
-        assert target_token.exported_state
         part.operations = [orignial_label] + \
                           [ResOperation(rop.JUMP, inputargs[:],
                                         None, descr=loop_jitcell_token)]
         try:
-            optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
-                           inline_short_preamble=False)
+            optimize_trace(metainterp_sd, part,
+                           jitdriver_sd.warmstate.enable_opts,
+                           inline_short_preamble=False, start_state=start_state)
         except InvalidLoop:
             return None
     assert part.operations[-1].getopnum() != rop.LABEL
@@ -791,7 +789,7 @@
     else:
         inline_short_preamble = True
     try:
-        optimize_trace(metainterp_sd, new_trace, state.enable_opts, inline_short_preamble)
+        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts, inline_short_preamble)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
@@ -807,7 +805,7 @@
         record_loop_or_bridge(metainterp_sd, new_trace)
         return target_token
     else:
-        metainterp.retrace_needed(new_trace)
+        metainterp.retrace_needed(new_trace, state)
         return None
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -617,7 +617,6 @@
         self.original_jitcell_token = None
 
         self.virtual_state = None
-        self.exported_state = None
         self.short_preamble = None
 
     def repr_of_descr(self):
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -178,13 +178,6 @@
             fail_args = ''
         return s_offset + res + op.getopname() + '(' + args + ')' + fail_args
 
-    def _log_inputarg_setup_ops(self, op):
-        target_token = op.getdescr()
-        if isinstance(target_token, TargetToken):
-            if target_token.exported_state:
-                for op in target_token.exported_state.inputarg_setup_ops:
-                    debug_print('    ' + self.repr_of_resop(op))
-
     def _log_operations(self, inputargs, operations, ops_offset):
         if not have_debug_prints():
             return
@@ -194,10 +187,10 @@
             args = ", ".join([self.repr_of_arg(arg) for arg in inputargs])
             debug_print('[' + args + ']')
         for i in range(len(operations)):
-            op = operations[i]
+            #op = operations[i]
             debug_print(self.repr_of_resop(operations[i], ops_offset))
-            if op.getopnum() == rop.LABEL:
-                self._log_inputarg_setup_ops(op)
+            #if op.getopnum() == rop.LABEL:
+            #    self._log_inputarg_setup_ops(op)
         if ops_offset and None in ops_offset:
             offset = ops_offset[None]
             debug_print("+%d: --end of the loop--" % offset)
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -47,7 +47,8 @@
 
     return optimizations, unroll
 
-def optimize_trace(metainterp_sd, loop, enable_opts, inline_short_preamble=True):
+def optimize_trace(metainterp_sd, loop, enable_opts,
+                   inline_short_preamble=True, start_state=None):
     """Optimize loop.operations to remove internal overheadish operations.
     """
 
@@ -57,7 +58,8 @@
                                                           loop.operations)
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
-            optimize_unroll(metainterp_sd, loop, optimizations, inline_short_preamble)
+            return optimize_unroll(metainterp_sd, loop, optimizations,
+                                   inline_short_preamble, start_state)
         else:
             optimizer = Optimizer(metainterp_sd, loop, optimizations)
             optimizer.propagate_all_forward()
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -42,8 +42,9 @@
             oldop = self.pure_operations.get(args, None)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 assert oldop.getopnum() == op.getopnum()
-                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
+                self.optimizer.make_equal_to(op.result,
+                                             self.getvalue(oldop.result),
+                                             True)
                 return
             else:
                 self.pure_operations[args] = op
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -32,6 +32,7 @@
                   if op.getopnum()==rop.LABEL]
         prv = 0
         last_label = []
+        state = None
         for nxt in labels + [len(loop.operations)]:
             assert prv != nxt
             operations = last_label + loop.operations[prv:nxt]
@@ -44,7 +45,7 @@
             part.operations = operations
 
             self.add_guard_future_condition(part)
-            self._do_optimize_loop(part, None)
+            state = self._do_optimize_loop(part, None, state)
             if part.operations[-1].getopnum() == rop.LABEL:
                 last_label = [part.operations.pop()]
             else:
@@ -496,7 +497,7 @@
 
 class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel):
 
-    def _do_optimize_loop(self, loop, call_pure_results):
+    def _do_optimize_loop(self, loop, call_pure_results, state):
         from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
         from rpython.jit.metainterp.optimizeopt.util import args_dict
         from rpython.jit.metainterp.optimizeopt.pure import OptPure
@@ -504,7 +505,7 @@
         self.loop = loop
         loop.call_pure_results = args_dict()
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True)
+        return optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True, state)
 
     def test_optimizer_renaming_boxes1(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -390,7 +390,7 @@
         assert equaloplists(optimized.operations,
                             expected.operations, False, remap, text_right)
 
-    def _do_optimize_loop(self, loop, call_pure_results):
+    def _do_optimize_loop(self, loop, call_pure_results, start_state=None):
         from rpython.jit.metainterp.optimizeopt import optimize_trace
         from rpython.jit.metainterp.optimizeopt.util import args_dict
 
@@ -405,7 +405,8 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        optimize_trace(metainterp_sd, loop, self.enable_opts)
+        return optimize_trace(metainterp_sd, loop, self.enable_opts,
+                              start_state=start_state)
 
     def unroll_and_optimize(self, loop, call_pure_results=None):
         self.add_guard_future_condition(loop)
@@ -425,7 +426,7 @@
         preamble.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(token))] + \
                               operations +  \
                               [ResOperation(rop.LABEL, jump_args, None, descr=token)]
-        self._do_optimize_loop(preamble, call_pure_results)
+        start_state = self._do_optimize_loop(preamble, call_pure_results)
 
         assert preamble.operations[-1].getopnum() == rop.LABEL
 
@@ -439,7 +440,7 @@
         assert loop.operations[0].getopnum() == rop.LABEL
         loop.inputargs = loop.operations[0].getarglist()
 
-        self._do_optimize_loop(loop, call_pure_results)
+        self._do_optimize_loop(loop, call_pure_results, start_state)
         extra_same_as = []
         while loop.operations[0].getopnum() != rop.LABEL:
             extra_same_as.append(loop.operations[0])
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -15,10 +15,11 @@
 
 # FIXME: Introduce some VirtualOptimizer super class instead
 
-def optimize_unroll(metainterp_sd, loop, optimizations, inline_short_preamble=True):
+def optimize_unroll(metainterp_sd, loop, optimizations,
+                    inline_short_preamble=True, start_state=None):
     opt = UnrollOptimizer(metainterp_sd, loop, optimizations)
     opt.inline_short_preamble = inline_short_preamble
-    opt.propagate_all_forward()
+    return opt.propagate_all_forward(start_state)
 
 
 class UnrollableOptimizer(Optimizer):
@@ -69,7 +70,7 @@
         prev = self.fix_snapshot(jump_args, snapshot.prev)
         return Snapshot(prev, new_snapshot_args)
 
-    def propagate_all_forward(self):
+    def propagate_all_forward(self, starting_state):
         loop = self.optimizer.loop
         self.optimizer.clear_newoperations()
 
@@ -94,7 +95,7 @@
         else:
             jumpop = None
 
-        self.import_state(start_label)
+        self.import_state(start_label, starting_state)
         self.optimizer.propagate_all_forward(clear=False)
 
         if not jumpop:
@@ -147,8 +148,9 @@
         KillHugeIntBounds(self.optimizer).apply()
 
         loop.operations = self.optimizer.get_newoperations()
-        self.export_state(stop_label)
+        final_state = self.export_state(stop_label)
         loop.operations.append(stop_label)
+        return final_state
 
     def jump_to_start_label(self, start_label, stop_label):
         if not start_label or not stop_label:
@@ -202,10 +204,9 @@
                 box = op.result
                 exported_values[box] = self.optimizer.getvalue(box)
 
-        target_token.exported_state = ExportedState(short_boxes, inputarg_setup_ops,
-                                                    exported_values)
+        return ExportedState(short_boxes, inputarg_setup_ops, exported_values)
 
-    def import_state(self, targetop):
+    def import_state(self, targetop, exported_state):
         if not targetop: # Trace did not start with a label
             self.inputargs = self.optimizer.loop.inputargs
             self.short = None
@@ -215,7 +216,6 @@
         self.inputargs = targetop.getarglist()
         target_token = targetop.getdescr()
         assert isinstance(target_token, TargetToken)
-        exported_state = target_token.exported_state
         if not exported_state:
             # No state exported, construct one without virtuals
             self.short = None
@@ -413,7 +413,6 @@
             if op.result:
                 op.result.forget_value()
         target_token.short_preamble = self.short
-        target_token.exported_state = None
 
     def ensure_short_op_emitted(self, op, optimizer, seen):
         if op is None:
@@ -561,7 +560,9 @@
 
             try:
                 # NB: the short_preamble ends with a jump
-                self._inline_short_preamble(target.short_preamble, inliner, patchguardop, target.assumed_classes)
+                self._inline_short_preamble(target.short_preamble, inliner,
+                                            patchguardop,
+                                            target.assumed_classes)
             except InvalidLoop:
                 #debug_print("Inlining failed unexpectedly",
                 #            "jumping to preamble instead")
@@ -572,7 +573,8 @@
         debug_stop('jit-log-virtualstate')
         return False
 
-    def _inline_short_preamble(self, short_preamble, inliner, patchguardop, assumed_classes):
+    def _inline_short_preamble(self, short_preamble, inliner, patchguardop,
+                               assumed_classes):
         i = 1
         # XXX this is intentiontal :-(. short_preamble can change during the
         # loop in some cases
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -585,13 +585,13 @@
 
 
 class ShortBoxes(object):
-    def __init__(self, optimizer, surviving_boxes, availible_boxes=None):
+    def __init__(self, optimizer, surviving_boxes, available_boxes=None):
         self.potential_ops = {}
         self.alternatives = {}
         self.synthetic = {}
         self.rename = {}
         self.optimizer = optimizer
-        self.availible_boxes = availible_boxes
+        self.available_boxes = available_boxes
         self.assumed_classes = {}
 
         if surviving_boxes is not None:
@@ -663,7 +663,7 @@
             return
         if box in self.short_boxes_in_production:
             raise BoxNotProducable
-        if self.availible_boxes is not None and box not in self.availible_boxes:
+        if self.available_boxes is not None and box not in self.available_boxes:
             raise BoxNotProducable
         self.short_boxes_in_production[box] = None
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1730,6 +1730,7 @@
 class MetaInterp(object):
     portal_call_depth = 0
     cancel_count = 0
+    exported_state = None
 
     def __init__(self, staticdata, jitdriver_sd):
         self.staticdata = staticdata
@@ -1750,9 +1751,10 @@
         self.call_ids = []
         self.current_call_id = 0
 
-    def retrace_needed(self, trace):
+    def retrace_needed(self, trace, exported_state):
         self.partial_trace = trace
         self.retracing_from = len(self.history.operations) - 1
+        self.exported_state = exported_state
         self.heapcache.reset()
 
 
@@ -2249,7 +2251,9 @@
                         raise SwitchToBlackhole(Counters.ABORT_BAD_LOOP) # For now
                 # Found!  Compile it as a loop.
                 # raises in case it works -- which is the common case
-                self.compile_loop(original_boxes, live_arg_boxes, start)
+                self.compile_loop(original_boxes, live_arg_boxes, start,
+                                  exported_state=self.exported_state)
+                self.exported_state = None
                 # creation of the loop was cancelled!
                 self.cancel_count += 1
                 if self.staticdata.warmrunnerdesc:
@@ -2363,7 +2367,7 @@
         return token
 
     def compile_loop(self, original_boxes, live_arg_boxes, start,
-                     try_disabling_unroll=False):
+                     try_disabling_unroll=False, exported_state=None):
         num_green_args = self.jitdriver_sd.num_green_args
         greenkey = original_boxes[:num_green_args]
         if not self.partial_trace:
@@ -2377,7 +2381,8 @@
                                                    original_boxes[num_green_args:],
                                                    live_arg_boxes[num_green_args:],
                                                    self.partial_trace,
-                                                   self.resumekey)
+                                                   self.resumekey,
+                                                   exported_state)
         else:
             target_token = compile.compile_loop(self, greenkey, start,
                                                 original_boxes[num_green_args:],
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -329,6 +329,18 @@
         res = self.interpret(g, [3])
         assert res == 42     # "did not crash"
 
+    def test_prepare_dict_update_2(self):
+        try:
+            from collections import OrderedDict
+        except ImportError:     # Python 2.6
+            py.test.skip("requires collections.OrderedDict")
+        def g(n):
+            d = OrderedDict()
+            prepare_dict_update(d, n)
+            return 42
+        res = self.interpret(g, [3])
+        assert res == 42     # "did not crash"
+
     def test_compute_hash(self):
         class Foo(object):
             pass
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -669,6 +669,7 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
+DictRepr.ll_newdict = staticmethod(ll_newdict)
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -692,26 +693,6 @@
     pass
 
 
-def rtype_r_dict(hop, i_force_non_null=None):
-    r_dict = hop.r_result
-    if not r_dict.custom_eq_hash:
-        raise TyperError("r_dict() call does not return an r_dict instance")
-    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
-    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
-    if i_force_non_null is not None:
-        assert i_force_non_null == 2
-        hop.inputarg(lltype.Void, arg=2)
-    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    hop.exception_cannot_occur()
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
-    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
-        hop.genop('setfield', [v_result, cname, v_eqfn])
-    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
-        hop.genop('setfield', [v_result, cname, v_hashfn])
-    return v_result
-
 # ____________________________________________________________
 #
 #  Iteration.
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -156,7 +156,7 @@
 
     def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
                  custom_eq_hash=None, force_non_null=False):
-        assert not force_non_null
+        #assert not force_non_null
         self.rtyper = rtyper
         self.finalized = False
         self.DICT = lltype.GcForwardReference()
@@ -232,6 +232,7 @@
                 raise TypeError("No prebuilt dicts of address keys")
             r_value = self.value_repr
             if isinstance(dictobj, objectmodel.r_dict):
+
                 if self.r_rdict_eqfn.lowleveltype != lltype.Void:
                     l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq)
                     l_dict.fnkeyeq = l_fn
@@ -289,6 +290,11 @@
         hop.exception_cannot_occur()
         return hop.gendirectcall(ll_dict_update, v_dic1, v_dic2)
 
+    def rtype_method__prepare_dict_update(self, hop):
+        v_dict, v_num = hop.inputargs(self, lltype.Signed)
+        hop.exception_cannot_occur()
+        hop.gendirectcall(ll_prepare_dict_update, v_dict, v_num)
+
     def _rtype_method_kvi(self, hop, ll_func):
         v_dic, = hop.inputargs(self)
         r_list = hop.r_result
@@ -594,10 +600,12 @@
     else:
         newitems = d.entries
     #
-    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
+    ENTRIES = lltype.typeOf(d).TO.entries.TO
+    ENTRY = ENTRIES.OF
     isrc = 0
     idst = 0
-    while isrc < len(d.entries):
+    isrclimit = d.num_used_items
+    while isrc < isrclimit:
         if d.entries.valid(isrc):
             src = d.entries[isrc]
             dst = newitems[idst]
@@ -610,9 +618,21 @@
                 dst.f_valid = True
             idst += 1
         isrc += 1
-    d.entries = newitems
     assert d.num_items == idst
     d.num_used_items = idst
+    if ((ENTRIES.must_clear_key or ENTRIES.must_clear_value) and
+            d.entries == newitems):
+        # must clear the extra entries: they may contain valid pointers
+        # which would create a temporary memory leak
+        while idst < isrclimit:
+            entry = newitems[idst]
+            if ENTRIES.must_clear_key:
+                entry.key = lltype.nullptr(ENTRY.key.TO)
+            if ENTRIES.must_clear_value:
+                entry.value = lltype.nullptr(ENTRY.value.TO)
+            idst += 1
+    else:
+        d.entries = newitems
 
     ll_dict_reindex(d, _ll_len_of_d_indexes(d))
 
@@ -653,11 +673,14 @@
     # make a 'new_size' estimate and shrink it if there are many
     # deleted entry markers.  See CPython for why it is a good idea to
     # quadruple the dictionary size as long as it's not too big.
-    num_items = d.num_items
-    if num_items > 50000:
-        new_estimate = num_items * 2
-    else:
-        new_estimate = num_items * 4
+    # (Quadrupling comes from '(d.num_items + d.num_items + 1) * 2'
+    # as long as num_items is not too large.)
+    num_extra = min(d.num_items + 1, 30000)
+    _ll_dict_resize_to(d, num_extra)
+ll_dict_resize.oopspec = 'odict.resize(d)'
+
+def _ll_dict_resize_to(d, num_extra):
+    new_estimate = (d.num_items + num_extra) * 2
     new_size = DICT_INITSIZE
     while new_size <= new_estimate:
         new_size *= 2
@@ -666,7 +689,6 @@
         ll_dict_remove_deleted_items(d)
     else:
         ll_dict_reindex(d, new_size)
-ll_dict_resize.oopspec = 'odict.resize(d)'
 
 def ll_dict_reindex(d, new_size):
     ll_malloc_indexes_and_choose_lookup(d, new_size)
@@ -831,6 +853,7 @@
     d.num_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
+OrderedDictRepr.ll_newdict = staticmethod(ll_newdict)
 
 def ll_newdict_size(DICT, orig_length_estimate):
     length_estimate = (orig_length_estimate // 2) * 3
@@ -856,23 +879,6 @@
     pass
 
 
-def rtype_r_dict(hop):
-    r_dict = hop.r_result
-    if not r_dict.custom_eq_hash:
-        raise TyperError("r_dict() call does not return an r_dict instance")
-    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
-    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
-    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    hop.exception_cannot_occur()
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
-    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
-        hop.genop('setfield', [v_result, cname, v_eqfn])
-    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
-        hop.genop('setfield', [v_result, cname, v_hashfn])
-    return v_result
-
 # ____________________________________________________________
 #
 #  Iteration.
@@ -981,6 +987,9 @@
 ll_dict_clear.oopspec = 'odict.clear(d)'
 
 def ll_dict_update(dic1, dic2):
+    if dic1 == dic2:
+        return
+    ll_prepare_dict_update(dic1, dic2.num_items)
     i = 0
     while i < dic2.num_used_items:
         entries = dic2.entries
@@ -994,6 +1003,22 @@
         i += 1
 ll_dict_update.oopspec = 'odict.update(dic1, dic2)'
 
+def ll_prepare_dict_update(d, num_extra):
+    # Prescale 'd' for 'num_extra' items, assuming that most items don't
+    # collide.  If this assumption is false, 'd' becomes too large by at
+    # most 'num_extra'.  The logic is based on:
+    #      (d.resize_counter - 1) // 3 = room left in d
+    #  so, if num_extra == 1, we need d.resize_counter > 3
+    #      if num_extra == 2, we need d.resize_counter > 6  etc.
+    # Note however a further hack: if num_extra <= d.num_items,
+    # we avoid calling _ll_dict_resize_to here.  This is to handle
+    # the case where dict.update() actually has a lot of collisions.
+    # If num_extra is much greater than d.num_items the conditional_call
+    # will trigger anyway, which is really the goal.
+    x = num_extra - d.num_items
+    jit.conditional_call(d.resize_counter <= x * 3,
+                         _ll_dict_resize_to, d, num_extra)
+
 # this is an implementation of keys(), values() and items()
 # in a single function.
 # note that by specialization on func, three different
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -724,15 +724,16 @@
     raise TyperError("hasattr is only suported on a constant")
 
 @typer_for(annmodel.SomeOrderedDict.knowntype)
+ at typer_for(objectmodel.r_dict)
 @typer_for(objectmodel.r_ordereddict)
-def rtype_ordered_dict(hop):
-    from rpython.rtyper.lltypesystem.rordereddict import ll_newdict
-
+def rtype_dict_constructor(hop, i_force_non_null=None):
+    # 'i_force_non_null' is ignored here; if it has any effect, it
+    # has already been applied to 'hop.r_result'
     hop.exception_cannot_occur()
     r_dict = hop.r_result
     cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
-    if hasattr(r_dict, 'r_dict_eqfn'):
+    v_result = hop.gendirectcall(r_dict.ll_newdict, cDICT)
+    if r_dict.custom_eq_hash:
         v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
         v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
         if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
@@ -743,9 +744,6 @@
             hop.genop('setfield', [v_result, cname, v_hashfn])
     return v_result
 
-from rpython.rtyper.lltypesystem.rdict import rtype_r_dict
-typer_for(objectmodel.r_dict)(rtype_r_dict)
-
 # _________________________________________________________________
 # weakrefs
 
diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py
--- a/rpython/rtyper/rdict.py
+++ b/rpython/rtyper/rdict.py
@@ -57,11 +57,10 @@
 
 
 def rtype_newdict(hop):
-    from rpython.rtyper.lltypesystem.rdict import ll_newdict
     hop.inputargs()    # no arguments expected
     r_dict = hop.r_result
     cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    v_result = hop.gendirectcall(r_dict.ll_newdict, cDICT)
     return v_result
 
 
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -878,6 +878,81 @@
         res = self.interpret(func, [])
         assert lltype.typeOf(res.item0) == lltype.typeOf(res.item1)
 
+    def test_r_dict(self):
+        class FooError(Exception):
+            pass
+        def myeq(n, m):
+            return n == m
+        def myhash(n):
+            if n < 0:
+                raise FooError
+            return -n
+        def f(n):
+            d = self.new_r_dict(myeq, myhash)
+            for i in range(10):
+                d[i] = i*i
+            try:
+                value1 = d[n]
+            except FooError:
+                value1 = 99
+            try:
+                value2 = n in d
+            except FooError:
+                value2 = 99
+            try:
+                value3 = d[-n]
+            except FooError:
+                value3 = 99
+            try:
+                value4 = (-n) in d
+            except FooError:
+                value4 = 99
+            return (value1 * 1000000 +
+                    value2 * 10000 +
+                    value3 * 100 +
+                    value4)
+        res = self.interpret(f, [5])
+        assert res == 25019999
+
+    def test_r_dict_popitem_hash(self):
+        def deq(n, m):
+            return n == m
+        def dhash(n):
+            return ~n
+        def func():
+            d = self.new_r_dict(deq, dhash)
+            d[5] = 2
+            d[6] = 3
+            k1, v1 = d.popitem()
+            assert len(d) == 1
+            k2, v2 = d.popitem()
+            try:
+                d.popitem()
+            except KeyError:
+                pass
+            else:
+                assert 0, "should have raised KeyError"
+            assert len(d) == 0
+            return k1*1000 + v1*100 + k2*10 + v2
+
+        res = self.interpret(func, [])
+        assert res in [5263, 6352]
+
+    def test_prebuilt_r_dict(self):
+        def deq(n, m):
+            return (n & 3) == (m & 3)
+        def dhash(n):
+            return n & 3
+        d = self.new_r_dict(deq, dhash)
+        d[0x123] = "abcd"
+        d[0x231] = "efgh"
+        def func():
+            return d[0x348973] + d[0x12981]
+
+        res = self.interpret(func, [])
+        res = self.ll_to_string(res)
+        assert res == "abcdefgh"
+
 
 class TestRDict(BaseTestRDict):
     @staticmethod
@@ -888,6 +963,10 @@
     def newdict2():
         return {}
 
+    @staticmethod
+    def new_r_dict(myeq, myhash):
+        return r_dict(myeq, myhash)
+
     def test_two_dicts_with_different_value_types(self):
         def func(i):
             d1 = {}
@@ -1043,66 +1122,6 @@
         assert lltype.typeOf(res.item1) == lltype.typeOf(res.item2)
         assert lltype.typeOf(res.item1) == lltype.typeOf(res.item3)
 
-    def test_r_dict(self):
-        class FooError(Exception):
-            pass
-        def myeq(n, m):
-            return n == m
-        def myhash(n):
-            if n < 0:
-                raise FooError
-            return -n
-        def f(n):
-            d = r_dict(myeq, myhash)
-            for i in range(10):
-                d[i] = i*i
-            try:
-                value1 = d[n]
-            except FooError:
-                value1 = 99
-            try:
-                value2 = n in d
-            except FooError:
-                value2 = 99
-            try:
-                value3 = d[-n]
-            except FooError:
-                value3 = 99
-            try:
-                value4 = (-n) in d
-            except FooError:
-                value4 = 99
-            return (value1 * 1000000 +
-                    value2 * 10000 +
-                    value3 * 100 +
-                    value4)
-        res = self.interpret(f, [5])
-        assert res == 25019999
-
-    def test_dict_popitem_hash(self):
-        def deq(n, m):
-            return n == m
-        def dhash(n):
-            return ~n
-        def func():
-            d = r_dict(deq, dhash)
-            d[5] = 2
-            d[6] = 3
-            k1, v1 = d.popitem()
-            assert len(d) == 1
-            k2, v2 = d.popitem()
-            try:
-                d.popitem()
-            except KeyError:
-                pass
-            else:
-                assert 0, "should have raised KeyError"
-            assert len(d) == 0
-            return k1*1000 + v1*100 + k2*10 + v2
-
-        res = self.interpret(func, [])
-        assert res in [5263, 6352]
-
     def test_nonnull_hint(self):
         def eq(a, b):
             return a == b
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -251,6 +251,16 @@
         assert rordereddict.ll_dict_pop_default(ll_d, llstr("k"), 40) == 40
         assert rordereddict.ll_dict_pop_default(ll_d, llstr("j"), 39) == 39
 
+    def test_bug_remove_deleted_items(self):
+        DICT = self._get_str_dict()
+        ll_d = rordereddict.ll_newdict(DICT)
+        for i in range(15):
+            rordereddict.ll_dict_setitem(ll_d, llstr(chr(i)), 5)
+        for i in range(15):
+            rordereddict.ll_dict_delitem(ll_d, llstr(chr(i)))
+        rordereddict.ll_prepare_dict_update(ll_d, 7)
+        # used to get UninitializedMemoryAccess
+
 class TestRDictDirectDummyKey(TestRDictDirect):
     class dummykeyobj:
         ll_dummy_value = llstr("dupa")
@@ -268,6 +278,10 @@
     def newdict2():
         return OrderedDict()
 
+    @staticmethod
+    def new_r_dict(myeq, myhash):
+        return objectmodel.r_ordereddict(myeq, myhash)
+
     def test_two_dicts_with_different_value_types(self):
         def func(i):
             d1 = OrderedDict()
@@ -285,62 +299,124 @@
         py.test.skip("I don't want to edit this file on two branches")
 
 
-    def test_r_dict(self):
-        class FooError(Exception):
-            pass
-        def myeq(n, m):
-            return n == m
-        def myhash(n):
-            if n < 0:
-                raise FooError
-            return -n
-        def f(n):
-            d = objectmodel.r_ordereddict(myeq, myhash)
-            for i in range(10):
-                d[i] = i*i
-            try:
-                value1 = d[n]
-            except FooError:
-                value1 = 99
-            try:
-                value2 = n in d
-            except FooError:
-                value2 = 99
-            try:
-                value3 = d[-n]
-            except FooError:
-                value3 = 99
-            try:
-                value4 = (-n) in d
-            except FooError:
-                value4 = 99
-            return (value1 * 1000000 +
-                    value2 * 10000 +
-                    value3 * 100 +
-                    value4)
-        res = self.interpret(f, [5])
-        assert res == 25019999
+class TestStress:
 
-    def test_dict_popitem_hash(self):
-        def deq(n, m):
-            return n == m
-        def dhash(n):
-            return ~n
-        def func():
-            d = objectmodel.r_ordereddict(deq, dhash)
-            d[5] = 2
-            d[6] = 3
-            k1, v1 = d.popitem()
-            assert len(d) == 1
-            k2, v2 = d.popitem()
-            try:
-                d.popitem()
-            except KeyError:
-                pass
+    def test_stress(self):
+        from rpython.annotator.dictdef import DictKey, DictValue
+        from rpython.annotator import model as annmodel
+        from rpython.rtyper import rint
+        from rpython.rtyper.test.test_rdict import not_really_random
+        rodct = rordereddict
+        dictrepr = rodct.OrderedDictRepr(
+                                  None, rint.signed_repr, rint.signed_repr,
+                                  DictKey(None, annmodel.SomeInteger()),
+                                  DictValue(None, annmodel.SomeInteger()))
+        dictrepr.setup()
+        l_dict = rodct.ll_newdict(dictrepr.DICT)
+        referencetable = [None] * 400
+        referencelength = 0
+        value = 0
+
+        def complete_check():
+            for n, refvalue in zip(range(len(referencetable)), referencetable):
+                try:
+                    gotvalue = rodct.ll_dict_getitem(l_dict, n)
+                except KeyError:
+                    assert refvalue is None
+                else:
+                    assert gotvalue == refvalue

From noreply at buildbot.pypy.org  Wed Dec 17 14:02:11 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 17 Dec 2014 14:02:11 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: merge
Message-ID: <20141217130211.DF23F1C08E7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: vmprof
Changeset: r74969:2261c6164873
Date: 2014-12-17 14:01 +0100
http://bitbucket.org/pypy/pypy/changeset/2261c6164873/

Log:	merge

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -100,6 +100,10 @@
         self._vmprof_virtual_ip = _vmprof.get_next_virtual_IP()
         self._vmprof_registered = 0
 
+# avoid rtyper warnings
+PyCode._vmprof_virtual_ip = 0
+PyCode._vmprof_registered = 0
+
 
 
 def get_virtual_ip(gc_frame):

From noreply at buildbot.pypy.org  Wed Dec 17 14:59:20 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Dec 2014 14:59:20 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: remove dead lines
Message-ID: <20141217135920.DD43A1D38C9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r74970:f3acf053a316
Date: 2014-12-17 14:59 +0100
http://bitbucket.org/pypy/pypy/changeset/f3acf053a316/

Log:	remove dead lines

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -42,7 +42,6 @@
 #
 
 def ll_call_lookup_function(d, key, hash, flag):
-    DICT = lltype.typeOf(d).TO
     fun = d.lookup_function_no
     if fun == FUNC_BYTE:
         return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
@@ -451,7 +450,6 @@
         rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_LONG, d.indexes))
 
 def ll_call_insert_clean_function(d, hash, i):
-    DICT = lltype.typeOf(d).TO
     if d.lookup_function_no == FUNC_BYTE:
         ll_dict_store_clean(d, hash, i, TYPE_BYTE)
     elif d.lookup_function_no == FUNC_SHORT:
@@ -464,7 +462,6 @@
         assert False
 
 def ll_call_delete_by_entry_index(d, hash, i):
-    DICT = lltype.typeOf(d).TO
     if d.lookup_function_no == FUNC_BYTE:
         ll_dict_delete_by_entry_index(d, hash, i, TYPE_BYTE)
     elif d.lookup_function_no == FUNC_SHORT:

From noreply at buildbot.pypy.org  Wed Dec 17 16:09:25 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Dec 2014 16:09:25 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: (ltratt, arigo)
Message-ID: <20141217150925.17BCC1C04B4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r74971:7724bdb58abf
Date: 2014-12-17 15:09 +0000
http://bitbucket.org/pypy/pypy/changeset/7724bdb58abf/

Log:	(ltratt, arigo)

	The previous check might have been correct, but we couldn't work out
	why. Rewrite it and add many extra asserts. (This is also needed
	because _ll_len_of_d_indexes() cannot be called in possibly-jitted
	code.)

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -633,11 +633,29 @@
 
     new_allocated = _overallocate_entries_len(len(d.entries))
 
-    # Detect an obscure case where the indexes numeric type is too
-    # small to store all the entry indexes
-    if (max(128, _ll_len_of_d_indexes(d)) - new_allocated
-                   < MIN_INDEXES_MINUS_ENTRIES):
+    # Detect a relatively rare case where the indexes numeric type is too
+    # small to store all the entry indexes: there would be 'new_allocated'
+    # entries, which may in corner cases be larger than 253 even though we
+    # have single bytes in 'd.indexes' (and the same for the larger
+    # boundaries).  The 'd.indexes' hashtable is never more than 2/3rd
+    # full, so we know that 'd.num_live_items' should be at most 2/3 * 256
+    # (or 65536 or etc.) so after the ll_dict_remove_deleted_items() below
+    # at least 1/3rd items in 'd.entries' are free.
+    fun = d.lookup_function_no
+    toobig = False
+    if fun == FUNC_BYTE:
+        assert d.num_live_items < ((1 << 8) - MIN_INDEXES_MINUS_ENTRIES)
+        toobig = new_allocated > ((1 << 8) - MIN_INDEXES_MINUS_ENTRIES)
+    elif fun == FUNC_SHORT:
+        assert d.num_live_items < ((1 << 16) - MIN_INDEXES_MINUS_ENTRIES)
+        toobig = new_allocated > ((1 << 16) - MIN_INDEXES_MINUS_ENTRIES)
+    elif IS_64BIT and fun == FUNC_INT:
+        assert d.num_live_items < ((1 << 32) - MIN_INDEXES_MINUS_ENTRIES)
+        toobig = new_allocated > ((1 << 32) - MIN_INDEXES_MINUS_ENTRIES)
+    #
+    if toobig:
         ll_dict_remove_deleted_items(d)
+        assert d.num_live_items == d.num_ever_used_items
         return True
 
     newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:25 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:25 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Experimental libgccjit backend
Message-ID: <20141217164825.7CF321C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74972:3096e72141bd
Date: 2014-12-17 11:37 -0500
http://bitbucket.org/pypy/pypy/changeset/3096e72141bd/

Log:	Experimental libgccjit backend


From noreply at buildbot.pypy.org  Wed Dec 17 17:48:31 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:31 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Reduce debug spew
Message-ID: <20141217164831.6BF401C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74977:4838a68bebba
Date: 2014-12-15 13:28 -0500
http://bitbucket.org/pypy/pypy/changeset/4838a68bebba/

Log:	Reduce debug spew

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -59,6 +59,8 @@
         print('assemble_loop')
         clt = CompiledLoopToken(self.cpu, looptoken.number)
         print(clt)
+        looptoken.compiled_loop_token = clt
+        clt._debug_nbargs = len(inputargs)
 
         # Make function:
         self.lvalue_for_box = {}
@@ -93,11 +95,13 @@
 
         self.b_current = self.lib.gcc_jit_function_new_block(self.fn, NULL)
 
-        for op in operations:       
+        for op in operations:
+            """
             print(op)
             print(type(op))
             print(dir(op))
             print(repr(op.getopname()))
+            """
             # Add a comment describing this ResOperation
             comment_text = str2charp(str(op))
             self.lib.gcc_jit_block_add_comment(self.b_current,
@@ -114,15 +118,24 @@
         if not jit_result:
             # FIXME: get error from context
             raise Exception("jit_result is NULL")
-        raise foo
+
+        fn_name = str2charp(loopname)
+        fn_ptr = self.lib.gcc_jit_result_get_code(jit_result, fn_name)
+        free_charp(fn_name)
+
+        looptoken._ll_function_addr = fn_ptr
+
+        # FIXME: this leaks the gcc_jit_result
 
     def expr_to_rvalue(self, expr):
+        """
         print('expr_to_rvalue')
         print(' %s' % expr)
         print(' %r' % expr)
         print(' %s' % type(expr))
         print(' %s' % dir(expr))
         print(' %s' % expr.__dict__)
+        """
 
         if isinstance(expr, BoxInt):
             return self.lib.gcc_jit_lvalue_as_rvalue(self.get_box_as_lvalue(expr))
@@ -132,6 +145,7 @@
             return self.lib.gcc_jit_context_new_rvalue_from_int(self.ctxt,
                                                                 self.t_int,
                                                                 r_int(expr.value))
+        raise ValueError('unhandled expr: %s' % expr)
 
     def get_box_as_lvalue(self, box):
         if box not in self.lvalue_for_box:
@@ -145,24 +159,27 @@
         return self.lvalue_for_box[box]
 
     def expr_to_lvalue(self, expr):
+        """
         print('expr_to_lvalue')
         print(' %s' % expr)
         print(' %r' % expr)
         print(' %s' % type(expr))
         print(' %s' % dir(expr))
         print(' %s' % expr.__dict__)
+        """
         if isinstance(expr, BoxInt):
             return self.get_box_as_lvalue(expr)
-        raise foo
+        raise ValueError('unhandled expr: %s' % expr)
 
     # Handling of specific ResOperation subclasses
 
     def _on_int_add(self, op):
+        """
         print(op._arg0)
         print(op._arg1)
         print(op.result)
         print(op.__dict__)
-
+        """
         rval0 = self.expr_to_rvalue(op._arg0)
         rval1 = self.expr_to_rvalue(op._arg1)
         lvalres = self.expr_to_lvalue(op.result)
@@ -180,7 +197,9 @@
                                               op_add)
 
     def _on_finish(self, op):
+        """
         print(op.__dict__)
+        """
         # FIXME: assume just 1-ary FINISH for now
         assert len(op._args) == 1
         result = op._args[0]
diff --git a/rpython/jit/backend/libgccjit/runner.py b/rpython/jit/backend/libgccjit/runner.py
--- a/rpython/jit/backend/libgccjit/runner.py
+++ b/rpython/jit/backend/libgccjit/runner.py
@@ -14,6 +14,7 @@
 
     def compile_loop(self, inputargs, operations, looptoken,
                      log=True, name='', logger=None):
+        """
         import sys
         sys.stderr.write('compile_loop:\n')
         for i, arg in enumerate(inputargs):
@@ -27,6 +28,6 @@
         sys.stderr.write('  name: %r\n' % (name, ))
         sys.stderr.write('  logger: %r\n' % (logger, ))
         sys.stderr.write('compile_loop: %r\n' % locals ())
-        #raise NotImplementedError
+        """
         return self.assembler.assemble_loop(inputargs, operations, looptoken, log,
                                             name, logger)

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:26 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:26 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: WIP, sent as
 https://mail.python.org/pipermail/pypy-dev/2014-December/012947.html
Message-ID: <20141217164826.C8AD21C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74973:2ae9fa4283cd
Date: 2014-12-15 09:57 -0500
http://bitbucket.org/pypy/pypy/changeset/2ae9fa4283cd/

Log:	WIP, sent as https://mail.python.org/pipermail/pypy-
	dev/2014-December/012947.html

diff --git a/rpython/jit/backend/detect_cpu.py b/rpython/jit/backend/detect_cpu.py
--- a/rpython/jit/backend/detect_cpu.py
+++ b/rpython/jit/backend/detect_cpu.py
@@ -99,6 +99,9 @@
 
 
 def getcpuclassname(backend_name="auto"):
+    # FIXME:
+    return 'rpython.jit.backend.libgccjit.runner', 'CPU'
+
     if backend_name == "auto":
         backend_name = autodetect()
     backend_name = backend_name.replace('_', '-')
diff --git a/rpython/jit/backend/libgccjit/__init__.py b/rpython/jit/backend/libgccjit/__init__.py
new file mode 100644
diff --git a/rpython/jit/backend/libgccjit/cffi_bindings.py b/rpython/jit/backend/libgccjit/cffi_bindings.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/libgccjit/cffi_bindings.py
@@ -0,0 +1,149 @@
+import os
+import re
+from collections import namedtuple
+
+# Hacks:
+INSTALL_PATH = '/home/david/coding-3/gcc-git-jit-pypy/install'
+INCLUDE_DIR = os.path.join(INSTALL_PATH, 'include')
+LIB_DIR = os.path.join(INSTALL_PATH, 'lib')
+BIN_DIR = os.path.join(INSTALL_PATH, 'bin')
+
+def append_to_envvar_path(envvar, path):
+    if envvar in os.environ:
+        os.environ[envvar] = path + ':' + os.environ[envvar]
+    else:
+        os.environ[envvar] = path
+    print('%s=%s' % (envvar, os.environ[envvar]))
+
+# It appears that we need to override os.environ['LD_LIBRARY_PATH']
+# before importing cffi for it to take account of this.
+append_to_envvar_path('LD_LIBRARY_PATH', LIB_DIR)
+# actually, for some reason I get:
+#  File "/usr/lib64/python2.7/site-packages/cffi/vengine_cpy.py", line 124, in load_library
+#    raise ffiplatform.VerificationError(error)
+# cffi.ffiplatform.VerificationError: importing '/home/david/coding-3/pypy-libgccjit/rpython/jit/backend/libgccjit/__pycache__/_cffi__x5c2f8978xf4274cdc.so': libgccjit.so.0: cannot open shared object file: No such file or directory
+# if LD_LIBRARY_PATH isn't set up before python starts up; issue with imp.load_dynamic ?
+
+# The library requires the correct driver to be in the PATH:
+append_to_envvar_path('PATH', BIN_DIR)
+
+os.system('env')
+
+import cffi
+
+ffi = cffi.FFI()
+
+with open(os.path.join(INCLUDE_DIR, 'libgccjit.h')) as f:
+    libgccjit_h_content = f.read()
+
+def toy_preprocessor(content):
+    """
+    ffi.cdef can't handle preprocessor directives.
+    We only have the idempotency guards and ifdef __cplusplus;
+    strip them out.
+    """
+    State = namedtuple('State', ('line', 'accepting_text'))
+    macros = {}
+    result = [] # list of lines
+    states = [State('default', accepting_text=True)]
+    for line in content.splitlines():
+        if 0:
+            print(repr(line))
+
+        m = re.match('#ifndef\s+(\S+)', line)
+        if m:
+            states.append(State(line,
+                                accepting_text=(m.group(1) not in macros)) )
+            continue
+        m = re.match('#ifdef\s+(\S+)', line)
+
+        if m:
+            states.append(State(line,
+                                accepting_text=(m.group(1) in macros)) )
+            continue
+
+        m = re.match('#define\s+(\S+)', line)
+        if m:
+            macros[m.group(1)] = ''
+            continue
+
+        m = re.match('#endif\s*', line)
+        if m:
+            states.pop()
+            continue
+
+        if states[-1].accepting_text:
+            result.append(line)
+
+    return '\n'.join(result)
+    
+libgccjit_h_content = toy_preprocessor(libgccjit_h_content)
+
+# print(libgccjit_h_content)
+
+ffi.cdef(libgccjit_h_content)
+
+lib = ffi.verify('#include "libgccjit.h"',
+                 libraries=['gccjit'],
+                 library_dirs=[LIB_DIR],
+                 include_dirs=[INCLUDE_DIR])
+
+ctxt = lib.gcc_jit_context_acquire()
+print ctxt
+
+lib.gcc_jit_context_set_bool_option(ctxt,
+        lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
+        1)
+lib.gcc_jit_context_set_int_option(ctxt,
+        lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
+        3)
+lib.gcc_jit_context_set_bool_option(ctxt,
+        lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
+        1)
+lib.gcc_jit_context_set_bool_option(ctxt,
+        lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
+        1)
+lib.gcc_jit_context_set_bool_option(ctxt,
+                                    lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
+                                    1)
+
+int_type = lib.gcc_jit_context_get_type(ctxt, lib.GCC_JIT_TYPE_INT)
+param = lib.gcc_jit_context_new_param(ctxt, ffi.NULL, int_type, "input")
+fn = lib.gcc_jit_context_new_function(ctxt,
+                                      ffi.NULL,
+                                      lib.GCC_JIT_FUNCTION_EXPORTED,
+                                      int_type,
+                                      "add_one_to",
+                                      1, [param], 0)
+v_res = lib.gcc_jit_function_new_local(fn, ffi.NULL, int_type, "v_res")
+
+b_initial = lib.gcc_jit_function_new_block(fn, "initial")
+
+c_one = lib.gcc_jit_context_new_rvalue_from_int(ctxt, int_type, 1)
+
+op_add = lib.gcc_jit_context_new_binary_op(ctxt, ffi.NULL,
+                                           lib.GCC_JIT_BINARY_OP_PLUS,
+                                           int_type,
+                                           lib.gcc_jit_param_as_rvalue(param),
+                                           c_one)
+lib.gcc_jit_block_add_assignment(b_initial, ffi.NULL,
+                                 v_res,
+                                 op_add)
+
+lib.gcc_jit_block_end_with_return(b_initial, ffi.NULL,
+                                  lib.gcc_jit_lvalue_as_rvalue(v_res))
+
+jit_result = lib.gcc_jit_context_compile(ctxt)
+
+lib.gcc_jit_context_release(ctxt)
+
+fn_ptr = lib.gcc_jit_result_get_code(jit_result, "add_one_to")
+if not fn_ptr:
+    raise Exception("fn_ptr is NULL")
+print('fn_ptr: %r' % fn_ptr)
+
+fn_result = ffi.cast("int(*)(int)", fn_ptr)(41)
+print('fn_result: %r' % fn_result)
+assert fn_result == 42
+
+lib.gcc_jit_result_release(jit_result)
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -0,0 +1,249 @@
+import os
+
+# Hacks:
+INSTALL_PATH = '/home/david/coding-3/gcc-git-jit-pypy/install'
+INCLUDE_DIR = os.path.join(INSTALL_PATH, 'include')
+LIB_DIR = os.path.join(INSTALL_PATH, 'lib')
+BIN_DIR = os.path.join(INSTALL_PATH, 'bin')
+
+def append_to_envvar_path(envvar, path):
+    if envvar in os.environ:
+        os.environ[envvar] = path + ':' + os.environ[envvar]
+    else:
+        os.environ[envvar] = path
+    print('%s=%s' % (envvar, os.environ[envvar]))
+
+# It appears that we need to override os.environ['LD_LIBRARY_PATH']
+# before importing cffi for it to take account of this.
+append_to_envvar_path('LD_LIBRARY_PATH', LIB_DIR)
+# actually, for some reason I get:
+#  File "/usr/lib64/python2.7/site-packages/cffi/vengine_cpy.py", line 124, in load_library
+#    raise ffiplatform.VerificationError(error)
+# cffi.ffiplatform.VerificationError: importing '/home/david/coding-3/pypy-libgccjit/rpython/jit/backend/libgccjit/__pycache__/_cffi__x5c2f8978xf4274cdc.so': libgccjit.so.0: cannot open shared object file: No such file or directory
+# if LD_LIBRARY_PATH isn't set up before python starts up; issue with imp.load_dynamic ?
+
+# The library requires the correct driver to be in the PATH:
+append_to_envvar_path('PATH', BIN_DIR)
+
+from rpython.rtyper.lltypesystem import rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rtyper.lltypesystem.rffi import *
+from rpython.rtyper.lltypesystem import lltype
+
+def make_eci():
+    eci = ExternalCompilationInfo(includes=['libgccjit.h'],
+                                  include_dirs=[INCLUDE_DIR],
+                                  libraries=['gccjit'],
+                                  library_dirs=[LIB_DIR])
+    return eci
+
+class Library:
+    def __init__(self, eci):
+        self.eci = eci
+
+        # Opaque types:
+        self.GCC_JIT_CONTEXT_P = lltype.Ptr(COpaque(name='gcc_jit_context',
+                                                    compilation_info=eci))
+        self.GCC_JIT_RESULT_P = lltype.Ptr(COpaque(name='gcc_jit_result',
+                                                   compilation_info=eci))
+        self.GCC_JIT_TYPE_P = lltype.Ptr(COpaque(name='gcc_jit_type',
+                                                 compilation_info=eci))
+        self.GCC_JIT_LOCATION_P = lltype.Ptr(COpaque(name='gcc_jit_location',
+                                                     compilation_info=eci))
+        self.GCC_JIT_PARAM_P = lltype.Ptr(COpaque(name='gcc_jit_param',
+                                                  compilation_info=eci))
+        self.GCC_JIT_LVALUE_P = lltype.Ptr(COpaque(name='gcc_jit_lvalue',
+                                                  compilation_info=eci))
+        self.GCC_JIT_RVALUE_P = lltype.Ptr(COpaque(name='gcc_jit_rvalue',
+                                                  compilation_info=eci))
+        self.GCC_JIT_FUNCTION_P = lltype.Ptr(COpaque(name='gcc_jit_function',
+                                                     compilation_info=eci))
+        self.GCC_JIT_BLOCK_P = lltype.Ptr(COpaque(name='gcc_jit_block',
+                                                     compilation_info=eci))
+
+        self.PARAM_P_P = lltype.Ptr(lltype.Array(self.GCC_JIT_PARAM_P,
+                                                 hints={'nolength': True}))
+
+        # Entrypoints:
+        for returntype, name, paramtypes in [
+                (self.GCC_JIT_CONTEXT_P,
+                 'gcc_jit_context_acquire', []),
+
+                (lltype.Void,
+                 'gcc_jit_context_release', [self.GCC_JIT_CONTEXT_P]),
+
+                (lltype.Void,
+                 'gcc_jit_context_set_int_option', [self.GCC_JIT_CONTEXT_P,
+                                                    INT, # FIXME: enum gcc_jit_int_option opt,
+                                                    INT]),
+                (lltype.Void,
+                 'gcc_jit_context_set_bool_option', [self.GCC_JIT_CONTEXT_P,
+                                                     INT, # FIXME: enum gcc_jit_bool_option opt,
+                                                     INT]),
+
+                (self.GCC_JIT_RESULT_P,
+                 'gcc_jit_context_compile', [self.GCC_JIT_CONTEXT_P]),
+
+
+                (VOIDP,
+                 'gcc_jit_result_get_code', [self.GCC_JIT_RESULT_P,
+                                             CCHARP]),
+
+                (lltype.Void,
+                 'gcc_jit_result_release', [self.GCC_JIT_RESULT_P]),
+
+                ############################################################
+                # Types
+                ############################################################
+                (self.GCC_JIT_TYPE_P,
+                 'gcc_jit_context_get_type', [self.GCC_JIT_CONTEXT_P,
+                                              INT]),
+
+                ############################################################
+                # Constructing functions.
+                ############################################################
+                (self.GCC_JIT_PARAM_P,
+                 'gcc_jit_context_new_param', [self.GCC_JIT_CONTEXT_P,
+                                               self.GCC_JIT_LOCATION_P,
+                                               self.GCC_JIT_TYPE_P,
+                                               CCHARP]),
+                (self.GCC_JIT_LVALUE_P,
+                 'gcc_jit_param_as_lvalue', [self.GCC_JIT_PARAM_P]),
+                (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_param_as_rvalue', [self.GCC_JIT_PARAM_P]),
+
+                (self.GCC_JIT_FUNCTION_P,
+                 'gcc_jit_context_new_function', [self.GCC_JIT_CONTEXT_P,
+                                                  self.GCC_JIT_LOCATION_P,
+                                                  INT, # enum gcc_jit_function_kind kind,
+                                                  self.GCC_JIT_TYPE_P,
+                                                  CCHARP,
+                                                  INT,
+                                                  self.PARAM_P_P,
+                                                  INT]),
+                (self.GCC_JIT_LVALUE_P,
+                 'gcc_jit_function_new_local', [self.GCC_JIT_FUNCTION_P,
+                                                self.GCC_JIT_LOCATION_P,
+                                                self.GCC_JIT_TYPE_P,
+                                                CCHARP]),
+
+                (self.GCC_JIT_BLOCK_P,
+                 'gcc_jit_function_new_block', [self.GCC_JIT_FUNCTION_P,
+                                                CCHARP]),
+
+                ############################################################
+                # lvalues, rvalues and expressions.
+                ############################################################
+                (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_lvalue_as_rvalue', [self.GCC_JIT_LVALUE_P]),
+
+                # Integer constants.
+                (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_new_rvalue_from_int', [self.GCC_JIT_CONTEXT_P,
+                                                         self.GCC_JIT_TYPE_P,
+                                                         INT]),
+                (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_zero', [self.GCC_JIT_CONTEXT_P,
+                                          self.GCC_JIT_TYPE_P]),
+                (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_one', [self.GCC_JIT_CONTEXT_P,
+                                         self.GCC_JIT_TYPE_P]),
+
+                (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_new_binary_op', [self.GCC_JIT_CONTEXT_P,
+                                                   self.GCC_JIT_LOCATION_P,
+                                                   INT, # enum gcc_jit_binary_op op,
+                                                   self.GCC_JIT_TYPE_P,
+                                                   self.GCC_JIT_RVALUE_P,
+                                                   self.GCC_JIT_RVALUE_P]),
+
+                ############################################################
+                # Statement-creation.
+                ############################################################
+                (lltype.Void,
+                 'gcc_jit_block_add_assignment', [self.GCC_JIT_BLOCK_P,
+                                                  self.GCC_JIT_LOCATION_P,
+                                                  self.GCC_JIT_LVALUE_P,
+                                                  self.GCC_JIT_RVALUE_P]),
+                (lltype.Void,
+                 'gcc_jit_block_end_with_return', [self.GCC_JIT_BLOCK_P,
+                                                   self.GCC_JIT_LOCATION_P,
+                                                   self.GCC_JIT_RVALUE_P]),
+        ]:
+            self.add_entrypoint(returntype, name, paramtypes)
+
+        # Enum values:
+        self.make_enum_values("""GCC_JIT_STR_OPTION_PROGNAME""")
+
+        self.make_enum_values("""GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL""")
+
+        self.make_enum_values("""GCC_JIT_BOOL_OPTION_DEBUGINFO,
+        GCC_JIT_BOOL_OPTION_DUMP_INITIAL_TREE,
+        GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
+        GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
+        GCC_JIT_BOOL_OPTION_DUMP_SUMMARY,
+        GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
+        GCC_JIT_BOOL_OPTION_SELFCHECK_GC,
+        GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
+        """)
+
+        self.make_enum_values("""GCC_JIT_TYPE_VOID,
+        GCC_JIT_TYPE_VOID_PTR,
+        GCC_JIT_TYPE_BOOL,
+        GCC_JIT_TYPE_CHAR,
+        GCC_JIT_TYPE_SIGNED_CHAR,
+        GCC_JIT_TYPE_UNSIGNED_CHAR,
+        GCC_JIT_TYPE_SHORT,
+        GCC_JIT_TYPE_UNSIGNED_SHORT,
+        GCC_JIT_TYPE_INT,
+        GCC_JIT_TYPE_UNSIGNED_INT,
+        GCC_JIT_TYPE_LONG,
+        GCC_JIT_TYPE_UNSIGNED_LONG,
+        GCC_JIT_TYPE_LONG_LONG,
+        GCC_JIT_TYPE_UNSIGNED_LONG_LONG,
+        GCC_JIT_TYPE_FLOAT,
+        GCC_JIT_TYPE_DOUBLE,
+        GCC_JIT_TYPE_LONG_DOUBLE,
+        GCC_JIT_TYPE_CONST_CHAR_PTR,
+        GCC_JIT_TYPE_SIZE_T,
+        GCC_JIT_TYPE_FILE_PTR,
+        GCC_JIT_TYPE_COMPLEX_FLOAT,
+        GCC_JIT_TYPE_COMPLEX_DOUBLE,
+        GCC_JIT_TYPE_COMPLEX_LONG_DOUBLE""")
+
+        self.make_enum_values("""GCC_JIT_FUNCTION_EXPORTED,
+        GCC_JIT_FUNCTION_INTERNAL,
+        GCC_JIT_FUNCTION_IMPORTED,
+        GCC_JIT_FUNCTION_ALWAYS_INLINE""")
+
+        self.make_enum_values(
+            """
+            GCC_JIT_BINARY_OP_PLUS,
+            GCC_JIT_BINARY_OP_MINUS,
+            GCC_JIT_BINARY_OP_MULT,
+            GCC_JIT_BINARY_OP_DIVIDE,
+            GCC_JIT_BINARY_OP_MODULO,
+            GCC_JIT_BINARY_OP_BITWISE_AND,
+            GCC_JIT_BINARY_OP_BITWISE_XOR,
+            GCC_JIT_BINARY_OP_BITWISE_OR,
+            GCC_JIT_BINARY_OP_LOGICAL_AND,
+            GCC_JIT_BINARY_OP_LOGICAL_OR,
+            GCC_JIT_BINARY_OP_LSHIFT,
+            GCC_JIT_BINARY_OP_RSHIFT
+            """)
+
+        self.null_location_ptr = lltype.nullptr(self.GCC_JIT_LOCATION_P.TO)
+
+
+    def add_entrypoint(self, returntype, name, paramtypes):
+        setattr(self, name,
+                llexternal(name, paramtypes, returntype,
+                           compilation_info=self.eci))
+
+    def make_enum_values(self, lines):
+        for value, name in enumerate(lines.split(',')):
+            name = name.strip()
+            if name:
+                setattr(self, name, value)
+        
+
diff --git a/rpython/jit/backend/libgccjit/runner.py b/rpython/jit/backend/libgccjit/runner.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/libgccjit/runner.py
@@ -0,0 +1,32 @@
+#from rpython.jit.backend import model
+from rpython.jit.backend.libgccjit.assembler import AssemblerLibgccjit
+from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU
+
+#class CPU(model.AbstractCPU):
+class CPU(AbstractLLCPU):
+    def __init__(self, rtyper, stats, opts=None, translate_support_code=False,
+                 gcdescr=None):
+        AbstractLLCPU.__init__(self, rtyper, stats, opts,
+                               translate_support_code, gcdescr)
+
+    def setup(self):
+        self.assembler = AssemblerLibgccjit(self)
+
+    def compile_loop(self, inputargs, operations, looptoken,
+                     log=True, name='', logger=None):
+        import sys
+        sys.stderr.write('compile_loop:\n')
+        for i, arg in enumerate(inputargs):
+            sys.stderr.write('  arg[%i] = %r\n' % (i, arg))
+            sys.stderr.write('    type(arg[%i]) = %r\n' % (i, type(arg)))
+        for i, op in enumerate(operations):
+            sys.stderr.write('  op[%i] = %r\n' % (i, op))
+            sys.stderr.write('    type(op[%i]) = %r\n' % (i, type(op)))
+        sys.stderr.write('  looptoken: %r\n' % (looptoken, ))
+        sys.stderr.write('  log: %r\n' % (log, ))
+        sys.stderr.write('  name: %r\n' % (name, ))
+        sys.stderr.write('  logger: %r\n' % (logger, ))
+        sys.stderr.write('compile_loop: %r\n' % locals ())
+        #raise NotImplementedError
+        return self.assembler.assemble_loop(inputargs, operations, looptoken, log,
+                                            name, logger)
diff --git a/rpython/jit/backend/libgccjit/test/__init__.py b/rpython/jit/backend/libgccjit/test/__init__.py
new file mode 100644
diff --git a/rpython/jit/backend/libgccjit/test/test_basic.py b/rpython/jit/backend/libgccjit/test/test_basic.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/libgccjit/test/test_basic.py
@@ -0,0 +1,15 @@
+import py
+from rpython.jit.backend.detect_cpu import getcpuclass
+from rpython.jit.metainterp.test import support, test_ajit
+
+class JitLibgccjitMixin(support.LLJitMixin):
+    type_system = 'lltype'
+    CPUClass = getcpuclass()
+
+    def check_jumps(self, maxcount):
+        pass
+
+class TestBasic(JitLibgccjitMixin, test_ajit.BaseLLtypeTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_basic.py
+    pass
diff --git a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
@@ -0,0 +1,160 @@
+
+import py
+import sys
+from rpython.rtyper.lltypesystem.rffi import *
+from rpython.rtyper.lltypesystem.rffi import _keeper_for_type # crap
+from rpython.rlib.rposix import get_errno, set_errno
+from rpython.translator.c.test.test_genc import compile as compile_c
+from rpython.rtyper.lltypesystem.lltype import Signed, Ptr, Char, malloc
+from rpython.rtyper.lltypesystem import lltype
+from rpython.translator import cdir
+from rpython.tool.udir import udir
+from rpython.rtyper.test.test_llinterp import interpret
+from rpython.annotator.annrpython import RPythonAnnotator
+from rpython.rtyper.rtyper import RPythonTyper
+from rpython.translator.backendopt.all import backend_optimizations
+from rpython.translator.translator import graphof
+from rpython.conftest import option
+from rpython.flowspace.model import summary
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rlib.rarithmetic import r_singlefloat
+
+"""
+def test_string():
+    eci = ExternalCompilationInfo(includes=['string.h'])
+    z = llexternal('strlen', [CCHARP], Signed, compilation_info=eci)
+
+    def f():
+        s = str2charp("xxx")
+        res = z(s)
+        free_charp(s)
+        return res
+
+    xf = compile_c(f, [], backendopt=False)
+    assert xf() == 3
+"""
+
+from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library
+
+def test_compile_empty_context():
+    eci = make_eci()
+
+    lib = Library(eci)
+
+    def f():
+        ctxt = lib.gcc_jit_context_acquire()
+        result = lib.gcc_jit_context_compile(ctxt)
+        lib.gcc_jit_context_release(ctxt)
+        lib.gcc_jit_result_release(result)
+        
+    f1 = compile_c(f, [], backendopt=False)
+    f1 ()
+    #assert False # to see stderr
+
+def make_param_array(lib, l):
+    array = lltype.malloc(lib.PARAM_P_P.TO,
+                          len(l),
+                          flavor='raw') # of maybe gc?
+    for i in range(len(l)):
+        array[i] = l[i]
+    return array
+    # FIXME: don't leak!
+    
+def test_compile_add_one_to():
+    eci = make_eci()
+
+    lib = Library(eci)
+
+    ft = lltype.FuncType([INT], INT)#, abi="C")
+    ftp = lltype.Ptr(ft)
+
+    def f():
+        ctxt = lib.gcc_jit_context_acquire()
+
+        lib.gcc_jit_context_set_bool_option(ctxt,
+                                            lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
+                                            1)
+        lib.gcc_jit_context_set_int_option(ctxt,
+                                           lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
+                                           3)
+        lib.gcc_jit_context_set_bool_option(ctxt,
+                                            lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
+                                            1)
+        lib.gcc_jit_context_set_bool_option(ctxt,
+                                            lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
+                                            1)
+        lib.gcc_jit_context_set_bool_option(ctxt,
+                                            lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
+                                            1)
+        t_int = lib.gcc_jit_context_get_type(ctxt, lib.GCC_JIT_TYPE_INT)
+        param = lib.gcc_jit_context_new_param(ctxt,
+                                              lib.null_location_ptr,
+                                              t_int,
+                                              "input")
+        # FIXME: how to build an array of params at this level?
+        # see liststr2charpp in rffi.py
+        
+        param_array = make_param_array(lib, [param])
+        fn = lib.gcc_jit_context_new_function(ctxt,
+                                              lib.null_location_ptr,
+                                              lib.GCC_JIT_FUNCTION_EXPORTED,
+                                              t_int,
+                                              "add_one_to",
+                                              1, param_array, 0)
+        lltype.free(param_array, flavor='raw')
+
+        v_res = lib.gcc_jit_function_new_local(fn,
+                                               lib.null_location_ptr,
+                                               t_int,
+                                               "v_res")
+        b_initial = lib.gcc_jit_function_new_block(fn, "initial")
+        c_one = lib.gcc_jit_context_new_rvalue_from_int(ctxt, t_int, 1)
+        op_add = lib.gcc_jit_context_new_binary_op(ctxt,
+                                                   lib.null_location_ptr,
+                                                   lib.GCC_JIT_BINARY_OP_PLUS,
+                                                   t_int,
+                                                   lib.gcc_jit_param_as_rvalue(param),
+                                                   c_one)
+        lib.gcc_jit_block_add_assignment(b_initial, lib.null_location_ptr,
+                                         v_res,
+                                         op_add)
+        lib.gcc_jit_block_end_with_return(b_initial, lib.null_location_ptr,
+                                          lib.gcc_jit_lvalue_as_rvalue(v_res))
+
+        jit_result = lib.gcc_jit_context_compile(ctxt)
+        lib.gcc_jit_context_release(ctxt)
+        if not jit_result:
+            # FIXME: get error from context
+            raise Exception("jit_result is NULL")
+
+        fn_ptr = lib.gcc_jit_result_get_code(jit_result, "add_one_to")
+        if not fn_ptr:
+            raise Exception("fn_ptr is NULL")
+        print('fn_ptr: %s' % fn_ptr)
+
+        #ft = lltype.FuncType([INT], INT)#, abi="C")
+        # looks like we can't build a FuncType inside RPython
+        # but we can use one built outside:
+        print(ft)
+        print(ftp)
+
+        typed_fn_ptr = cast(ftp, fn_ptr)
+        print(typed_fn_ptr)
+        fn_result = typed_fn_ptr (r_int(41))
+        #print('fn_result: %d' % fn_result)
+        #assert fn_result == r_int(42)
+
+        # and it looks like we can't create a functionptr from this
+        # FuncType:
+        #funcptr = lltype.functionptr(ft)
+
+        lib.gcc_jit_result_release(jit_result)
+
+        return int(fn_result)
+        
+    f1 = compile_c(f, [], backendopt=False)
+    assert f1() == 42
+    #assert False # to see stderr
+    
+# TODO: test of an error
+# should turn it into an exception, and capture the error

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:32 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:32 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Implement an OO-style rffi
 wrapper to libgccjit
Message-ID: <20141217164832.85F591C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74978:947cc4471f74
Date: 2014-12-16 06:46 -0500
http://bitbucket.org/pypy/pypy/changeset/947cc4471f74/

Log:	Implement an OO-style rffi wrapper to libgccjit

diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -46,6 +46,15 @@
     return array
     # FIXME: don't leak!
 
+def make_field_array(lib, l):
+    array = lltype.malloc(lib.FIELD_P_P.TO,
+                          len(l),
+                          flavor='raw') # of maybe gc?
+    for i in range(len(l)):
+        array[i] = l[i]
+    return array
+    # FIXME: don't leak!
+
 class Library:
     def __init__(self, eci):
         self.eci = eci
@@ -57,6 +66,10 @@
                                                    compilation_info=eci))
         self.GCC_JIT_TYPE_P = lltype.Ptr(COpaque(name='gcc_jit_type',
                                                  compilation_info=eci))
+        self.GCC_JIT_FIELD_P = lltype.Ptr(COpaque(name='gcc_jit_field',
+                                                  compilation_info=eci))
+        self.GCC_JIT_STRUCT_P = lltype.Ptr(COpaque(name='gcc_jit_struct',
+                                                   compilation_info=eci))
         self.GCC_JIT_LOCATION_P = lltype.Ptr(COpaque(name='gcc_jit_location',
                                                      compilation_info=eci))
         self.GCC_JIT_PARAM_P = lltype.Ptr(COpaque(name='gcc_jit_param',
@@ -70,6 +83,8 @@
         self.GCC_JIT_BLOCK_P = lltype.Ptr(COpaque(name='gcc_jit_block',
                                                      compilation_info=eci))
 
+        self.FIELD_P_P = lltype.Ptr(lltype.Array(self.GCC_JIT_FIELD_P,
+                                                 hints={'nolength': True}))
         self.PARAM_P_P = lltype.Ptr(lltype.Array(self.GCC_JIT_PARAM_P,
                                                  hints={'nolength': True}))
 
@@ -94,6 +109,11 @@
                  'gcc_jit_context_compile', [self.GCC_JIT_CONTEXT_P]),
 
 
+                (lltype.Void,
+                 'gcc_jit_context_dump_to_file', [self.GCC_JIT_CONTEXT_P,
+                                                  CCHARP,
+                                                  INT]),
+
                 (VOIDP,
                  'gcc_jit_result_get_code', [self.GCC_JIT_RESULT_P,
                                              CCHARP]),
@@ -108,6 +128,34 @@
                  'gcc_jit_context_get_type', [self.GCC_JIT_CONTEXT_P,
                                               INT]),
 
+                (self.GCC_JIT_TYPE_P,
+                 'gcc_jit_type_get_pointer', [self.GCC_JIT_TYPE_P]),
+
+                (self.GCC_JIT_FIELD_P,
+                 'gcc_jit_context_new_field', [self.GCC_JIT_CONTEXT_P,
+                                               self.GCC_JIT_LOCATION_P,
+                                               self.GCC_JIT_TYPE_P,
+                                               CCHARP]),
+                (self.GCC_JIT_STRUCT_P,
+                 'gcc_jit_context_new_struct_type', [self.GCC_JIT_CONTEXT_P,
+                                                     self.GCC_JIT_LOCATION_P,
+                                                     CCHARP,
+                                                     INT,
+                                                     self.FIELD_P_P]),
+
+                (self.GCC_JIT_STRUCT_P,
+                 'gcc_jit_context_new_opaque_struct', [self.GCC_JIT_CONTEXT_P,
+                                                       self.GCC_JIT_LOCATION_P,
+                                                       CCHARP]),
+                (self.GCC_JIT_TYPE_P,
+                 'gcc_jit_struct_as_type', [self.GCC_JIT_STRUCT_P]),
+
+                (lltype.Void,
+                 'gcc_jit_struct_set_fields', [self.GCC_JIT_STRUCT_P,
+                                               self.GCC_JIT_LOCATION_P,
+                                               INT,
+                                               self.FIELD_P_P]),
+
                 ############################################################
                 # Constructing functions.
                 ############################################################
@@ -159,6 +207,11 @@
                                          self.GCC_JIT_TYPE_P]),
 
                 (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_new_rvalue_from_ptr', [self.GCC_JIT_CONTEXT_P,
+                                                         self.GCC_JIT_TYPE_P,
+                                                         VOIDP]),
+
+                (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_context_new_binary_op', [self.GCC_JIT_CONTEXT_P,
                                                    self.GCC_JIT_LOCATION_P,
                                                    INT, # enum gcc_jit_binary_op op,
@@ -166,6 +219,11 @@
                                                    self.GCC_JIT_RVALUE_P,
                                                    self.GCC_JIT_RVALUE_P]),
 
+                (self.GCC_JIT_LVALUE_P,
+                 'gcc_jit_rvalue_dereference_field', [self.GCC_JIT_RVALUE_P,
+                                                      self.GCC_JIT_LOCATION_P,
+                                                      self.GCC_JIT_FIELD_P]),
+
                 ############################################################
                 # Statement-creation.
                 ############################################################
@@ -259,5 +317,181 @@
             name = name.strip()
             if name:
                 setattr(self, name, r_int(value))
-        
 
+# An object-oriented interfact to the library
+
+class Wrapper:
+    def __init__(self, lib):
+        self.lib = lib
+
+class Context(Wrapper):
+    def __init__(self, lib, inner_ctxt):
+        Wrapper.__init__(self, lib)
+        self.inner_ctxt = inner_ctxt
+
+    @staticmethod
+    def acquire(lib):
+        return Context(lib, lib.gcc_jit_context_acquire())
+
+    def release(self):
+        self.lib.gcc_jit_context_release(self.inner_ctxt)
+
+    def set_bool_option(self, key, val):
+        self.lib.gcc_jit_context_set_bool_option(self.inner_ctxt,
+                                                 key, val)
+
+    def set_int_option(self, key, val):
+        self.lib.gcc_jit_context_set_int_option(self.inner_ctxt,
+                                                key, val)
+
+    def compile(self):
+        inner_result = self.lib.gcc_jit_context_compile(self.inner_ctxt)
+        if not inner_result:
+            # FIXME: get error from context
+            raise Exception("result is NULL")
+        return Result(self.lib, inner_result)
+
+    def get_type(self, r_enum):
+        return Type(self.lib,
+                    self.lib.gcc_jit_context_get_type(self.inner_ctxt,
+                                                      r_enum))
+
+    def new_field(self, type_, name):
+        name_charp = str2charp(name)
+        field = self.lib.gcc_jit_context_new_field(self.inner_ctxt,
+                                                   self.lib.null_location_ptr,
+                                                   type_.inner_type,
+                                                   name_charp)
+        free_charp(name_charp)
+        return Field(self.lib, field)
+    
+    def new_rvalue_from_int(self, type_, llvalue):
+        return RValue(self.lib,
+                      self.lib.gcc_jit_context_new_rvalue_from_int(self.inner_ctxt,
+                                                                   type_.inner_type,
+                                                                   llvalue))
+
+    def new_binary_op(self, op, type_, a, b):
+        return RValue(self.lib,
+                      self.lib.gcc_jit_context_new_binary_op(self.inner_ctxt,
+                                                             self.lib.null_location_ptr,
+                                                             op,
+                                                             type_.inner_type,
+                                                             a.inner_rvalue, b.inner_rvalue))
+
+    def new_param(self, type_, name):
+        name_charp = str2charp(name)
+        param = self.lib.gcc_jit_context_new_param(self.inner_ctxt,
+                                                   self.lib.null_location_ptr,
+                                                   type_.inner_type,
+                                                   name_charp)
+        free_charp(name_charp)
+        return Param(self.lib, param)
+
+    def new_function(self, kind, returntype, name, params, is_variadic):
+        name_charp = str2charp(name)
+        raw_param_array = lltype.malloc(self.lib.PARAM_P_P.TO,
+                                    len(params),
+                                    flavor='raw') # of maybe gc?
+        for i in range(len(params)):
+            raw_param_array[i] = params[i].inner_param
+
+        fn = self.lib.gcc_jit_context_new_function(self.inner_ctxt,
+                                                   self.lib.null_location_ptr,
+                                                   kind,
+                                                   returntype.inner_type,
+                                                   name_charp,
+                                                   r_int(len(params)),
+                                                   raw_param_array,
+                                                   is_variadic)
+        lltype.free(raw_param_array, flavor='raw')
+        free_charp(name_charp)
+
+        return Function(self.lib, fn)
+
+
+class Type(Wrapper):
+    def __init__(self, lib, inner_type):
+        Wrapper.__init__(self, lib)
+        self.inner_type = inner_type
+
+
+class Field(Wrapper):
+    def __init__(self, lib, inner_field):
+        Wrapper.__init__(self, lib)
+        self.inner_field = inner_field
+
+class RValue(Wrapper):
+    def __init__(self, lib, inner_rvalue):
+        Wrapper.__init__(self, lib)
+        self.inner_rvalue = inner_rvalue
+
+class LValue(Wrapper):
+    def __init__(self, lib, inner_lvalue):
+        Wrapper.__init__(self, lib)
+        self.inner_lvalue = inner_lvalue
+
+    def as_rvalue(self):
+        return RValue(self.lib,
+                      self.lib.gcc_jit_lvalue_as_rvalue(self.inner_lvalue))
+
+class Param(Wrapper):
+    def __init__(self, lib, inner_param):
+        Wrapper.__init__(self, lib)
+        self.inner_param = inner_param
+
+    def as_rvalue(self):
+        return RValue(self.lib,
+                      self.lib.gcc_jit_param_as_rvalue(self.inner_param))
+
+class Function(Wrapper):
+    def __init__(self, lib, inner_function):
+        Wrapper.__init__(self, lib)
+        self.inner_function = inner_function
+
+    def new_local(self, type_, name):
+        name_charp = str2charp(name)
+        local = self.lib.gcc_jit_function_new_local(self.inner_function,
+                                                    self.lib.null_location_ptr,
+                                                    type_.inner_type,
+                                                    name_charp)
+        free_charp(name_charp)
+        return LValue(self.lib, local)
+
+    def new_block(self, name):
+        name_charp = str2charp(name)
+        block = self.lib.gcc_jit_function_new_block(self.inner_function,
+                                                    name_charp)
+        free_charp(name_charp)
+        return Block(self.lib, block)
+
+class Block(Wrapper):
+    def __init__(self, lib, inner_block):
+        Wrapper.__init__(self, lib)
+        self.inner_block = inner_block
+
+    def add_assignment(self, lvalue, rvalue):
+        self.lib.gcc_jit_block_add_assignment(self.inner_block,
+                                              self.lib.null_location_ptr,
+                                              lvalue.inner_lvalue,
+                                              rvalue.inner_rvalue)
+
+    def end_with_return(self, rvalue):
+        self.lib.gcc_jit_block_end_with_return(self.inner_block,
+                                               self.lib.null_location_ptr,
+                                               rvalue.inner_rvalue)
+
+class Result(Wrapper):
+    def __init__(self, lib, inner_result):
+        Wrapper.__init__(self, lib)
+        self.inner_result = inner_result
+
+    def get_code(self, name):
+        name_charp = str2charp(name)
+        fn_ptr = self.lib.gcc_jit_result_get_code(self.inner_result,
+                                                  name_charp)
+        free_charp(name_charp)
+        return fn_ptr
+
+    def release(self):
+        self.lib.gcc_jit_result_release(self.inner_result)
diff --git a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
@@ -34,7 +34,7 @@
     assert xf() == 3
 """
 
-from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library, make_param_array
+from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library, make_param_array, Context
 
 def test_compile_empty_context():
     eci = make_eci()
@@ -84,6 +84,7 @@
                                               t_int,
                                               param_name)
         free_charp(param_name)
+
         # FIXME: how to build an array of params at this level?
         # see liststr2charpp in rffi.py
         
@@ -156,6 +157,75 @@
     f1 = compile_c(f, [], backendopt=False)
     assert f1() == 42
     assert False # to see stderr
+
+def test_oo_compile_add_one_to():
+    eci = make_eci()
+
+    lib = Library(eci)
+
+    ft = lltype.FuncType([INT], INT)#, abi="C")
+    ftp = lltype.Ptr(ft)
+
+    def f():
+        ctxt = Context.acquire(lib)
+        # FIXME: etc:
+        ctxt.set_bool_option(lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
+                             r_int(1))
+        ctxt.set_int_option(lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
+                            r_int(3))
+        ctxt.set_bool_option(lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
+                             r_int(1))
+        ctxt.set_bool_option(lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
+                             r_int(1))
+        ctxt.set_bool_option(lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
+                             r_int(1))
+        t_int = ctxt.get_type(lib.GCC_JIT_TYPE_INT)
+        param = ctxt.new_param(t_int, "input")
+        fn = ctxt.new_function(lib.GCC_JIT_FUNCTION_EXPORTED,
+                               t_int,
+                               "add_one_to",
+                               [param],
+                               r_int(0))
+        v_res = fn.new_local(t_int, "v_res")
+        b_initial = fn.new_block("initial")
+        c_one = ctxt.new_rvalue_from_int(t_int, r_int(1))
+        op_add = ctxt.new_binary_op(lib.GCC_JIT_BINARY_OP_PLUS,
+                                    t_int,
+                                    param.as_rvalue(),
+                                    c_one)
+        b_initial.add_assignment(v_res, op_add)
+        b_initial.end_with_return(v_res.as_rvalue())
+
+        jit_result = ctxt.compile()
+        ctxt.release()
+        fn_ptr = jit_result.get_code("add_one_to")
+        if not fn_ptr:
+            raise Exception("fn_ptr is NULL")
+        print('fn_ptr: %s' % fn_ptr)
+
+        #ft = lltype.FuncType([INT], INT)#, abi="C")
+        # looks like we can't build a FuncType inside RPython
+        # but we can use one built outside:
+        print(ft)
+        print(ftp)
+
+        typed_fn_ptr = cast(ftp, fn_ptr)
+        print(typed_fn_ptr)
+        fn_result = typed_fn_ptr (r_int(41))
+        #print('fn_result: %d' % fn_result)
+        #assert fn_result == r_int(42)
+
+        # and it looks like we can't create a functionptr from this
+        # FuncType:
+        #funcptr = lltype.functionptr(ft)
+
+        jit_result.release()
+
+        return int(fn_result)
+        
+    f1 = compile_c(f, [], backendopt=False)
+    assert f1() == 42
+    assert False # to see stderr
     
 # TODO: test of an error
 # should turn it into an exception, and capture the error

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:33 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:33 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Work-in-progress
Message-ID: <20141217164833.A3E371C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74979:cf027885030b
Date: 2014-12-16 10:12 -0500
http://bitbucket.org/pypy/pypy/changeset/cf027885030b/

Log:	Work-in-progress

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -1,8 +1,14 @@
+from rpython.jit.backend.llsupport import jitframe
 from rpython.jit.backend.llsupport.assembler import BaseAssembler
+from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
+from rpython.jit.backend.llsupport.regalloc import FrameManager
 from rpython.jit.backend.model import CompiledLoopToken
-from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library, make_param_array
+from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library, make_param_array, make_field_array, Context, Type
 from rpython.jit.metainterp.history import BoxInt, ConstInt
+from rpython.jit.metainterp.resoperation import *
+from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref, cast_object_to_ptr
 from rpython.rtyper.lltypesystem.rffi import *
+from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
 
 class AssemblerLibgccjit(BaseAssembler):
     _regalloc = None
@@ -30,29 +36,36 @@
         self.num_anon_loops = 0
 
         self.make_context()
-        print(self.ctxt)
-        self.t_int = self.lib.gcc_jit_context_get_type(self.ctxt,
-                                                       self.lib.GCC_JIT_TYPE_INT)
+        self.t_int = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_INT)
+        self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
 
     def make_context(self):
         eci = make_eci()
         self.lib = Library(eci)
-        self.ctxt = self.lib.gcc_jit_context_acquire()
-        self.lib.gcc_jit_context_set_bool_option(self.ctxt,
-                                            self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
-                                            r_int(1))
-        self.lib.gcc_jit_context_set_int_option(self.ctxt,
-                                           self.lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
-                                           r_int(3))
-        self.lib.gcc_jit_context_set_bool_option(self.ctxt,
-                                            self.lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
-                                            r_int(1))
-        self.lib.gcc_jit_context_set_bool_option(self.ctxt,
-                                            self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
-                                            r_int(1))
-        self.lib.gcc_jit_context_set_bool_option(self.ctxt,
-                                            self.lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
-                                            r_int(1))
+        self.ctxt = Context.acquire(self.lib)#self.lib.gcc_jit_context_acquire()
+        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
+                                  r_int(1))
+        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DEBUGINFO,
+                                  r_int(1))
+        self.ctxt.set_int_option(self.lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
+                                 r_int(3))
+        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
+                                  r_int(1))
+        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
+                                  r_int(1))
+        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
+                                  r_int(1))
+        
+    def setup(self, looptoken):
+        allblocks = self.get_asmmemmgr_blocks(looptoken)
+        self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr,
+                                                        allblocks)
+
+    def get_asmmemmgr_blocks(self, looptoken):
+        clt = looptoken.compiled_loop_token
+        if clt.asmmemmgr_blocks is None:
+            clt.asmmemmgr_blocks = []
+        return clt.asmmemmgr_blocks
 
     def assemble_loop(self, inputargs, operations, looptoken, log,
                       loopname, logger):
@@ -62,10 +75,94 @@
         looptoken.compiled_loop_token = clt
         clt._debug_nbargs = len(inputargs)
 
+        self.setup(looptoken)
+
+        frame_info = self.datablockwrapper.malloc_aligned(
+            jitframe.JITFRAMEINFO_SIZE, alignment=64) #WORD)
+        clt.frame_info = rffi.cast(jitframe.JITFRAMEINFOPTR, frame_info)
+        clt.allgcrefs = []
+        clt.frame_info.clear() # for now
+
+        self.lvalue_for_box = {}
+
+        print(jitframe.JITFRAME)
+        print(dir(jitframe.JITFRAME))
+        print('jitframe.JITFRAME._flds: %r' % jitframe.JITFRAME._flds)
+
+        # For now, build a "struct jit_frame".
+        # This will have the fields of jitframe, but instead of 
+        # jf_frame, we'll have any input and output boxes:
+        visible_boxes = []
+        for arg in inputargs:
+            visible_boxes.append(arg)
+        for op in operations:
+            if op.getopname() == 'finish':
+                for arg in op._args:
+                    if arg not in visible_boxes:
+                        visible_boxes.append(arg)
+        print('visible_boxes: %r' % visible_boxes)
+
+        fields = []
+        def make_field(name, jit_type):
+            field = self.ctxt.new_field(jit_type,name)
+            fields.append(field)
+            return field
+        make_field('jfi_frame_depth', self.t_int)
+        make_field('jfi_frame_size', self.t_int)
+
+        t_JITFRAMEINFO = (
+            self.ctxt.new_struct_type ("JITFRAMEINFO",
+                                       fields).as_type())
+
+        t_JITFRAMEINFOPTR = t_JITFRAMEINFO.get_pointer()
+
+        struct_jit_frame = self.ctxt.new_opaque_struct ("JITFRAME")
+
+        t_jit_frame_ptr = struct_jit_frame.as_type().get_pointer()
+
+        fields = []
+        # FIXME: Does the GCStruct implicitly add any fields?
+        # If I put this here, then the arguments appear to be written to the
+        # place where I expect:
+        make_field('hack_rtti', self.t_void_ptr)
+        make_field('jf_frame_info', t_JITFRAMEINFOPTR)
+        self.field_jf_descr = make_field('jf_descr', self.t_void_ptr)
+        make_field('jf_force_descr', self.t_void_ptr)
+        make_field('jf_gcmap', self.t_void_ptr)
+        make_field('jf_extra_stack_depth', self.t_int)
+        make_field('jf_savedata', self.t_void_ptr)
+        make_field('jf_guard_exc', self.t_void_ptr)
+        make_field('jf_forward', t_jit_frame_ptr)
+        # jf_frame:
+        self.field_for_box = {}
+        locs = []
+        #loc = jitframe.getofs('jf_frame')
+        for loc, box in enumerate(visible_boxes):
+            field = make_field(str(box), self.t_int)
+            locs.append(loc) # hack!
+            self.field_for_box[box] = field
+
+        struct_jit_frame.set_fields (fields)
+
         # Make function:
-        self.lvalue_for_box = {}
         print('  inputargs: %r' % (inputargs, ))
+        #jitframe.JITFRAMEINFOPTR
         params = []
+
+        self.param_frame = self.ctxt.new_param(t_jit_frame_ptr, "jitframe")
+        params.append(self.param_frame)
+
+        self.param_addr = self.ctxt.new_param(self.t_void_ptr, "addr")
+        params.append(self.param_addr)
+
+        # For now, generate lvalues for the "visible boxes" directly as
+        # field lookups within the jit_frame:
+        for box in visible_boxes:
+            self.lvalue_for_box[box] = (
+                self.param_frame.as_rvalue().
+                  dereference_field (self.field_for_box[box]))
+
+        """
         for arg in inputargs:
             param_name = str2charp(str(arg))
             param = self.lib.gcc_jit_context_new_param(self.ctxt,
@@ -75,56 +172,44 @@
             self.lvalue_for_box[arg] = self.lib.gcc_jit_param_as_lvalue(param)
             free_charp(param_name)
             params.append(param)
+        """
         
         print("loopname: %r" % loopname)
         if not loopname:
             loopname = 'anonloop_%i' % self.num_anon_loops
             self.num_anon_loops += 1
-        fn_name = str2charp(loopname)
-        param_array = make_param_array(self.lib, params)
-        self.fn = self.lib.gcc_jit_context_new_function(self.ctxt,
-                                                        self.lib.null_location_ptr,
-                                                        self.lib.GCC_JIT_FUNCTION_EXPORTED,
-                                                        self.t_int,
-                                                        fn_name,
-                                                        r_int(len(params)),
-                                                        param_array,
-                                                        r_int(0))
-        lltype.free(param_array, flavor='raw')
-        free_charp(fn_name)
+        self.fn = self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_EXPORTED,
+                                         t_jit_frame_ptr, # self.t_int,
+                                         loopname,
+                                         params,
+                                         r_int(0))
 
-        self.b_current = self.lib.gcc_jit_function_new_block(self.fn, NULL)
+        self.b_current = self.fn.new_block()
 
         for op in operations:
-            """
             print(op)
             print(type(op))
             print(dir(op))
             print(repr(op.getopname()))
-            """
             # Add a comment describing this ResOperation
-            comment_text = str2charp(str(op))
-            self.lib.gcc_jit_block_add_comment(self.b_current,
-                                               self.lib.null_location_ptr,
-                                               comment_text)
-            free_charp(comment_text)
+            self.b_current.add_comment(str(op))
 
             # Compile the operation itself...
-            methname = '_on_%s' % op.getopname()
+            methname = '_emit_%s' % op.getopname()
             getattr(self, methname) (op)
 
-        jit_result = self.lib.gcc_jit_context_compile(self.ctxt)
-        self.lib.gcc_jit_context_release(self.ctxt)
-        if not jit_result:
-            # FIXME: get error from context
-            raise Exception("jit_result is NULL")
+        
+        self.ctxt.dump_to_file("/tmp/foo.c", r_int(1))
 
-        fn_name = str2charp(loopname)
-        fn_ptr = self.lib.gcc_jit_result_get_code(jit_result, fn_name)
-        free_charp(fn_name)
+        jit_result = self.ctxt.compile()
+        self.ctxt.release()
+
+        fn_ptr = jit_result.get_code(loopname)
 
         looptoken._ll_function_addr = fn_ptr
 
+        looptoken.compiled_loop_token._ll_initial_locs = locs
+        
         # FIXME: this leaks the gcc_jit_result
 
     def expr_to_rvalue(self, expr):
@@ -138,24 +223,20 @@
         """
 
         if isinstance(expr, BoxInt):
-            return self.lib.gcc_jit_lvalue_as_rvalue(self.get_box_as_lvalue(expr))
+            return self.get_box_as_lvalue(expr).as_rvalue()
         elif isinstance(expr, ConstInt):
             #print('value: %r' % expr.value)
             #print('type(value): %r' % type(expr.value))
-            return self.lib.gcc_jit_context_new_rvalue_from_int(self.ctxt,
-                                                                self.t_int,
-                                                                r_int(expr.value))
+            return self.ctxt.new_rvalue_from_int(self.t_int,
+                                                 r_int(expr.value))
         raise ValueError('unhandled expr: %s' % expr)
 
     def get_box_as_lvalue(self, box):
         if box not in self.lvalue_for_box:
-            local_name = str2charp(str(box))
+            raise foo
             self.lvalue_for_box[box] = (
-                self.lib.gcc_jit_function_new_local(self.fn,
-                                                    self.lib.null_location_ptr,
-                                                    self.t_int, # FIXME: use correct type
-                                                    local_name))
-            free_charp(local_name)
+                self.fn.new_local(self.t_int, # FIXME: use correct type
+                                  local_name))
         return self.lvalue_for_box[box]
 
     def expr_to_lvalue(self, expr):
@@ -173,7 +254,7 @@
 
     # Handling of specific ResOperation subclasses
 
-    def _on_int_add(self, op):
+    def _emit_int_add(self, op):
         """
         print(op._arg0)
         print(op._arg1)
@@ -185,24 +266,21 @@
         lvalres = self.expr_to_lvalue(op.result)
 
         op_add = (
-            self.lib.gcc_jit_context_new_binary_op(self.ctxt,
-                                                   self.lib.null_location_ptr,
-                                                   self.lib.GCC_JIT_BINARY_OP_PLUS,
-                                                   self.t_int,
-                                                   rval0, rval1))
-        #self.lib.gcc_jit_object_get_debug_string(gcc_jit_rvalue_as_object(op_add))
-        self.lib.gcc_jit_block_add_assignment(self.b_current,
-                                              self.lib.null_location_ptr,
-                                              lvalres,
-                                              op_add)
+            self.ctxt.new_binary_op(self.lib.GCC_JIT_BINARY_OP_PLUS,
+                                    self.t_int,
+                                    rval0, rval1))
+        self.b_current.add_assignment(lvalres, op_add)
 
-    def _on_finish(self, op):
-        """
-        print(op.__dict__)
-        """
-        # FIXME: assume just 1-ary FINISH for now
-        assert len(op._args) == 1
-        result = op._args[0]
-        self.lib.gcc_jit_block_end_with_return(self.b_current,
-                                               self.lib.null_location_ptr,
-                                               self.expr_to_rvalue(result))
+    def _emit_finish(self, op):
+        # Write back to the jf_descr:
+        #  "jitframe->jf_descr = op.getdescr();"
+        descr = rffi.cast(lltype.Signed,
+                          cast_instance_to_gcref(op.getdescr()))
+
+        self.b_current.add_assignment(
+            self.param_frame.as_rvalue ().dereference_field (
+                self.field_jf_descr),
+            self.ctxt.new_rvalue_from_ptr (self.t_void_ptr,
+                                           rffi.cast(VOIDP, descr)))
+
+        self.b_current.end_with_return(self.param_frame.as_rvalue ())
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -351,6 +351,13 @@
             raise Exception("result is NULL")
         return Result(self.lib, inner_result)
 
+    def dump_to_file(self, path, update_locations):
+        path_charp = str2charp(path)
+        self.lib.gcc_jit_context_dump_to_file(self.inner_ctxt,
+                                              path_charp,
+                                              update_locations)
+        free_charp(path_charp)
+
     def get_type(self, r_enum):
         return Type(self.lib,
                     self.lib.gcc_jit_context_get_type(self.inner_ctxt,
@@ -364,13 +371,45 @@
                                                    name_charp)
         free_charp(name_charp)
         return Field(self.lib, field)
+
+    def new_struct_type(self, name, fields):
+        name_charp = str2charp(name)
+        field_array = lltype.malloc(self.lib.FIELD_P_P.TO,
+                                    len(fields),
+                                    flavor='raw') # of maybe gc?
+        for i in range(len(fields)):
+            field_array[i] = fields[i].inner_field
+        inner_struct = (
+            self.lib.gcc_jit_context_new_struct_type(self.inner_ctxt,
+                                                     self.lib.null_location_ptr,
+                                                     name_charp,
+                                                     r_int(len(fields)),
+                                                     field_array))
+        lltype.free(field_array, flavor='raw')
+        free_charp(name_charp)
+        return Struct(self.lib, inner_struct)
     
+    def new_opaque_struct(self, name):
+        name_charp = str2charp(name)
+        inner_struct = (
+            self.lib.gcc_jit_context_new_opaque_struct(self.inner_ctxt,
+                                                       self.lib.null_location_ptr,
+                                                       name_charp))
+        free_charp(name_charp)
+        return Struct(self.lib, inner_struct)
+
     def new_rvalue_from_int(self, type_, llvalue):
         return RValue(self.lib,
                       self.lib.gcc_jit_context_new_rvalue_from_int(self.inner_ctxt,
                                                                    type_.inner_type,
                                                                    llvalue))
 
+    def new_rvalue_from_ptr(self, type_, llvalue):
+        return RValue(self.lib,
+                      self.lib.gcc_jit_context_new_rvalue_from_ptr(self.inner_ctxt,
+                                                                   type_.inner_type,
+                                                                   llvalue))
+
     def new_binary_op(self, op, type_, a, b):
         return RValue(self.lib,
                       self.lib.gcc_jit_context_new_binary_op(self.inner_ctxt,
@@ -415,17 +454,48 @@
         Wrapper.__init__(self, lib)
         self.inner_type = inner_type
 
+    def get_pointer(self):
+        return Type(self.lib,
+                    self.lib.gcc_jit_type_get_pointer(self.inner_type))
 
 class Field(Wrapper):
     def __init__(self, lib, inner_field):
         Wrapper.__init__(self, lib)
         self.inner_field = inner_field
 
+class Struct(Wrapper):
+    def __init__(self, lib, inner_struct):
+        Wrapper.__init__(self, lib)
+        self.inner_struct = inner_struct
+
+    def as_type(self):
+        return Type(self.lib,
+                    self.lib.gcc_jit_struct_as_type(self.inner_struct))
+
+
+    def set_fields(self, fields):
+        field_array = lltype.malloc(self.lib.FIELD_P_P.TO,
+                                    len(fields),
+                                    flavor='raw') # of maybe gc?
+        for i in range(len(fields)):
+            field_array[i] = fields[i].inner_field
+        self.lib.gcc_jit_struct_set_fields(self.inner_struct,
+                                           self.lib.null_location_ptr,
+                                           r_int(len(fields)),
+                                           field_array)
+        lltype.free(field_array, flavor='raw')
+
 class RValue(Wrapper):
     def __init__(self, lib, inner_rvalue):
         Wrapper.__init__(self, lib)
         self.inner_rvalue = inner_rvalue
 
+    def dereference_field(self, field):
+        return LValue(self.lib,
+                      self.lib.gcc_jit_rvalue_dereference_field(self.inner_rvalue,
+                                                                self.lib.null_location_ptr,
+                                                                field.inner_field))
+
 class LValue(Wrapper):
     def __init__(self, lib, inner_lvalue):
         Wrapper.__init__(self, lib)
@@ -458,11 +528,15 @@
         free_charp(name_charp)
         return LValue(self.lib, local)
 
-    def new_block(self, name):
-        name_charp = str2charp(name)
+    def new_block(self, name=None):
+        if name is not None:
+            name_charp = str2charp(name)
+        else:
+            name_charp = NULL
         block = self.lib.gcc_jit_function_new_block(self.inner_function,
                                                     name_charp)
-        free_charp(name_charp)
+        if name_charp:
+            free_charp(name_charp)
         return Block(self.lib, block)
 
 class Block(Wrapper):
@@ -476,6 +550,13 @@
                                               lvalue.inner_lvalue,
                                               rvalue.inner_rvalue)
 
+    def add_comment(self, text):
+        text_charp = str2charp(text)
+        self.lib.gcc_jit_block_add_comment(self.inner_block,
+                                           self.lib.null_location_ptr,
+                                           text_charp)
+        free_charp(text_charp)
+
     def end_with_return(self, rvalue):
         self.lib.gcc_jit_block_end_with_return(self.inner_block,
                                                self.lib.null_location_ptr,

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:27 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:27 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Set _nowrapper=True when
 creating rffi bindings for libgccjit
Message-ID: <20141217164827.ED2DA1C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74974:52abcc39fd47
Date: 2014-12-15 11:10 -0500
http://bitbucket.org/pypy/pypy/changeset/52abcc39fd47/

Log:	Set _nowrapper=True when creating rffi bindings for libgccjit

diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -238,12 +238,13 @@
     def add_entrypoint(self, returntype, name, paramtypes):
         setattr(self, name,
                 llexternal(name, paramtypes, returntype,
-                           compilation_info=self.eci))
+                           compilation_info=self.eci,
+                           _nowrapper=True))
 
     def make_enum_values(self, lines):
         for value, name in enumerate(lines.split(',')):
             name = name.strip()
             if name:
-                setattr(self, name, value)
+                setattr(self, name, r_int(value))
         
 
diff --git a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
@@ -73,42 +73,50 @@
 
         lib.gcc_jit_context_set_bool_option(ctxt,
                                             lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
-                                            1)
+                                            r_int(1))
         lib.gcc_jit_context_set_int_option(ctxt,
                                            lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
-                                           3)
+                                           r_int(3))
         lib.gcc_jit_context_set_bool_option(ctxt,
                                             lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
-                                            1)
+                                            r_int(1))
         lib.gcc_jit_context_set_bool_option(ctxt,
                                             lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
-                                            1)
+                                            r_int(1))
         lib.gcc_jit_context_set_bool_option(ctxt,
                                             lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
-                                            1)
+                                            r_int(1))
         t_int = lib.gcc_jit_context_get_type(ctxt, lib.GCC_JIT_TYPE_INT)
+        param_name = str2charp("input")
         param = lib.gcc_jit_context_new_param(ctxt,
                                               lib.null_location_ptr,
                                               t_int,
-                                              "input")
+                                              param_name)
+        free_charp(param_name)
         # FIXME: how to build an array of params at this level?
         # see liststr2charpp in rffi.py
         
+        fn_name = str2charp("add_one_to")
         param_array = make_param_array(lib, [param])
         fn = lib.gcc_jit_context_new_function(ctxt,
                                               lib.null_location_ptr,
                                               lib.GCC_JIT_FUNCTION_EXPORTED,
                                               t_int,
-                                              "add_one_to",
-                                              1, param_array, 0)
+                                              fn_name,
+                                              r_int(1), param_array, r_int(0))
         lltype.free(param_array, flavor='raw')
+        free_charp(fn_name)
 
+        local_name = str2charp("v_res")
         v_res = lib.gcc_jit_function_new_local(fn,
                                                lib.null_location_ptr,
                                                t_int,
-                                               "v_res")
-        b_initial = lib.gcc_jit_function_new_block(fn, "initial")
-        c_one = lib.gcc_jit_context_new_rvalue_from_int(ctxt, t_int, 1)
+                                               local_name)
+        free_charp(local_name)
+        block_name = str2charp("initial")
+        b_initial = lib.gcc_jit_function_new_block(fn, block_name)
+        free_charp(block_name)
+        c_one = lib.gcc_jit_context_new_rvalue_from_int(ctxt, t_int, r_int(1))
         op_add = lib.gcc_jit_context_new_binary_op(ctxt,
                                                    lib.null_location_ptr,
                                                    lib.GCC_JIT_BINARY_OP_PLUS,
@@ -127,7 +135,9 @@
             # FIXME: get error from context
             raise Exception("jit_result is NULL")
 
-        fn_ptr = lib.gcc_jit_result_get_code(jit_result, "add_one_to")
+        fn_name = str2charp("add_one_to")
+        fn_ptr = lib.gcc_jit_result_get_code(jit_result, fn_name)
+        free_charp(fn_name)
         if not fn_ptr:
             raise Exception("fn_ptr is NULL")
         print('fn_ptr: %s' % fn_ptr)
@@ -154,7 +164,7 @@
         
     f1 = compile_c(f, [], backendopt=False)
     assert f1() == 42
-    #assert False # to see stderr
+    assert False # to see stderr
     
 # TODO: test of an error
 # should turn it into an exception, and capture the error

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:34 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:34 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_compile_linear_loop
	working
Message-ID: <20141217164834.B67661C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74980:0b696e11a782
Date: 2014-12-16 11:39 -0500
http://bitbucket.org/pypy/pypy/changeset/0b696e11a782/

Log:	Get test_compile_linear_loop working

	I see occasional glibc reports about corrupt heap on exit

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -36,25 +36,28 @@
         self.num_anon_loops = 0
 
         self.make_context()
-        self.t_int = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_INT)
+        self.t_long = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_LONG)
         self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
 
     def make_context(self):
         eci = make_eci()
         self.lib = Library(eci)
         self.ctxt = Context.acquire(self.lib)#self.lib.gcc_jit_context_acquire()
-        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
-                                  r_int(1))
+        if 0:
+            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
+                                      r_int(1))
         self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DEBUGINFO,
                                   r_int(1))
-        self.ctxt.set_int_option(self.lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
-                                 r_int(3))
+        if 1:
+            self.ctxt.set_int_option(self.lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
+                                     r_int(3))
         self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
                                   r_int(1))
         self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
                                   r_int(1))
-        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
-                                  r_int(1))
+        if 0:
+            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
+                                      r_int(1))
         
     def setup(self, looptoken):
         allblocks = self.get_asmmemmgr_blocks(looptoken)
@@ -90,25 +93,17 @@
         print('jitframe.JITFRAME._flds: %r' % jitframe.JITFRAME._flds)
 
         # For now, build a "struct jit_frame".
-        # This will have the fields of jitframe, but instead of 
-        # jf_frame, we'll have any input and output boxes:
-        visible_boxes = []
-        for arg in inputargs:
-            visible_boxes.append(arg)
-        for op in operations:
-            if op.getopname() == 'finish':
-                for arg in op._args:
-                    if arg not in visible_boxes:
-                        visible_boxes.append(arg)
-        print('visible_boxes: %r' % visible_boxes)
+        # This will have the fields of jitframe,
+        # We'll construct numbered arg0..argN for use
+        # for inputargs and the outputs
 
         fields = []
         def make_field(name, jit_type):
             field = self.ctxt.new_field(jit_type,name)
             fields.append(field)
             return field
-        make_field('jfi_frame_depth', self.t_int)
-        make_field('jfi_frame_size', self.t_int)
+        make_field('jfi_frame_depth', self.t_long)
+        make_field('jfi_frame_size', self.t_long)
 
         t_JITFRAMEINFO = (
             self.ctxt.new_struct_type ("JITFRAMEINFO",
@@ -122,25 +117,36 @@
 
         fields = []
         # FIXME: Does the GCStruct implicitly add any fields?
-        # If I put this here, then the arguments appear to be written to the
-        # place where I expect:
-        make_field('hack_rtti', self.t_void_ptr)
+        # If I omit this, then the jf_* fields appears in the same
+        # place in llmodel.py's get_latest_descr deadframe as
+        # in my generated code as seen in gdb.
+        #make_field('hack_rtti', self.t_void_ptr)
         make_field('jf_frame_info', t_JITFRAMEINFOPTR)
         self.field_jf_descr = make_field('jf_descr', self.t_void_ptr)
         make_field('jf_force_descr', self.t_void_ptr)
         make_field('jf_gcmap', self.t_void_ptr)
-        make_field('jf_extra_stack_depth', self.t_int)
+        make_field('jf_extra_stack_depth', self.t_long)
         make_field('jf_savedata', self.t_void_ptr)
         make_field('jf_guard_exc', self.t_void_ptr)
         make_field('jf_forward', t_jit_frame_ptr)
-        # jf_frame:
-        self.field_for_box = {}
+        # FIXME: for some reason there's an implicit word here;
+        # create it
+        make_field('jf_frame', self.t_long)
+
         locs = []
         #loc = jitframe.getofs('jf_frame')
-        for loc, box in enumerate(visible_boxes):
-            field = make_field(str(box), self.t_int)
-            locs.append(loc) # hack!
-            self.field_for_box[box] = field
+
+        max_args = len(inputargs)
+        for op in operations:
+            if op.getopname() == 'finish':
+                max_args = max(max_args, len(op._args))
+            # FIXME: other ops
+
+        self.field_for_arg_idx = {}
+        for idx in range(max_args):
+            self.field_for_arg_idx[idx] = make_field("arg%i" % idx,
+                                                     self.t_long)
+            locs.append(idx) # hack!
 
         struct_jit_frame.set_fields (fields)
 
@@ -155,37 +161,25 @@
         self.param_addr = self.ctxt.new_param(self.t_void_ptr, "addr")
         params.append(self.param_addr)
 
-        # For now, generate lvalues for the "visible boxes" directly as
-        # field lookups within the jit_frame:
-        for box in visible_boxes:
-            self.lvalue_for_box[box] = (
-                self.param_frame.as_rvalue().
-                  dereference_field (self.field_for_box[box]))
-
-        """
-        for arg in inputargs:
-            param_name = str2charp(str(arg))
-            param = self.lib.gcc_jit_context_new_param(self.ctxt,
-                                                       self.lib.null_location_ptr,
-                                                       self.t_int, # FIXME: use correct type
-                                                       param_name)
-            self.lvalue_for_box[arg] = self.lib.gcc_jit_param_as_lvalue(param)
-            free_charp(param_name)
-            params.append(param)
-        """
-        
         print("loopname: %r" % loopname)
         if not loopname:
             loopname = 'anonloop_%i' % self.num_anon_loops
             self.num_anon_loops += 1
         self.fn = self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_EXPORTED,
-                                         t_jit_frame_ptr, # self.t_int,
+                                         t_jit_frame_ptr, # self.t_long,
                                          loopname,
                                          params,
                                          r_int(0))
 
         self.b_current = self.fn.new_block()
 
+        # Get initial values from input args:
+        for idx, arg in enumerate(inputargs):
+            self.b_current.add_comment("inputargs[%i]: %s" % (idx, arg))
+            self.b_current.add_assignment(
+                self.get_box_as_lvalue(arg),
+                self.get_arg_as_lvalue(idx).as_rvalue())
+
         for op in operations:
             print(op)
             print(type(op))
@@ -227,18 +221,20 @@
         elif isinstance(expr, ConstInt):
             #print('value: %r' % expr.value)
             #print('type(value): %r' % type(expr.value))
-            return self.ctxt.new_rvalue_from_int(self.t_int,
+            return self.ctxt.new_rvalue_from_int(self.t_long,
                                                  r_int(expr.value))
         raise ValueError('unhandled expr: %s' % expr)
 
     def get_box_as_lvalue(self, box):
         if box not in self.lvalue_for_box:
-            raise foo
-            self.lvalue_for_box[box] = (
-                self.fn.new_local(self.t_int, # FIXME: use correct type
-                                  local_name))
+            self.lvalue_for_box[box] = self.fn.new_local(self.t_long,
+                                                         str(box))
         return self.lvalue_for_box[box]
 
+    def get_arg_as_lvalue(self, idx):
+        return self.param_frame.as_rvalue ().dereference_field (
+            self.field_for_arg_idx[idx])
+
     def expr_to_lvalue(self, expr):
         """
         print('expr_to_lvalue')
@@ -267,11 +263,18 @@
 
         op_add = (
             self.ctxt.new_binary_op(self.lib.GCC_JIT_BINARY_OP_PLUS,
-                                    self.t_int,
+                                    self.t_long,
                                     rval0, rval1))
         self.b_current.add_assignment(lvalres, op_add)
 
     def _emit_finish(self, op):
+        # Write outputs back:
+        for idx, arg in enumerate(op._args):
+            #self.b_current.add_comment("  op._args[%i]: %s" % (idx, arg))
+            self.b_current.add_assignment(
+                self.get_arg_as_lvalue(idx),
+                self.get_box_as_lvalue(arg).as_rvalue())
+
         # Write back to the jf_descr:
         #  "jitframe->jf_descr = op.getdescr();"
         descr = rffi.cast(lltype.Signed,

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:29 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:29 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: WIP: Initial compilation of
 test_compile_linear_loop
Message-ID: <20141217164829.2651C1C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74975:5a904ca20a80
Date: 2014-12-15 12:51 -0500
http://bitbucket.org/pypy/pypy/changeset/5a904ca20a80/

Log:	WIP: Initial compilation of test_compile_linear_loop

	Tested via: ./pytest.py
	rpython/jit/backend/libgccjit/test/test_runner.py -k
	test_compile_linear_loop

	Doesn't yet run the compiled code

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -0,0 +1,181 @@
+from rpython.jit.backend.llsupport.assembler import BaseAssembler
+from rpython.jit.backend.model import CompiledLoopToken
+from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library, make_param_array
+from rpython.jit.metainterp.history import BoxInt, ConstInt
+from rpython.rtyper.lltypesystem.rffi import *
+
+class AssemblerLibgccjit(BaseAssembler):
+    _regalloc = None
+    #_output_loop_log = None
+    #_second_tmp_reg = ecx
+
+    DEBUG_FRAME_DEPTH = False
+
+    def __init__(self, cpu, translate_support_code=False):
+        BaseAssembler.__init__(self, cpu, translate_support_code)
+        #self.verbose = False
+        self.verbose = True
+        self.loop_run_counters = []
+        self.float_const_neg_addr = 0
+        self.float_const_abs_addr = 0
+        self.malloc_slowpath = 0
+        self.malloc_slowpath_varsize = 0
+        self.wb_slowpath = [0, 0, 0, 0, 0]
+        #self.setup_failure_recovery()
+        self.datablockwrapper = None
+        self.stack_check_slowpath = 0
+        self.propagate_exception_path = 0
+        #self.teardown()
+
+        self.num_anon_loops = 0
+
+        self.make_context()
+        print(self.ctxt)
+        self.t_int = self.lib.gcc_jit_context_get_type(self.ctxt,
+                                                       self.lib.GCC_JIT_TYPE_INT)
+
+    def make_context(self):
+        eci = make_eci()
+        self.lib = Library(eci)
+        self.ctxt = self.lib.gcc_jit_context_acquire()
+        self.lib.gcc_jit_context_set_bool_option(self.ctxt,
+                                            self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
+                                            r_int(1))
+        self.lib.gcc_jit_context_set_int_option(self.ctxt,
+                                           self.lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
+                                           r_int(3))
+        self.lib.gcc_jit_context_set_bool_option(self.ctxt,
+                                            self.lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
+                                            r_int(1))
+        self.lib.gcc_jit_context_set_bool_option(self.ctxt,
+                                            self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
+                                            r_int(1))
+        self.lib.gcc_jit_context_set_bool_option(self.ctxt,
+                                            self.lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
+                                            r_int(1))
+
+    def assemble_loop(self, inputargs, operations, looptoken, log,
+                      loopname, logger):
+        print('assemble_loop')
+        clt = CompiledLoopToken(self.cpu, looptoken.number)
+        print(clt)
+
+        # Make function:
+        self.lvalue_for_box = {}
+        print('  inputargs: %r' % (inputargs, ))
+        params = []
+        for arg in inputargs:
+            param_name = str2charp(str(arg))
+            param = self.lib.gcc_jit_context_new_param(self.ctxt,
+                                                       self.lib.null_location_ptr,
+                                                       self.t_int, # FIXME: use correct type
+                                                       param_name)
+            self.lvalue_for_box[arg] = self.lib.gcc_jit_param_as_lvalue(param)
+            free_charp(param_name)
+            params.append(param)
+        
+        print("loopname: %r" % loopname)
+        if not loopname:
+            loopname = 'anonloop_%i' % self.num_anon_loops
+            self.num_anon_loops += 1
+        fn_name = str2charp(loopname)
+        param_array = make_param_array(self.lib, params)
+        self.fn = self.lib.gcc_jit_context_new_function(self.ctxt,
+                                                        self.lib.null_location_ptr,
+                                                        self.lib.GCC_JIT_FUNCTION_EXPORTED,
+                                                        self.t_int,
+                                                        fn_name,
+                                                        r_int(len(params)),
+                                                        param_array,
+                                                        r_int(0))
+        lltype.free(param_array, flavor='raw')
+        free_charp(fn_name)
+
+        self.b_current = self.lib.gcc_jit_function_new_block(self.fn, NULL)
+
+        for op in operations:
+            print(op)
+            print(type(op))
+            print(dir(op))
+            print(repr(op.getopname()))
+            methname = '_on_%s' % op.getopname()
+            getattr(self, methname) (op)
+
+        jit_result = self.lib.gcc_jit_context_compile(self.ctxt)
+        self.lib.gcc_jit_context_release(self.ctxt)
+        if not jit_result:
+            # FIXME: get error from context
+            raise Exception("jit_result is NULL")
+        raise foo
+
+    def expr_to_rvalue(self, expr):
+        print('expr_to_rvalue')
+        print(' %s' % expr)
+        print(' %r' % expr)
+        print(' %s' % type(expr))
+        print(' %s' % dir(expr))
+        print(' %s' % expr.__dict__)
+
+        if isinstance(expr, BoxInt):
+            return self.lib.gcc_jit_lvalue_as_rvalue(self.get_box_as_lvalue(expr))
+        elif isinstance(expr, ConstInt):
+            #print('value: %r' % expr.value)
+            #print('type(value): %r' % type(expr.value))
+            return self.lib.gcc_jit_context_new_rvalue_from_int(self.ctxt,
+                                                                self.t_int,
+                                                                r_int(expr.value))
+
+    def get_box_as_lvalue(self, box):
+        if box not in self.lvalue_for_box:
+            local_name = str2charp(str(box))
+            self.lvalue_for_box[box] = (
+                self.lib.gcc_jit_function_new_local(self.fn,
+                                                    self.lib.null_location_ptr,
+                                                    self.t_int, # FIXME: use correct type
+                                                    local_name))
+            free_charp(local_name)
+        return self.lvalue_for_box[box]
+
+    def expr_to_lvalue(self, expr):
+        print('expr_to_lvalue')
+        print(' %s' % expr)
+        print(' %r' % expr)
+        print(' %s' % type(expr))
+        print(' %s' % dir(expr))
+        print(' %s' % expr.__dict__)
+        if isinstance(expr, BoxInt):
+            return self.get_box_as_lvalue(expr)
+        raise foo
+
+    # Handling of specific ResOperation subclasses
+
+    def _on_int_add(self, op):
+        print(op._arg0)
+        print(op._arg1)
+        print(op.result)
+        print(op.__dict__)
+
+        rval0 = self.expr_to_rvalue(op._arg0)
+        rval1 = self.expr_to_rvalue(op._arg1)
+        lvalres = self.expr_to_lvalue(op.result)
+
+        op_add = (
+            self.lib.gcc_jit_context_new_binary_op(self.ctxt,
+                                                   self.lib.null_location_ptr,
+                                                   self.lib.GCC_JIT_BINARY_OP_PLUS,
+                                                   self.t_int,
+                                                   rval0, rval1))
+        #self.lib.gcc_jit_object_get_debug_string(gcc_jit_rvalue_as_object(op_add))
+        self.lib.gcc_jit_block_add_assignment(self.b_current,
+                                              self.lib.null_location_ptr,
+                                              lvalres,
+                                              op_add)
+
+    def _on_finish(self, op):
+        print(op.__dict__)
+        # FIXME: assume just 1-ary FINISH for now
+        assert len(op._args) == 1
+        result = op._args[0]
+        self.lib.gcc_jit_block_end_with_return(self.b_current,
+                                               self.lib.null_location_ptr,
+                                               self.expr_to_rvalue(result))
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -37,6 +37,15 @@
                                   library_dirs=[LIB_DIR])
     return eci
 
+def make_param_array(lib, l):
+    array = lltype.malloc(lib.PARAM_P_P.TO,
+                          len(l),
+                          flavor='raw') # of maybe gc?
+    for i in range(len(l)):
+        array[i] = l[i]
+    return array
+    # FIXME: don't leak!
+
 class Library:
     def __init__(self, eci):
         self.eci = eci
diff --git a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
@@ -34,7 +34,7 @@
     assert xf() == 3
 """
 
-from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library
+from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library, make_param_array
 
 def test_compile_empty_context():
     eci = make_eci()
@@ -51,15 +51,6 @@
     f1 ()
     #assert False # to see stderr
 
-def make_param_array(lib, l):
-    array = lltype.malloc(lib.PARAM_P_P.TO,
-                          len(l),
-                          flavor='raw') # of maybe gc?
-    for i in range(len(l)):
-        array[i] = l[i]
-    return array
-    # FIXME: don't leak!
-    
 def test_compile_add_one_to():
     eci = make_eci()
 
diff --git a/rpython/jit/backend/libgccjit/test/test_runner.py b/rpython/jit/backend/libgccjit/test/test_runner.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/libgccjit/test/test_runner.py
@@ -0,0 +1,31 @@
+import py
+from rpython.jit.backend.detect_cpu import getcpuclass
+from rpython.jit.backend.test.runner_test import LLtypeBackendTest,\
+     boxfloat, constfloat
+from rpython.jit.metainterp.history import (BasicFailDescr, BasicFinalDescr,
+                                            BoxInt)
+from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.tool.oparser import parse
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper import rclass
+from rpython.rtyper.annlowlevel import llhelper
+from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.metainterp.history import JitCellToken, TargetToken
+from rpython.jit.codewriter import longlong
+
+
+CPU = getcpuclass()
+
+class FakeStats(object):
+    pass
+
+
+class TestLibGccjit(LLtypeBackendTest):
+
+    # for the individual tests see
+    # ====> ../../test/runner_test.py
+
+    def get_cpu(self):
+        cpu = CPU(rtyper=None, stats=FakeStats())
+        cpu.setup_once()
+        return cpu

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:35 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:35 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Add initial comment to IR.
	WIP on emit methods
Message-ID: <20141217164835.C90071C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74981:f58bfe1daf4d
Date: 2014-12-16 11:49 -0500
http://bitbucket.org/pypy/pypy/changeset/f58bfe1daf4d/

Log:	Add initial comment to IR. WIP on emit methods

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -173,6 +173,16 @@
 
         self.b_current = self.fn.new_block()
 
+        # Add an initial comment summarizing the loop
+        text = '\n\tinputargs: %s\n\n' % inputargs
+        for op in operations:
+            print(op)
+            print(type(op))
+            print(dir(op))
+            print(repr(op.getopname()))
+            text += '\t%s\n' % op
+        self.b_current.add_comment(str(text))
+    
         # Get initial values from input args:
         for idx, arg in enumerate(inputargs):
             self.b_current.add_comment("inputargs[%i]: %s" % (idx, arg))
@@ -189,7 +199,7 @@
             self.b_current.add_comment(str(op))
 
             # Compile the operation itself...
-            methname = '_emit_%s' % op.getopname()
+            methname = 'emit_%s' % op.getopname()
             getattr(self, methname) (op)
 
         
@@ -250,7 +260,7 @@
 
     # Handling of specific ResOperation subclasses
 
-    def _emit_int_add(self, op):
+    def emit_int_add(self, op):
         """
         print(op._arg0)
         print(op._arg1)
@@ -267,7 +277,25 @@
                                     rval0, rval1))
         self.b_current.add_assignment(lvalres, op_add)
 
-    def _emit_finish(self, op):
+    """
+    def emit_label(self, op):
+        print(op)
+        print(op.__dict__)
+
+    def emit_int_le(self, op):
+        print(op)
+        print(op.__dict__)
+
+    def emit_int_ge(self, op):
+        print(op)
+        print(op.__dict__)
+
+    def emit_guard_true(self, op):
+        print(op)
+        print(op.__dict__)
+    """
+
+    def emit_finish(self, op):
         # Write outputs back:
         for idx, arg in enumerate(op._args):
             #self.b_current.add_comment("  op._args[%i]: %s" % (idx, arg))

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:30 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:30 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: WIP: Add comments describing
 each ResOperation into the libgccjit IR
Message-ID: <20141217164830.547CA1C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74976:f6a3bd3f6dbb
Date: 2014-12-15 12:56 -0500
http://bitbucket.org/pypy/pypy/changeset/f6a3bd3f6dbb/

Log:	WIP: Add comments describing each ResOperation into the libgccjit IR

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -93,11 +93,19 @@
 
         self.b_current = self.lib.gcc_jit_function_new_block(self.fn, NULL)
 
-        for op in operations:
+        for op in operations:       
             print(op)
             print(type(op))
             print(dir(op))
             print(repr(op.getopname()))
+            # Add a comment describing this ResOperation
+            comment_text = str2charp(str(op))
+            self.lib.gcc_jit_block_add_comment(self.b_current,
+                                               self.lib.null_location_ptr,
+                                               comment_text)
+            free_charp(comment_text)
+
+            # Compile the operation itself...
             methname = '_on_%s' % op.getopname()
             getattr(self, methname) (op)
 
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -175,6 +175,10 @@
                                                   self.GCC_JIT_LVALUE_P,
                                                   self.GCC_JIT_RVALUE_P]),
                 (lltype.Void,
+                 'gcc_jit_block_add_comment', [self.GCC_JIT_BLOCK_P,
+                                               self.GCC_JIT_LOCATION_P,
+                                               CCHARP]),
+                (lltype.Void,
                  'gcc_jit_block_end_with_return', [self.GCC_JIT_BLOCK_P,
                                                    self.GCC_JIT_LOCATION_P,
                                                    self.GCC_JIT_RVALUE_P]),

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:36 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:36 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_compile_loop to work
Message-ID: <20141217164836.EB5051C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74982:e540b044a92d
Date: 2014-12-16 14:33 -0500
http://bitbucket.org/pypy/pypy/changeset/e540b044a92d/

Log:	Get test_compile_loop to work

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -37,13 +37,15 @@
 
         self.make_context()
         self.t_long = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_LONG)
+        self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
         self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
 
     def make_context(self):
         eci = make_eci()
         self.lib = Library(eci)
         self.ctxt = Context.acquire(self.lib)#self.lib.gcc_jit_context_acquire()
-        if 0:
+        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_TREE, r_int(1))
+        if 1:
             self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
                                       r_int(1))
         self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DEBUGINFO,
@@ -55,7 +57,7 @@
                                   r_int(1))
         self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
                                   r_int(1))
-        if 0:
+        if 1:
             self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
                                       r_int(1))
         
@@ -171,7 +173,9 @@
                                          params,
                                          r_int(0))
 
-        self.b_current = self.fn.new_block()
+        self.b_current = self.fn.new_block("initial")
+        self.label_for_descr = {}
+        self.block_for_label_descr = {}
 
         # Add an initial comment summarizing the loop
         text = '\n\tinputargs: %s\n\n' % inputargs
@@ -205,6 +209,8 @@
         
         self.ctxt.dump_to_file("/tmp/foo.c", r_int(1))
 
+        #raise foo
+
         jit_result = self.ctxt.compile()
         self.ctxt.release()
 
@@ -213,7 +219,7 @@
         looptoken._ll_function_addr = fn_ptr
 
         looptoken.compiled_loop_token._ll_initial_locs = locs
-        
+
         # FIXME: this leaks the gcc_jit_result
 
     def expr_to_rvalue(self, expr):
@@ -277,32 +283,95 @@
                                     rval0, rval1))
         self.b_current.add_assignment(lvalres, op_add)
 
-    """
     def emit_label(self, op):
         print(op)
         print(op.__dict__)
+        #print('op.getdescr(): %r' % op.getdescr())
+        #print('op.getdescr().__dict__: %r' % op.getdescr().__dict__)
+
+        b_new = self.fn.new_block(str(op))
+        self.block_for_label_descr[op.getdescr()] = b_new
+        self.label_for_descr[op.getdescr()] = op
+        self.b_current.end_with_jump(b_new)
+        self.b_current = b_new
+
+    def emit_jump(self, jumpop):
+        print(jumpop)
+        print(jumpop.__dict__)
+        label = self.label_for_descr[jumpop.getdescr()]
+        print(jumpop.getdescr())
+        print('label: %r' % label)
+
+        assert len(jumpop._args) == len(label._args)
+
+        # We need to write to the boxes listed in the label's args
+        # with the values from those listed in the jump's args.
+        # However, there are potential cases like JUMP(i0, i1) going to LABEL(i1, i0)
+        # where all such assignments happen "simultaneously".
+        # Hence we need to set up temporaries.
+        # First pass: capture the value of relevant boxes at the JUMP:
+        tmps = []
+        for i in range(len(jumpop._args)):
+            tmps.append(self.fn.new_local(self.t_long, "tmp%i" % i))
+            self.b_current.add_assignment(
+                tmps[i],
+                self.get_box_as_lvalue(jumpop._args[i]).as_rvalue())
+        # Second pass: write the values to the boxes at the LABEL:
+        for i in range(len(jumpop._args)):
+            self.b_current.add_assignment(
+                self.get_box_as_lvalue(label._args[i]),
+                tmps[i].as_rvalue())
+            
+        self.b_current.end_with_jump(self.block_for_label_descr[jumpop.getdescr()])
+
+    def impl_int_cmp(self, op, gcc_jit_comparison):
+        rval0 = self.expr_to_rvalue(op._arg0)
+        rval1 = self.expr_to_rvalue(op._arg1)
+        lvalres = self.expr_to_lvalue(op.result)
+        op_cmp = (
+            self.ctxt.new_cast(
+                self.ctxt.new_comparison(gcc_jit_comparison,
+                                         rval0, rval1),
+                self.t_long)
+            )
+        self.b_current.add_assignment(lvalres,
+                                      op_cmp)
 
     def emit_int_le(self, op):
-        print(op)
-        print(op.__dict__)
+        self.impl_int_cmp(op, self.lib.GCC_JIT_COMPARISON_LE)
 
     def emit_int_ge(self, op):
-        print(op)
-        print(op.__dict__)
+        self.impl_int_cmp(op, self.lib.GCC_JIT_COMPARISON_GE)
 
     def emit_guard_true(self, op):
         print(op)
         print(op.__dict__)
-    """
+        b_true = self.fn.new_block("on_true_at_%s" % op)
+        b_false = self.fn.new_block("on_false_at_%s" % op)
+        boolval = self.ctxt.new_cast(self.expr_to_rvalue(op._arg0),
+                                     self.t_bool)
+        self.b_current.end_with_conditional(boolval,
+                                            b_true, b_false)
 
-    def emit_finish(self, op):
+        self.b_current = b_false
+        self.impl_write_output_args(op._fail_args)
+        self.impl_write_jf_descr(op)
+        self.b_current.end_with_return(self.param_frame.as_rvalue ())
+        rd_locs = []
+        for idx, arg in enumerate(op._fail_args):
+            rd_locs.append(idx)
+        op.getdescr().rd_locs = rd_locs
+
+        self.b_current = b_true
+
+    def impl_write_output_args(self, args):
         # Write outputs back:
-        for idx, arg in enumerate(op._args):
-            #self.b_current.add_comment("  op._args[%i]: %s" % (idx, arg))
+        for idx, arg in enumerate(args):
             self.b_current.add_assignment(
                 self.get_arg_as_lvalue(idx),
                 self.get_box_as_lvalue(arg).as_rvalue())
 
+    def impl_write_jf_descr(self, op):
         # Write back to the jf_descr:
         #  "jitframe->jf_descr = op.getdescr();"
         descr = rffi.cast(lltype.Signed,
@@ -313,5 +382,8 @@
                 self.field_jf_descr),
             self.ctxt.new_rvalue_from_ptr (self.t_void_ptr,
                                            rffi.cast(VOIDP, descr)))
-
+    
+    def emit_finish(self, op):
+        self.impl_write_output_args(op._args)
+        self.impl_write_jf_descr(op)
         self.b_current.end_with_return(self.param_frame.as_rvalue ())
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -219,6 +219,19 @@
                                                    self.GCC_JIT_RVALUE_P,
                                                    self.GCC_JIT_RVALUE_P]),
 
+                (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_new_comparison', [self.GCC_JIT_CONTEXT_P,
+                                                    self.GCC_JIT_LOCATION_P,
+                                                    INT, # enum gcc_jit_binary_op op,
+                                                    self.GCC_JIT_RVALUE_P,
+                                                    self.GCC_JIT_RVALUE_P]),
+
+                (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_new_cast', [self.GCC_JIT_CONTEXT_P,
+                                              self.GCC_JIT_LOCATION_P,
+                                              self.GCC_JIT_RVALUE_P,
+                                              self.GCC_JIT_TYPE_P]),
+
                 (self.GCC_JIT_LVALUE_P,
                  'gcc_jit_rvalue_dereference_field', [self.GCC_JIT_RVALUE_P,
                                                       self.GCC_JIT_LOCATION_P,
@@ -237,6 +250,16 @@
                                                self.GCC_JIT_LOCATION_P,
                                                CCHARP]),
                 (lltype.Void,
+                 'gcc_jit_block_end_with_conditional', [self.GCC_JIT_BLOCK_P,
+                                                        self.GCC_JIT_LOCATION_P,
+                                                        self.GCC_JIT_RVALUE_P,
+                                                        self.GCC_JIT_BLOCK_P,
+                                                        self.GCC_JIT_BLOCK_P]),
+                (lltype.Void,
+                 'gcc_jit_block_end_with_jump', [self.GCC_JIT_BLOCK_P,
+                                                 self.GCC_JIT_LOCATION_P,
+                                                 self.GCC_JIT_BLOCK_P]),
+                (lltype.Void,
                  'gcc_jit_block_end_with_return', [self.GCC_JIT_BLOCK_P,
                                                    self.GCC_JIT_LOCATION_P,
                                                    self.GCC_JIT_RVALUE_P]),
@@ -303,6 +326,16 @@
             GCC_JIT_BINARY_OP_RSHIFT
             """)
 
+        self.make_enum_values(
+            """
+            GCC_JIT_COMPARISON_EQ,
+            GCC_JIT_COMPARISON_NE,
+            GCC_JIT_COMPARISON_LT,
+            GCC_JIT_COMPARISON_LE,
+            GCC_JIT_COMPARISON_GT,
+            GCC_JIT_COMPARISON_GE
+            """)
+
         self.null_location_ptr = lltype.nullptr(self.GCC_JIT_LOCATION_P.TO)
 
 
@@ -418,6 +451,13 @@
                                                              type_.inner_type,
                                                              a.inner_rvalue, b.inner_rvalue))
 
+    def new_comparison(self, op, a, b):
+        return RValue(self.lib,
+                      self.lib.gcc_jit_context_new_comparison(self.inner_ctxt,
+                                                              self.lib.null_location_ptr,
+                                                              op,
+                                                              a.inner_rvalue, b.inner_rvalue))
+
     def new_param(self, type_, name):
         name_charp = str2charp(name)
         param = self.lib.gcc_jit_context_new_param(self.inner_ctxt,
@@ -448,6 +488,12 @@
 
         return Function(self.lib, fn)
 
+    def new_cast(self, rvalue, type_):
+        return RValue(self.lib,
+                      self.lib.gcc_jit_context_new_cast(self.inner_ctxt,
+                                                        self.lib.null_location_ptr,
+                                                        rvalue.inner_rvalue,
+                                                        type_.inner_type))
 
 class Type(Wrapper):
     def __init__(self, lib, inner_type):
@@ -557,6 +603,18 @@
                                            text_charp)
         free_charp(text_charp)
 
+    def end_with_conditional(self, boolval, on_true, on_false):
+        self.lib.gcc_jit_block_end_with_conditional(self.inner_block,
+                                                    self.lib.null_location_ptr,
+                                                    boolval.inner_rvalue,
+                                                    on_true.inner_block,
+                                                    on_false.inner_block)
+
+    def end_with_jump (self, target):
+        self.lib.gcc_jit_block_end_with_jump (self.inner_block,
+                                              self.lib.null_location_ptr,
+                                              target.inner_block)
+
     def end_with_return(self, rvalue):
         self.lib.gcc_jit_block_end_with_return(self.inner_block,
                                                self.lib.null_location_ptr,

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:38 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:38 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get parts of
	test_int_operations to work
Message-ID: <20141217164838.0A3681C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74983:c46f9a7b13de
Date: 2014-12-17 00:39 -0500
http://bitbucket.org/pypy/pypy/changeset/c46f9a7b13de/

Log:	Get parts of test_int_operations to work

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -35,16 +35,15 @@
 
         self.num_anon_loops = 0
 
-        self.make_context()
-        self.t_long = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_LONG)
-        self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
-        self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
+        self.sizeof_signed = rffi.sizeof(lltype.Signed)
 
     def make_context(self):
         eci = make_eci()
         self.lib = Library(eci)
         self.ctxt = Context.acquire(self.lib)#self.lib.gcc_jit_context_acquire()
-        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_TREE, r_int(1))
+        if 0:
+            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_TREE,
+                                      r_int(1))
         if 1:
             self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
                                       r_int(1))
@@ -52,7 +51,7 @@
                                   r_int(1))
         if 1:
             self.ctxt.set_int_option(self.lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
-                                     r_int(3))
+                                     r_int(2))
         self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
                                   r_int(1))
         self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
@@ -88,6 +87,11 @@
         clt.allgcrefs = []
         clt.frame_info.clear() # for now
 
+        self.make_context()
+        self.t_long = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_LONG)
+        self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
+        self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
+
         self.lvalue_for_box = {}
 
         print(jitframe.JITFRAME)
@@ -135,20 +139,22 @@
         # create it
         make_field('jf_frame', self.t_long)
 
-        locs = []
+        initial_locs = []
         #loc = jitframe.getofs('jf_frame')
 
         max_args = len(inputargs)
         for op in operations:
             if op.getopname() == 'finish':
                 max_args = max(max_args, len(op._args))
+            if op.getopname() == 'guard_true':
+                max_args = max(max_args, len(op._fail_args))
             # FIXME: other ops
 
         self.field_for_arg_idx = {}
         for idx in range(max_args):
             self.field_for_arg_idx[idx] = make_field("arg%i" % idx,
                                                      self.t_long)
-            locs.append(idx) # hack!
+            initial_locs.append(idx * self.sizeof_signed) # hack!
 
         struct_jit_frame.set_fields (fields)
 
@@ -206,6 +212,10 @@
             methname = 'emit_%s' % op.getopname()
             getattr(self, methname) (op)
 
+        # Ensure that the frame is large enough
+        baseofs = self.cpu.get_baseofs_of_frame_field()
+        clt.frame_info.update_frame_depth(baseofs,
+                                          max_args)
         
         self.ctxt.dump_to_file("/tmp/foo.c", r_int(1))
 
@@ -218,7 +228,7 @@
 
         looptoken._ll_function_addr = fn_ptr
 
-        looptoken.compiled_loop_token._ll_initial_locs = locs
+        looptoken.compiled_loop_token._ll_initial_locs = initial_locs
 
         # FIXME: this leaks the gcc_jit_result
 
@@ -266,32 +276,37 @@
 
     # Handling of specific ResOperation subclasses
 
-    def emit_int_add(self, op):
-        """
-        print(op._arg0)
-        print(op._arg1)
-        print(op.result)
-        print(op.__dict__)
-        """
-        rval0 = self.expr_to_rvalue(op._arg0)
-        rval1 = self.expr_to_rvalue(op._arg1)
-        lvalres = self.expr_to_lvalue(op.result)
+    def impl_int_binop(self, resop, gcc_jit_binary_op):
+        rval0 = self.expr_to_rvalue(resop._arg0)
+        rval1 = self.expr_to_rvalue(resop._arg1)
+        lvalres = self.expr_to_lvalue(resop.result)
 
-        op_add = (
-            self.ctxt.new_binary_op(self.lib.GCC_JIT_BINARY_OP_PLUS,
-                                    self.t_long,
-                                    rval0, rval1))
-        self.b_current.add_assignment(lvalres, op_add)
+        binop_expr = self.ctxt.new_binary_op(gcc_jit_binary_op,
+                                             self.t_long,
+                                             rval0, rval1)
+        self.b_current.add_assignment(lvalres, binop_expr)
 
-    def emit_label(self, op):
-        print(op)
-        print(op.__dict__)
-        #print('op.getdescr(): %r' % op.getdescr())
-        #print('op.getdescr().__dict__: %r' % op.getdescr().__dict__)
+    def emit_int_add(self, resop):
+        self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_PLUS)
 
-        b_new = self.fn.new_block(str(op))
-        self.block_for_label_descr[op.getdescr()] = b_new
-        self.label_for_descr[op.getdescr()] = op
+    def emit_int_sub(self, resop):
+        self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_MINUS)
+
+    def emit_int_mul(self, resop):
+        self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_MULT)
+
+    def emit_int_floordiv(self, resop):
+        self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_DIVIDE)
+
+    def emit_label(self, resop):
+        print(resop)
+        print(resop.__dict__)
+        #print('resop.getdescr(): %r' % resop.getdescr())
+        #print('resop.getdescr().__dict__: %r' % resop.getdescr().__dict__)
+
+        b_new = self.fn.new_block(str(resop))
+        self.block_for_label_descr[resop.getdescr()] = b_new
+        self.label_for_descr[resop.getdescr()] = resop
         self.b_current.end_with_jump(b_new)
         self.b_current = b_new
 
@@ -324,58 +339,83 @@
             
         self.b_current.end_with_jump(self.block_for_label_descr[jumpop.getdescr()])
 
-    def impl_int_cmp(self, op, gcc_jit_comparison):
-        rval0 = self.expr_to_rvalue(op._arg0)
-        rval1 = self.expr_to_rvalue(op._arg1)
-        lvalres = self.expr_to_lvalue(op.result)
-        op_cmp = (
+    def impl_int_cmp(self, resop, gcc_jit_comparison):
+        rval0 = self.expr_to_rvalue(resop._arg0)
+        rval1 = self.expr_to_rvalue(resop._arg1)
+        lvalres = self.expr_to_lvalue(resop.result)
+        resop_cmp = (
             self.ctxt.new_cast(
                 self.ctxt.new_comparison(gcc_jit_comparison,
                                          rval0, rval1),
                 self.t_long)
             )
         self.b_current.add_assignment(lvalres,
-                                      op_cmp)
+                                      resop_cmp)
 
-    def emit_int_le(self, op):
-        self.impl_int_cmp(op, self.lib.GCC_JIT_COMPARISON_LE)
+    def emit_int_eq(self, resop):
+        self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_EQ)
 
-    def emit_int_ge(self, op):
-        self.impl_int_cmp(op, self.lib.GCC_JIT_COMPARISON_GE)
+    def emit_int_le(self, resop):
+        self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_LE)
 
-    def emit_guard_true(self, op):
-        print(op)
-        print(op.__dict__)
-        b_true = self.fn.new_block("on_true_at_%s" % op)
-        b_false = self.fn.new_block("on_false_at_%s" % op)
-        boolval = self.ctxt.new_cast(self.expr_to_rvalue(op._arg0),
+    def emit_int_ge(self, resop):
+        self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
+
+    def impl_guard(self, resop, istrue):
+        print(resop)
+        print(resop.__dict__)
+        b_true = self.fn.new_block("on_true_at_%s" % resop)
+        b_false = self.fn.new_block("on_false_at_%s" % resop)
+        boolval = self.ctxt.new_cast(self.expr_to_rvalue(resop._arg0),
                                      self.t_bool)
         self.b_current.end_with_conditional(boolval,
                                             b_true, b_false)
 
-        self.b_current = b_false
-        self.impl_write_output_args(op._fail_args)
-        self.impl_write_jf_descr(op)
+        if istrue:
+            self.b_current = b_false
+        else:
+            self.b_current = b_true
+        self.impl_write_output_args(resop._fail_args)
+        self.impl_write_jf_descr(resop)
         self.b_current.end_with_return(self.param_frame.as_rvalue ())
         rd_locs = []
-        for idx, arg in enumerate(op._fail_args):
-            rd_locs.append(idx)
-        op.getdescr().rd_locs = rd_locs
+        for idx, arg in enumerate(resop._fail_args):
+            rd_locs.append(idx * self.sizeof_signed)
+        resop.getdescr().rd_locs = rd_locs
 
-        self.b_current = b_true
+        if istrue:
+            self.b_current = b_true
+        else:
+            self.b_current = b_false
+
+    def emit_guard_true(self, resop):
+        self.impl_guard(resop, r_int(1))
+        
+    def emit_guard_false(self, resop):
+        self.impl_guard(resop, r_int(0))
 
     def impl_write_output_args(self, args):
         # Write outputs back:
         for idx, arg in enumerate(args):
-            self.b_current.add_assignment(
-                self.get_arg_as_lvalue(idx),
-                self.get_box_as_lvalue(arg).as_rvalue())
+            if arg is not None:
+                self.b_current.add_assignment(
+                    self.get_arg_as_lvalue(idx),
+                    self.get_box_as_lvalue(arg).as_rvalue())
+            else:
+                # FIXME: see test_compile_with_holes_in_fail_args
+                raise ValueError("how to handle holes in fail args?")
+                """
+                self.b_current.add_assignment(
+                    self.get_arg_as_lvalue(idx),
+                    self.ctxt.new_rvalue_from_int (self.t_long,
+                                                   r_int(0)))
+                """
 
-    def impl_write_jf_descr(self, op):
+    def impl_write_jf_descr(self, resop):
         # Write back to the jf_descr:
-        #  "jitframe->jf_descr = op.getdescr();"
+        #  "jitframe->jf_descr = resop.getdescr();"
         descr = rffi.cast(lltype.Signed,
-                          cast_instance_to_gcref(op.getdescr()))
+                          cast_instance_to_gcref(resop.getdescr()))
 
         self.b_current.add_assignment(
             self.param_frame.as_rvalue ().dereference_field (
@@ -383,7 +423,7 @@
             self.ctxt.new_rvalue_from_ptr (self.t_void_ptr,
                                            rffi.cast(VOIDP, descr)))
     
-    def emit_finish(self, op):
-        self.impl_write_output_args(op._args)
-        self.impl_write_jf_descr(op)
+    def emit_finish(self, resop):
+        self.impl_write_output_args(resop._args)
+        self.impl_write_jf_descr(resop)
         self.b_current.end_with_return(self.param_frame.as_rvalue ())

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:39 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:39 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: t_long -> t_Signed,
 and get the appropriately sized type
Message-ID: <20141217164839.22F9E1C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74984:b74f1446a2c7
Date: 2014-12-17 00:45 -0500
http://bitbucket.org/pypy/pypy/changeset/b74f1446a2c7/

Log:	t_long -> t_Signed, and get the appropriately sized type

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -88,7 +88,8 @@
         clt.frame_info.clear() # for now
 
         self.make_context()
-        self.t_long = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_LONG)
+        self.t_Signed = self.ctxt.get_int_type(r_int(self.sizeof_signed),
+                                               r_int(1))
         self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
         self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
 
@@ -108,8 +109,8 @@
             field = self.ctxt.new_field(jit_type,name)
             fields.append(field)
             return field
-        make_field('jfi_frame_depth', self.t_long)
-        make_field('jfi_frame_size', self.t_long)
+        make_field('jfi_frame_depth', self.t_Signed)
+        make_field('jfi_frame_size', self.t_Signed)
 
         t_JITFRAMEINFO = (
             self.ctxt.new_struct_type ("JITFRAMEINFO",
@@ -131,13 +132,13 @@
         self.field_jf_descr = make_field('jf_descr', self.t_void_ptr)
         make_field('jf_force_descr', self.t_void_ptr)
         make_field('jf_gcmap', self.t_void_ptr)
-        make_field('jf_extra_stack_depth', self.t_long)
+        make_field('jf_extra_stack_depth', self.t_Signed)
         make_field('jf_savedata', self.t_void_ptr)
         make_field('jf_guard_exc', self.t_void_ptr)
         make_field('jf_forward', t_jit_frame_ptr)
         # FIXME: for some reason there's an implicit word here;
         # create it
-        make_field('jf_frame', self.t_long)
+        make_field('jf_frame', self.t_Signed)
 
         initial_locs = []
         #loc = jitframe.getofs('jf_frame')
@@ -153,7 +154,7 @@
         self.field_for_arg_idx = {}
         for idx in range(max_args):
             self.field_for_arg_idx[idx] = make_field("arg%i" % idx,
-                                                     self.t_long)
+                                                     self.t_Signed)
             initial_locs.append(idx * self.sizeof_signed) # hack!
 
         struct_jit_frame.set_fields (fields)
@@ -174,7 +175,7 @@
             loopname = 'anonloop_%i' % self.num_anon_loops
             self.num_anon_loops += 1
         self.fn = self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_EXPORTED,
-                                         t_jit_frame_ptr, # self.t_long,
+                                         t_jit_frame_ptr,
                                          loopname,
                                          params,
                                          r_int(0))
@@ -247,13 +248,13 @@
         elif isinstance(expr, ConstInt):
             #print('value: %r' % expr.value)
             #print('type(value): %r' % type(expr.value))
-            return self.ctxt.new_rvalue_from_int(self.t_long,
+            return self.ctxt.new_rvalue_from_int(self.t_Signed,
                                                  r_int(expr.value))
         raise ValueError('unhandled expr: %s' % expr)
 
     def get_box_as_lvalue(self, box):
         if box not in self.lvalue_for_box:
-            self.lvalue_for_box[box] = self.fn.new_local(self.t_long,
+            self.lvalue_for_box[box] = self.fn.new_local(self.t_Signed,
                                                          str(box))
         return self.lvalue_for_box[box]
 
@@ -282,7 +283,7 @@
         lvalres = self.expr_to_lvalue(resop.result)
 
         binop_expr = self.ctxt.new_binary_op(gcc_jit_binary_op,
-                                             self.t_long,
+                                             self.t_Signed,
                                              rval0, rval1)
         self.b_current.add_assignment(lvalres, binop_expr)
 
@@ -327,7 +328,7 @@
         # First pass: capture the value of relevant boxes at the JUMP:
         tmps = []
         for i in range(len(jumpop._args)):
-            tmps.append(self.fn.new_local(self.t_long, "tmp%i" % i))
+            tmps.append(self.fn.new_local(self.t_Signed, "tmp%i" % i))
             self.b_current.add_assignment(
                 tmps[i],
                 self.get_box_as_lvalue(jumpop._args[i]).as_rvalue())
@@ -347,7 +348,7 @@
             self.ctxt.new_cast(
                 self.ctxt.new_comparison(gcc_jit_comparison,
                                          rval0, rval1),
-                self.t_long)
+                self.t_Signed)
             )
         self.b_current.add_assignment(lvalres,
                                       resop_cmp)
@@ -407,7 +408,7 @@
                 """
                 self.b_current.add_assignment(
                     self.get_arg_as_lvalue(idx),
-                    self.ctxt.new_rvalue_from_int (self.t_long,
+                    self.ctxt.new_rvalue_from_int (self.t_Signed,
                                                    r_int(0)))
                 """
 

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:40 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:40 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Support more of
 test_int_operations (requires gcc_jit_context_new_rvalue_from_long)
Message-ID: <20141217164840.3C30D1C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74985:478e510f4673
Date: 2014-12-17 01:36 -0500
http://bitbucket.org/pypy/pypy/changeset/478e510f4673/

Log:	Support more of test_int_operations (requires
	gcc_jit_context_new_rvalue_from_long)

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -248,8 +248,8 @@
         elif isinstance(expr, ConstInt):
             #print('value: %r' % expr.value)
             #print('type(value): %r' % type(expr.value))
-            return self.ctxt.new_rvalue_from_int(self.t_Signed,
-                                                 r_int(expr.value))
+            return self.ctxt.new_rvalue_from_long(self.t_Signed,
+                                                  r_long(expr.value))
         raise ValueError('unhandled expr: %s' % expr)
 
     def get_box_as_lvalue(self, box):
@@ -299,6 +299,24 @@
     def emit_int_floordiv(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_DIVIDE)
 
+    def emit_int_mod(self, resop):
+        self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_MODULO)
+
+    def emit_int_and(self, resop):
+        self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_BITWISE_AND)
+
+    def emit_int_or(self, resop):
+        self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_BITWISE_OR)
+
+    def emit_int_xor(self, resop):
+        self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_BITWISE_XOR)
+
+    def emit_int_rshift(self, resop):
+        self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_RSHIFT)
+
+    def emit_int_lshift(self, resop):
+        self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_LSHIFT)
+
     def emit_label(self, resop):
         print(resop)
         print(resop.__dict__)
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -129,6 +129,11 @@
                                               INT]),
 
                 (self.GCC_JIT_TYPE_P,
+                 'gcc_jit_context_get_int_type', [self.GCC_JIT_CONTEXT_P,
+                                                  INT,
+                                                  INT]),
+
+                (self.GCC_JIT_TYPE_P,
                  'gcc_jit_type_get_pointer', [self.GCC_JIT_TYPE_P]),
 
                 (self.GCC_JIT_FIELD_P,
@@ -200,6 +205,10 @@
                                                          self.GCC_JIT_TYPE_P,
                                                          INT]),
                 (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_new_rvalue_from_long', [self.GCC_JIT_CONTEXT_P,
+                                                          self.GCC_JIT_TYPE_P,
+                                                          LONG]),
+                (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_context_zero', [self.GCC_JIT_CONTEXT_P,
                                           self.GCC_JIT_TYPE_P]),
                 (self.GCC_JIT_RVALUE_P,
@@ -396,6 +405,12 @@
                     self.lib.gcc_jit_context_get_type(self.inner_ctxt,
                                                       r_enum))
 
+    def get_int_type(self, num_bytes, is_signed):
+        return Type(self.lib,
+                    self.lib.gcc_jit_context_get_int_type(self.inner_ctxt,
+                                                          num_bytes,
+                                                          is_signed))
+
     def new_field(self, type_, name):
         name_charp = str2charp(name)
         field = self.lib.gcc_jit_context_new_field(self.inner_ctxt,
@@ -437,6 +452,12 @@
                                                                    type_.inner_type,
                                                                    llvalue))
 
+    def new_rvalue_from_long(self, type_, llvalue):
+        return RValue(self.lib,
+                      self.lib.gcc_jit_context_new_rvalue_from_long(self.inner_ctxt,
+                                                                    type_.inner_type,
+                                                                    llvalue))
+
     def new_rvalue_from_ptr(self, type_, llvalue):
         return RValue(self.lib,
                       self.lib.gcc_jit_context_new_rvalue_from_ptr(self.inner_ctxt,

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:41 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:41 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Start supporting floats
 (requires GCC_JIT_UNARY_OP_ABS)
Message-ID: <20141217164841.659E81C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74986:6ed15f284d0e
Date: 2014-12-17 10:46 -0500
http://bitbucket.org/pypy/pypy/changeset/6ed15f284d0e/

Log:	Start supporting floats (requires GCC_JIT_UNARY_OP_ABS)

	Convert arg-handling to use a union. Implement some float
	operations.

	Requires new GCC_JIT_UNARY_OP_ABS (not yet in upstream gcc repo)

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -4,7 +4,7 @@
 from rpython.jit.backend.llsupport.regalloc import FrameManager
 from rpython.jit.backend.model import CompiledLoopToken
 from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library, make_param_array, make_field_array, Context, Type
-from rpython.jit.metainterp.history import BoxInt, ConstInt
+from rpython.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, ConstFloat
 from rpython.jit.metainterp.resoperation import *
 from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref, cast_object_to_ptr
 from rpython.rtyper.lltypesystem.rffi import *
@@ -90,9 +90,15 @@
         self.make_context()
         self.t_Signed = self.ctxt.get_int_type(r_int(self.sizeof_signed),
                                                r_int(1))
+        self.t_float = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_DOUBLE) # FIXME                                          
         self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
         self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
 
+        self.u_signed = self.ctxt.new_field(self.t_Signed, "u_signed")
+        self.u_float = self.ctxt.new_field(self.t_float, "u_float")
+        self.t_any = self.ctxt.new_union_type ("any",
+                                               [self.u_signed,
+                                                self.u_float])
         self.lvalue_for_box = {}
 
         print(jitframe.JITFRAME)
@@ -154,7 +160,7 @@
         self.field_for_arg_idx = {}
         for idx in range(max_args):
             self.field_for_arg_idx[idx] = make_field("arg%i" % idx,
-                                                     self.t_Signed)
+                                                     self.t_any)
             initial_locs.append(idx * self.sizeof_signed) # hack!
 
         struct_jit_frame.set_fields (fields)
@@ -197,9 +203,21 @@
         # Get initial values from input args:
         for idx, arg in enumerate(inputargs):
             self.b_current.add_comment("inputargs[%i]: %s" % (idx, arg))
+            # (gdb) p *(double*)&jitframe->arg0
+            # $7 = 10.5
+            # (gdb) p *(double*)&jitframe->arg1
+            # $8 = -2.25
+            #src_ptr_rvalue = self.get_arg_as_lvalue(idx).get_address()
+            #src_ptr_rvalue = self.ctxt.new_cast(src_ptr_rvalue, self.t_float.get_pointer())
+            #src_rvalue = self.ctxt.new_dereference(src_ptr_rvalue)
+            # or do it as a union
+            field = self.get_union_field_for_box(arg)
+            src_rvalue = self.get_arg_as_lvalue(idx).access_field(field).as_rvalue()
+            # FIXME: this may need a cast:
+            #src_rvalue = self.ctxt.new_cast(src_rvalue, self.t_float)
             self.b_current.add_assignment(
                 self.get_box_as_lvalue(arg),
-                self.get_arg_as_lvalue(idx).as_rvalue())
+                src_rvalue)
 
         for op in operations:
             print(op)
@@ -243,19 +261,25 @@
         print(' %s' % expr.__dict__)
         """
 
-        if isinstance(expr, BoxInt):
+        if isinstance(expr, (BoxInt, BoxFloat)):
             return self.get_box_as_lvalue(expr).as_rvalue()
         elif isinstance(expr, ConstInt):
             #print('value: %r' % expr.value)
             #print('type(value): %r' % type(expr.value))
             return self.ctxt.new_rvalue_from_long(self.t_Signed,
                                                   r_long(expr.value))
-        raise ValueError('unhandled expr: %s' % expr)
+        elif isinstance(expr, ConstFloat):
+            #print('value: %r' % expr.value)
+            #print('type(value): %r' % type(expr.value))
+            return self.ctxt.new_rvalue_from_double(self.t_float,
+                                                    expr.value)#r_double(expr.value))
+        raise ValueError('unhandled expr: %s %s' % (expr, type(expr)))
 
     def get_box_as_lvalue(self, box):
         if box not in self.lvalue_for_box:
-            self.lvalue_for_box[box] = self.fn.new_local(self.t_Signed,
-                                                         str(box))
+            self.lvalue_for_box[box] = (
+                self.fn.new_local(self.get_type_for_box(box),
+                                  str(box)))
         return self.lvalue_for_box[box]
 
     def get_arg_as_lvalue(self, idx):
@@ -271,17 +295,34 @@
         print(' %s' % dir(expr))
         print(' %s' % expr.__dict__)
         """
-        if isinstance(expr, BoxInt):
+        if isinstance(expr, (BoxInt, BoxFloat)):
             return self.get_box_as_lvalue(expr)
+            
         raise ValueError('unhandled expr: %s' % expr)
 
+    def get_type_for_box(self, box):
+        if isinstance(box, BoxInt):
+            return self.t_Signed;
+        elif isinstance(box, BoxFloat):
+            return self.t_float;
+        else:
+            raise ValueError('unhandled box: %s %s' % (box, type(box)))
+
+    def get_union_field_for_box(self, box):
+        if isinstance(box, BoxInt):
+            return self.u_signed;
+        elif isinstance(box, BoxFloat):
+            return self.u_float;
+        else:
+            raise ValueError('unhandled box: %s %s' % (box, type(box)))        
+
     # Handling of specific ResOperation subclasses
 
+    # Binary operations on "int":
     def impl_int_binop(self, resop, gcc_jit_binary_op):
         rval0 = self.expr_to_rvalue(resop._arg0)
         rval1 = self.expr_to_rvalue(resop._arg1)
         lvalres = self.expr_to_lvalue(resop.result)
-
         binop_expr = self.ctxt.new_binary_op(gcc_jit_binary_op,
                                              self.t_Signed,
                                              rval0, rval1)
@@ -289,34 +330,58 @@
 
     def emit_int_add(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_PLUS)
-
     def emit_int_sub(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_MINUS)
-
     def emit_int_mul(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_MULT)
-
     def emit_int_floordiv(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_DIVIDE)
-
     def emit_int_mod(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_MODULO)
-
     def emit_int_and(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_BITWISE_AND)
-
     def emit_int_or(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_BITWISE_OR)
-
     def emit_int_xor(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_BITWISE_XOR)
-
     def emit_int_rshift(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_RSHIFT)
-
     def emit_int_lshift(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_LSHIFT)
 
+    # "FLOAT" binary ops:
+    def impl_float_binop(self, resop, gcc_jit_binary_op):
+        rval0 = self.expr_to_rvalue(resop._arg0)
+        rval1 = self.expr_to_rvalue(resop._arg1)
+        lvalres = self.expr_to_lvalue(resop.result)
+        binop_expr = self.ctxt.new_binary_op(gcc_jit_binary_op,
+                                             self.t_float,
+                                             rval0, rval1)
+        self.b_current.add_assignment(lvalres, binop_expr)
+
+    def emit_float_add(self, resop):
+        self.impl_float_binop(resop, self.lib.GCC_JIT_BINARY_OP_PLUS)
+    def emit_float_sub(self, resop):
+        self.impl_float_binop(resop, self.lib.GCC_JIT_BINARY_OP_MINUS)
+    def emit_float_mul(self, resop):
+        self.impl_float_binop(resop, self.lib.GCC_JIT_BINARY_OP_MULT)
+    def emit_float_truediv(self, resop):
+        self.impl_float_binop(resop, self.lib.GCC_JIT_BINARY_OP_DIVIDE)
+
+    # "FLOAT" unary ops:
+    def _impl_float_unaryop(self, resop, gcc_jit_unary_op):
+        rvalue = self.expr_to_rvalue(resop._arg0)
+        lvalres = self.expr_to_lvalue(resop.result)
+        unaryop_expr = self.ctxt.new_unary_op(gcc_jit_unary_op,
+                                            self.t_float,
+                                            rvalue)
+        self.b_current.add_assignment(lvalres, unaryop_expr)
+
+    def emit_float_neg(self, resop):
+        self._impl_float_unaryop(resop, self.lib.GCC_JIT_UNARY_OP_MINUS)
+    def emit_float_abs(self, resop):
+        self._impl_float_unaryop(resop, self.lib.GCC_JIT_UNARY_OP_ABS)
+
     def emit_label(self, resop):
         print(resop)
         print(resop.__dict__)
@@ -358,6 +423,7 @@
             
         self.b_current.end_with_jump(self.block_for_label_descr[jumpop.getdescr()])
 
+    # "INT" comparisons:
     def impl_int_cmp(self, resop, gcc_jit_comparison):
         rval0 = self.expr_to_rvalue(resop._arg0)
         rval1 = self.expr_to_rvalue(resop._arg1)
@@ -370,16 +436,50 @@
             )
         self.b_current.add_assignment(lvalres,
                                       resop_cmp)
-
+    def emit_int_lt(self, resop):
+        self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_LT)
+    def emit_int_le(self, resop):
+        self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_LE)
     def emit_int_eq(self, resop):
         self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_EQ)
-
-    def emit_int_le(self, resop):
-        self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_LE)
-
+    def emit_int_ne(self, resop):
+        self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_NE)
+    def emit_int_gt(self, resop):
+        self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_GT)
     def emit_int_ge(self, resop):
         self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
 
+    # "UINT" comparisons:
+
+    # (TODO)
+
+    # "FLOAT" comparisons:
+    def impl_float_cmp(self, resop, gcc_jit_comparison):
+        rval0 = self.expr_to_rvalue(resop._arg0)
+        rval1 = self.expr_to_rvalue(resop._arg1)
+        lvalres = self.expr_to_lvalue(resop.result)
+        resop_cmp = (
+            self.ctxt.new_cast(
+                self.ctxt.new_comparison(gcc_jit_comparison,
+                                         rval0, rval1),
+                self.t_Signed)
+            )
+        self.b_current.add_assignment(lvalres,
+                                      resop_cmp)
+
+    def emit_float_lt(self, resop):
+        self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_LT)
+    def emit_float_le(self, resop):
+        self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_LE)
+    def emit_float_eq(self, resop):
+        self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_EQ)
+    def emit_float_ne(self, resop):
+        self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_NE)
+    def emit_float_gt(self, resop):
+        self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_GT)
+    def emit_float_ge(self, resop):
+        self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
+
     def impl_guard(self, resop, istrue):
         print(resop)
         print(resop.__dict__)
@@ -417,9 +517,10 @@
         # Write outputs back:
         for idx, arg in enumerate(args):
             if arg is not None:
-                self.b_current.add_assignment(
-                    self.get_arg_as_lvalue(idx),
-                    self.get_box_as_lvalue(arg).as_rvalue())
+                src_rvalue = self.get_box_as_lvalue(arg).as_rvalue()
+                field = self.get_union_field_for_box(arg)
+                dst_lvalue = self.get_arg_as_lvalue(idx).access_field(field)
+                self.b_current.add_assignment(dst_lvalue, src_rvalue)
             else:
                 # FIXME: see test_compile_with_holes_in_fail_args
                 raise ValueError("how to handle holes in fail args?")
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -161,6 +161,13 @@
                                                INT,
                                                self.FIELD_P_P]),
 
+                (self.GCC_JIT_TYPE_P,
+                 'gcc_jit_context_new_union_type', [self.GCC_JIT_CONTEXT_P,
+                                                    self.GCC_JIT_LOCATION_P,
+                                                    CCHARP,
+                                                    INT,
+                                                    self.FIELD_P_P]),
+
                 ############################################################
                 # Constructing functions.
                 ############################################################
@@ -214,13 +221,23 @@
                 (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_context_one', [self.GCC_JIT_CONTEXT_P,
                                          self.GCC_JIT_TYPE_P]),
-
+                (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_new_rvalue_from_double', [self.GCC_JIT_CONTEXT_P,
+                                                            self.GCC_JIT_TYPE_P,
+                                                            DOUBLE]),
                 (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_context_new_rvalue_from_ptr', [self.GCC_JIT_CONTEXT_P,
                                                          self.GCC_JIT_TYPE_P,
                                                          VOIDP]),
 
                 (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_new_unary_op', [self.GCC_JIT_CONTEXT_P,
+                                                  self.GCC_JIT_LOCATION_P,
+                                                  INT, # enum gcc_jit_unary_op op,
+                                                  self.GCC_JIT_TYPE_P,
+                                                  self.GCC_JIT_RVALUE_P]),
+
+                (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_context_new_binary_op', [self.GCC_JIT_CONTEXT_P,
                                                    self.GCC_JIT_LOCATION_P,
                                                    INT, # enum gcc_jit_binary_op op,
@@ -231,7 +248,7 @@
                 (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_context_new_comparison', [self.GCC_JIT_CONTEXT_P,
                                                     self.GCC_JIT_LOCATION_P,
-                                                    INT, # enum gcc_jit_binary_op op,
+                                                    INT, # enum gcc_jit_comparison op,
                                                     self.GCC_JIT_RVALUE_P,
                                                     self.GCC_JIT_RVALUE_P]),
 
@@ -242,6 +259,11 @@
                                               self.GCC_JIT_TYPE_P]),
 
                 (self.GCC_JIT_LVALUE_P,
+                 'gcc_jit_lvalue_access_field', [self.GCC_JIT_LVALUE_P,
+                                                 self.GCC_JIT_LOCATION_P,
+                                                 self.GCC_JIT_FIELD_P]),
+
+                (self.GCC_JIT_LVALUE_P,
                  'gcc_jit_rvalue_dereference_field', [self.GCC_JIT_RVALUE_P,
                                                       self.GCC_JIT_LOCATION_P,
                                                       self.GCC_JIT_FIELD_P]),
@@ -321,6 +343,14 @@
 
         self.make_enum_values(
             """
+            GCC_JIT_UNARY_OP_MINUS,
+            GCC_JIT_UNARY_OP_BITWISE_NEGATE,
+            GCC_JIT_UNARY_OP_LOGICAL_NEGATE,
+            GCC_JIT_UNARY_OP_ABS
+            """)
+
+        self.make_enum_values(
+            """
             GCC_JIT_BINARY_OP_PLUS,
             GCC_JIT_BINARY_OP_MINUS,
             GCC_JIT_BINARY_OP_MULT,
@@ -446,6 +476,23 @@
         free_charp(name_charp)
         return Struct(self.lib, inner_struct)
 
+    def new_union_type(self, name, fields):
+        name_charp = str2charp(name)
+        field_array = lltype.malloc(self.lib.FIELD_P_P.TO,
+                                    len(fields),
+                                    flavor='raw') # of maybe gc?
+        for i in range(len(fields)):
+            field_array[i] = fields[i].inner_field
+        inner_type = (
+            self.lib.gcc_jit_context_new_union_type(self.inner_ctxt,
+                                                    self.lib.null_location_ptr,
+                                                    name_charp,
+                                                    r_int(len(fields)),
+                                                    field_array))
+        lltype.free(field_array, flavor='raw')
+        free_charp(name_charp)
+        return Type(self.lib, inner_type)
+
     def new_rvalue_from_int(self, type_, llvalue):
         return RValue(self.lib,
                       self.lib.gcc_jit_context_new_rvalue_from_int(self.inner_ctxt,
@@ -458,12 +505,26 @@
                                                                     type_.inner_type,
                                                                     llvalue))
 
+    def new_rvalue_from_double(self, type_, llvalue):
+        return RValue(self.lib,
+                      self.lib.gcc_jit_context_new_rvalue_from_double(self.inner_ctxt,
+                                                                      type_.inner_type,
+                                                                      llvalue))
+
     def new_rvalue_from_ptr(self, type_, llvalue):
         return RValue(self.lib,
                       self.lib.gcc_jit_context_new_rvalue_from_ptr(self.inner_ctxt,
                                                                    type_.inner_type,
                                                                    llvalue))
 
+    def new_unary_op(self, op, type_, rvalue):
+        return RValue(self.lib,
+                      self.lib.gcc_jit_context_new_unary_op(self.inner_ctxt,
+                                                            self.lib.null_location_ptr,
+                                                            op,
+                                                            type_.inner_type,
+                                                             rvalue.inner_rvalue))
+
     def new_binary_op(self, op, type_, a, b):
         return RValue(self.lib,
                       self.lib.gcc_jit_context_new_binary_op(self.inner_ctxt,
@@ -572,6 +633,12 @@
         return RValue(self.lib,
                       self.lib.gcc_jit_lvalue_as_rvalue(self.inner_lvalue))
 
+    def access_field(self, field):
+        return LValue(self.lib,
+                      self.lib.gcc_jit_lvalue_access_field (self.inner_lvalue,
+                                                            self.lib.null_location_ptr,
+                                                            field.inner_field))
+
 class Param(Wrapper):
     def __init__(self, lib, inner_param):
         Wrapper.__init__(self, lib)
diff --git a/rpython/jit/backend/libgccjit/runner.py b/rpython/jit/backend/libgccjit/runner.py
--- a/rpython/jit/backend/libgccjit/runner.py
+++ b/rpython/jit/backend/libgccjit/runner.py
@@ -4,6 +4,9 @@
 
 #class CPU(model.AbstractCPU):
 class CPU(AbstractLLCPU):
+
+    supports_floats = True
+
     def __init__(self, rtyper, stats, opts=None, translate_support_code=False,
                  gcdescr=None):
         AbstractLLCPU.__init__(self, rtyper, stats, opts,

From noreply at buildbot.pypy.org  Wed Dec 17 17:48:42 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 17:48:42 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_float_operations to
	pass
Message-ID: <20141217164842.97E2B1C01C3@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74987:f2406d6b5d16
Date: 2014-12-17 11:26 -0500
http://bitbucket.org/pypy/pypy/changeset/f2406d6b5d16/

Log:	Get test_float_operations to pass

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -105,10 +105,13 @@
         print(dir(jitframe.JITFRAME))
         print('jitframe.JITFRAME._flds: %r' % jitframe.JITFRAME._flds)
 
-        # For now, build a "struct jit_frame".
+        # For now, build a "struct JITFRAME".
         # This will have the fields of jitframe,
-        # We'll construct numbered arg0..argN for use
+        # plus additional numbered arg0..argN for use
         # for inputargs and the outputs
+        #
+        # They'll each be of type "union any", so that we can freely read/write
+        # to them as any of the fundamental types.
 
         fields = []
         def make_field(name, jit_type):
@@ -382,6 +385,20 @@
     def emit_float_abs(self, resop):
         self._impl_float_unaryop(resop, self.lib.GCC_JIT_UNARY_OP_ABS)
 
+    # "CAST_" operations:
+    def emit_cast_float_to_int(self, resop):
+        rvalue = self.expr_to_rvalue(resop._arg0)
+        lvalres = self.expr_to_lvalue(resop.result)
+        cast_expr = self.ctxt.new_cast(rvalue, self.t_Signed)
+        self.b_current.add_assignment(lvalres, cast_expr)
+    def emit_cast_int_to_float(self, resop):
+        rvalue = self.expr_to_rvalue(resop._arg0)
+        lvalres = self.expr_to_lvalue(resop.result)
+        cast_expr = self.ctxt.new_cast(rvalue, self.t_float)
+        self.b_current.add_assignment(lvalres, cast_expr)
+
+    # These are out-of-order compared to the list in resoperation.py:
+
     def emit_label(self, resop):
         print(resop)
         print(resop.__dict__)
@@ -449,9 +466,7 @@
     def emit_int_ge(self, resop):
         self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
 
-    # "UINT" comparisons:
-
-    # (TODO)
+    # "UINT" comparisons:  TODO
 
     # "FLOAT" comparisons:
     def impl_float_cmp(self, resop, gcc_jit_comparison):

From noreply at buildbot.pypy.org  Wed Dec 17 18:06:25 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 17 Dec 2014 18:06:25 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Reorder emit_ methods to
 match the ordering within resoperation.py
Message-ID: <20141217170625.A9AC41D3673@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r74988:c5989b35a619
Date: 2014-12-17 12:11 -0500
http://bitbucket.org/pypy/pypy/changeset/c5989b35a619/

Log:	Reorder emit_ methods to match the ordering within resoperation.py

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -319,7 +319,124 @@
         else:
             raise ValueError('unhandled box: %s %s' % (box, type(box)))        
 
-    # Handling of specific ResOperation subclasses
+    # Handling of specific ResOperation subclasses: each one is
+    # a method named "emit_foo", where "foo" is the str() of the resop.
+    #
+    # Keep these in the same order as _oplist within
+    # rpython/jit/metainterp/resoperation.py.
+
+    # JUMP and FINISH:
+
+    def emit_jump(self, jumpop):
+        print(jumpop)
+        print(jumpop.__dict__)
+        label = self.label_for_descr[jumpop.getdescr()]
+        print(jumpop.getdescr())
+        print('label: %r' % label)
+
+        assert len(jumpop._args) == len(label._args)
+
+        # We need to write to the boxes listed in the label's args
+        # with the values from those listed in the jump's args.
+        # However, there are potential cases like JUMP(i0, i1) going to LABEL(i1, i0)
+        # where all such assignments happen "simultaneously".
+        # Hence we need to set up temporaries.
+        # First pass: capture the value of relevant boxes at the JUMP:
+        tmps = []
+        for i in range(len(jumpop._args)):
+            tmps.append(self.fn.new_local(self.t_Signed, "tmp%i" % i))
+            self.b_current.add_assignment(
+                tmps[i],
+                self.get_box_as_lvalue(jumpop._args[i]).as_rvalue())
+        # Second pass: write the values to the boxes at the LABEL:
+        for i in range(len(jumpop._args)):
+            self.b_current.add_assignment(
+                self.get_box_as_lvalue(label._args[i]),
+                tmps[i].as_rvalue())
+            
+        self.b_current.end_with_jump(self.block_for_label_descr[jumpop.getdescr()])
+
+    def emit_finish(self, resop):
+        self._impl_write_output_args(resop._args)
+        self._impl_write_jf_descr(resop)
+        self.b_current.end_with_return(self.param_frame.as_rvalue ())
+
+    def emit_label(self, resop):
+        print(resop)
+        print(resop.__dict__)
+        #print('resop.getdescr(): %r' % resop.getdescr())
+        #print('resop.getdescr().__dict__: %r' % resop.getdescr().__dict__)
+
+        b_new = self.fn.new_block(str(resop))
+        self.block_for_label_descr[resop.getdescr()] = b_new
+        self.label_for_descr[resop.getdescr()] = resop
+        self.b_current.end_with_jump(b_new)
+        self.b_current = b_new
+
+    # GUARD_*
+
+    def _impl_guard(self, resop, istrue):
+        print(resop)
+        print(resop.__dict__)
+        b_true = self.fn.new_block("on_true_at_%s" % resop)
+        b_false = self.fn.new_block("on_false_at_%s" % resop)
+        boolval = self.ctxt.new_cast(self.expr_to_rvalue(resop._arg0),
+                                     self.t_bool)
+        self.b_current.end_with_conditional(boolval,
+                                            b_true, b_false)
+
+        if istrue:
+            self.b_current = b_false
+        else:
+            self.b_current = b_true
+        self._impl_write_output_args(resop._fail_args)
+        self._impl_write_jf_descr(resop)
+        self.b_current.end_with_return(self.param_frame.as_rvalue ())
+        rd_locs = []
+        for idx, arg in enumerate(resop._fail_args):
+            rd_locs.append(idx * self.sizeof_signed)
+        resop.getdescr().rd_locs = rd_locs
+
+        if istrue:
+            self.b_current = b_true
+        else:
+            self.b_current = b_false
+
+    def emit_guard_true(self, resop):
+        self._impl_guard(resop, r_int(1))
+        
+    def emit_guard_false(self, resop):
+        self._impl_guard(resop, r_int(0))
+
+    def _impl_write_output_args(self, args):
+        # Write outputs back:
+        for idx, arg in enumerate(args):
+            if arg is not None:
+                src_rvalue = self.get_box_as_lvalue(arg).as_rvalue()
+                field = self.get_union_field_for_box(arg)
+                dst_lvalue = self.get_arg_as_lvalue(idx).access_field(field)
+                self.b_current.add_assignment(dst_lvalue, src_rvalue)
+            else:
+                # FIXME: see test_compile_with_holes_in_fail_args
+                raise ValueError("how to handle holes in fail args?")
+                """
+                self.b_current.add_assignment(
+                    self.get_arg_as_lvalue(idx),
+                    self.ctxt.new_rvalue_from_int (self.t_Signed,
+                                                   r_int(0)))
+                """
+
+    def _impl_write_jf_descr(self, resop):
+        # Write back to the jf_descr:
+        #  "jitframe->jf_descr = resop.getdescr();"
+        descr = rffi.cast(lltype.Signed,
+                          cast_instance_to_gcref(resop.getdescr()))
+
+        self.b_current.add_assignment(
+            self.param_frame.as_rvalue ().dereference_field (
+                self.field_jf_descr),
+            self.ctxt.new_rvalue_from_ptr (self.t_void_ptr,
+                                           rffi.cast(VOIDP, descr)))
 
     # Binary operations on "int":
     def impl_int_binop(self, resop, gcc_jit_binary_op):
@@ -397,50 +514,8 @@
         cast_expr = self.ctxt.new_cast(rvalue, self.t_float)
         self.b_current.add_assignment(lvalres, cast_expr)
 
-    # These are out-of-order compared to the list in resoperation.py:
-
-    def emit_label(self, resop):
-        print(resop)
-        print(resop.__dict__)
-        #print('resop.getdescr(): %r' % resop.getdescr())
-        #print('resop.getdescr().__dict__: %r' % resop.getdescr().__dict__)
-
-        b_new = self.fn.new_block(str(resop))
-        self.block_for_label_descr[resop.getdescr()] = b_new
-        self.label_for_descr[resop.getdescr()] = resop
-        self.b_current.end_with_jump(b_new)
-        self.b_current = b_new
-
-    def emit_jump(self, jumpop):
-        print(jumpop)
-        print(jumpop.__dict__)
-        label = self.label_for_descr[jumpop.getdescr()]
-        print(jumpop.getdescr())
-        print('label: %r' % label)
-
-        assert len(jumpop._args) == len(label._args)
-
-        # We need to write to the boxes listed in the label's args
-        # with the values from those listed in the jump's args.
-        # However, there are potential cases like JUMP(i0, i1) going to LABEL(i1, i0)
-        # where all such assignments happen "simultaneously".
-        # Hence we need to set up temporaries.
-        # First pass: capture the value of relevant boxes at the JUMP:
-        tmps = []
-        for i in range(len(jumpop._args)):
-            tmps.append(self.fn.new_local(self.t_Signed, "tmp%i" % i))
-            self.b_current.add_assignment(
-                tmps[i],
-                self.get_box_as_lvalue(jumpop._args[i]).as_rvalue())
-        # Second pass: write the values to the boxes at the LABEL:
-        for i in range(len(jumpop._args)):
-            self.b_current.add_assignment(
-                self.get_box_as_lvalue(label._args[i]),
-                tmps[i].as_rvalue())
-            
-        self.b_current.end_with_jump(self.block_for_label_descr[jumpop.getdescr()])
-
-    # "INT" comparisons:
+    # Comparisons:
+    #   "INT" comparisons:
     def impl_int_cmp(self, resop, gcc_jit_comparison):
         rval0 = self.expr_to_rvalue(resop._arg0)
         rval1 = self.expr_to_rvalue(resop._arg1)
@@ -466,9 +541,9 @@
     def emit_int_ge(self, resop):
         self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
 
-    # "UINT" comparisons:  TODO
+    #   "UINT" comparisons:  TODO
 
-    # "FLOAT" comparisons:
+    #   "FLOAT" comparisons:
     def impl_float_cmp(self, resop, gcc_jit_comparison):
         rval0 = self.expr_to_rvalue(resop._arg0)
         rval1 = self.expr_to_rvalue(resop._arg1)
@@ -495,70 +570,3 @@
     def emit_float_ge(self, resop):
         self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
 
-    def impl_guard(self, resop, istrue):
-        print(resop)
-        print(resop.__dict__)
-        b_true = self.fn.new_block("on_true_at_%s" % resop)
-        b_false = self.fn.new_block("on_false_at_%s" % resop)
-        boolval = self.ctxt.new_cast(self.expr_to_rvalue(resop._arg0),
-                                     self.t_bool)
-        self.b_current.end_with_conditional(boolval,
-                                            b_true, b_false)
-
-        if istrue:
-            self.b_current = b_false
-        else:
-            self.b_current = b_true
-        self.impl_write_output_args(resop._fail_args)
-        self.impl_write_jf_descr(resop)
-        self.b_current.end_with_return(self.param_frame.as_rvalue ())
-        rd_locs = []
-        for idx, arg in enumerate(resop._fail_args):
-            rd_locs.append(idx * self.sizeof_signed)
-        resop.getdescr().rd_locs = rd_locs
-
-        if istrue:
-            self.b_current = b_true
-        else:
-            self.b_current = b_false
-
-    def emit_guard_true(self, resop):
-        self.impl_guard(resop, r_int(1))
-        
-    def emit_guard_false(self, resop):
-        self.impl_guard(resop, r_int(0))
-
-    def impl_write_output_args(self, args):
-        # Write outputs back:
-        for idx, arg in enumerate(args):
-            if arg is not None:
-                src_rvalue = self.get_box_as_lvalue(arg).as_rvalue()
-                field = self.get_union_field_for_box(arg)
-                dst_lvalue = self.get_arg_as_lvalue(idx).access_field(field)
-                self.b_current.add_assignment(dst_lvalue, src_rvalue)
-            else:
-                # FIXME: see test_compile_with_holes_in_fail_args
-                raise ValueError("how to handle holes in fail args?")
-                """
-                self.b_current.add_assignment(
-                    self.get_arg_as_lvalue(idx),
-                    self.ctxt.new_rvalue_from_int (self.t_Signed,
-                                                   r_int(0)))
-                """
-
-    def impl_write_jf_descr(self, resop):
-        # Write back to the jf_descr:
-        #  "jitframe->jf_descr = resop.getdescr();"
-        descr = rffi.cast(lltype.Signed,
-                          cast_instance_to_gcref(resop.getdescr()))
-
-        self.b_current.add_assignment(
-            self.param_frame.as_rvalue ().dereference_field (
-                self.field_jf_descr),
-            self.ctxt.new_rvalue_from_ptr (self.t_void_ptr,
-                                           rffi.cast(VOIDP, descr)))
-    
-    def emit_finish(self, resop):
-        self.impl_write_output_args(resop._args)
-        self.impl_write_jf_descr(resop)
-        self.b_current.end_with_return(self.param_frame.as_rvalue ())

From noreply at buildbot.pypy.org  Wed Dec 17 18:12:48 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 17 Dec 2014 18:12:48 +0100 (CET)
Subject: [pypy-commit] cffi default: Add dlopen flags to verify on pypy
Message-ID: <20141217171248.1C6291D367E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r1587:0853a872dedf
Date: 2014-10-21 12:55 +0200
http://bitbucket.org/cffi/cffi/changeset/0853a872dedf/

Log:	Add dlopen flags to verify on pypy

diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -58,12 +58,12 @@
             modname = self.verifier.get_module_name()
             prnt("void %s%s(void) { }\n" % (prefix, modname))
 
-    def load_library(self):
+    def load_library(self, flags=0):
         # import it with the CFFI backend
         backend = self.ffi._backend
         # needs to make a path that contains '/', on Posix
         filename = os.path.join(os.curdir, self.verifier.modulefilename)
-        module = backend.load_library(filename)
+        module = backend.load_library(filename, flags)
         #
         # call loading_gen_struct() to get the struct layout inferred by
         # the C compiler
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -16,7 +16,7 @@
 class Verifier(object):
 
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
-                 ext_package=None, tag='', force_generic_engine=False, **kwds):
+                 ext_package=None, tag='', force_generic_engine=False, flags=0, **kwds):
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
@@ -24,6 +24,7 @@
         vengine_class = _locate_engine_class(ffi, force_generic_engine)
         self._vengine = vengine_class(self)
         self._vengine.patch_extension_kwds(kwds)
+        self.flags = flags
         self.kwds = kwds
         #
         if modulename:
@@ -158,7 +159,7 @@
 
     def _load_library(self):
         assert self._has_module
-        return self._vengine.load_library()
+        return self._vengine.load_library(self.flags)
 
 # ____________________________________________________________
 
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -1962,3 +1962,17 @@
             n = (1 << 29) + i
             lib.SetLastError(n)
             assert ffi.getwinerror()[0] == n
+
+def test_verify_dlopen_flags():
+    ffi1 = FFI()
+    ffi2 = FFI()
+
+    ffi1.cdef("int foo;")
+    ffi2.cdef("int foo;")
+
+    lib1 = ffi1.verify("int foo;", modulename="foo1", flags=ffi1.RTLD_GLOBAL)
+    lib2 = ffi2.verify("int foo;", modulename="foo2", flags=ffi2.RTLD_GLOBAL)
+
+    lib1.foo = 42
+
+    assert lib2.foo == 42

From noreply at buildbot.pypy.org  Wed Dec 17 18:12:49 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 17 Dec 2014 18:12:49 +0100 (CET)
Subject: [pypy-commit] cffi default: Import verify dlopen flags for cpython
Message-ID: <20141217171249.23F7D1D367E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r1588:2ff481ead26d
Date: 2014-10-21 13:28 +0200
http://bitbucket.org/cffi/cffi/changeset/2ff481ead26d/

Log:	Import verify dlopen flags for cpython

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -138,15 +138,19 @@
         prnt()
         prnt('#endif')
 
-    def load_library(self):
+    def load_library(self, flags=0):
         # XXX review all usages of 'self' here!
         # import it as a new extension module
+        previous_flags = sys.getdlopenflags()
         try:
+            sys.setdlopenflags(flags)
             module = imp.load_dynamic(self.verifier.get_module_name(),
                                       self.verifier.modulefilename)
         except ImportError as e:
             error = "importing %r: %s" % (self.verifier.modulefilename, e)
             raise ffiplatform.VerificationError(error)
+        finally:
+            sys.setdlopenflags(previous_flags)
         #
         # call loading_cpy_struct() to get the struct layout inferred by
         # the C compiler
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -1965,14 +1965,18 @@
 
 def test_verify_dlopen_flags():
     ffi1 = FFI()
-    ffi2 = FFI()
+    ffi1.cdef("int foo;")
 
-    ffi1.cdef("int foo;")
-    ffi2.cdef("int foo;")
-
-    lib1 = ffi1.verify("int foo;", modulename="foo1", flags=ffi1.RTLD_GLOBAL)
-    lib2 = ffi2.verify("int foo;", modulename="foo2", flags=ffi2.RTLD_GLOBAL)
+    lib1 = ffi1.verify("int foo;", flags=ffi1.RTLD_GLOBAL | ffi1.RTLD_LAZY)
+    lib2 = get_second_lib()
 
     lib1.foo = 42
 
     assert lib2.foo == 42
+
+def get_second_lib():
+    # Hack, using modulename makes the test fail
+    ffi2 = FFI()
+    ffi2.cdef("int foo;")
+    lib2 = ffi2.verify("int foo;", flags=ffi2.RTLD_GLOBAL | ffi2.RTLD_LAZY)
+    return lib2

From noreply at buildbot.pypy.org  Wed Dec 17 18:12:50 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 17 Dec 2014 18:12:50 +0100 (CET)
Subject: [pypy-commit] cffi default: Fix failing tests
Message-ID: <20141217171250.35A1F1D367E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r1589:f0241abbaab0
Date: 2014-10-21 15:38 +0200
http://bitbucket.org/cffi/cffi/changeset/f0241abbaab0/

Log:	Fix failing tests

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -138,12 +138,13 @@
         prnt()
         prnt('#endif')
 
-    def load_library(self, flags=0):
+    def load_library(self, flags=None):
         # XXX review all usages of 'self' here!
         # import it as a new extension module
         previous_flags = sys.getdlopenflags()
         try:
-            sys.setdlopenflags(flags)
+            if flags is not None:
+                sys.setdlopenflags(flags)
             module = imp.load_dynamic(self.verifier.get_module_name(),
                                       self.verifier.modulefilename)
         except ImportError as e:
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -16,7 +16,7 @@
 class Verifier(object):
 
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
-                 ext_package=None, tag='', force_generic_engine=False, flags=0, **kwds):
+                 ext_package=None, tag='', force_generic_engine=False, flags=None, **kwds):
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
@@ -159,7 +159,10 @@
 
     def _load_library(self):
         assert self._has_module
-        return self._vengine.load_library(self.flags)
+        if self.flags is not None:
+            return self._vengine.load_library(self.flags)
+        else:
+            return self._vengine.load_library()
 
 # ____________________________________________________________
 

From noreply at buildbot.pypy.org  Wed Dec 17 18:12:51 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 17 Dec 2014 18:12:51 +0100 (CET)
Subject: [pypy-commit] cffi default: Add documentation for the flags
	argument of verify
Message-ID: <20141217171251.5843A1D367E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r1590:36d8fcb90561
Date: 2014-12-17 17:50 +0100
http://bitbucket.org/cffi/cffi/changeset/36d8fcb90561/

Log:	Add documentation for the flags argument of verify

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -500,7 +500,7 @@
 The verification step
 ---------------------
 
-``ffi.verify(source, tmpdir=.., ext_package=.., modulename=.., **kwargs)``:
+``ffi.verify(source, tmpdir=.., ext_package=.., modulename=.., flags=.., **kwargs)``:
 verifies that the current ffi signatures
 compile on this machine, and return a dynamic library object.  The
 dynamic library can be used to call functions and access global
@@ -667,6 +667,10 @@
    check.  Be sure to have other means of clearing the ``tmpdir``
    whenever you change your sources.
 
+.. versionadded:: 0.9
+   The optional ``flags`` argument has been added, see ``man dlopen`` (ignored
+   on Windows).  It defaults to ``ffi.RTLD_NOW``.
+
 This function returns a "library" object that gets closed when it goes
 out of scope.  Make sure you keep the library object around as long as
 needed.

From noreply at buildbot.pypy.org  Wed Dec 17 18:12:52 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 17 Dec 2014 18:12:52 +0100 (CET)
Subject: [pypy-commit] cffi default: Make sure the flags kwarg of verify has
 no effect on Windows
Message-ID: <20141217171252.506DF1D367E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r1591:b7065ad45a58
Date: 2014-12-17 17:57 +0100
http://bitbucket.org/cffi/cffi/changeset/b7065ad45a58/

Log:	Make sure the flags kwarg of verify has no effect on Windows

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -141,9 +141,10 @@
     def load_library(self, flags=None):
         # XXX review all usages of 'self' here!
         # import it as a new extension module
-        previous_flags = sys.getdlopenflags()
+        if hasattr(sys, "getdlopenflags"):
+            previous_flags = sys.getdlopenflags()
         try:
-            if flags is not None:
+            if hasattr(sys, "setdlopenflags") and flags is not None:
                 sys.setdlopenflags(flags)
             module = imp.load_dynamic(self.verifier.get_module_name(),
                                       self.verifier.modulefilename)
@@ -151,7 +152,8 @@
             error = "importing %r: %s" % (self.verifier.modulefilename, e)
             raise ffiplatform.VerificationError(error)
         finally:
-            sys.setdlopenflags(previous_flags)
+            if hasattr(sys, "setdlopenflags"):
+                sys.setdlopenflags(previous_flags)
         #
         # call loading_cpy_struct() to get the struct layout inferred by
         # the C compiler

From noreply at buildbot.pypy.org  Wed Dec 17 18:12:53 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed, 17 Dec 2014 18:12:53 +0100 (CET)
Subject: [pypy-commit] cffi default: Merge heads
Message-ID: <20141217171253.836D61D367E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert <romain.py at gmail.com>
Branch: 
Changeset: r1592:9267d5f13672
Date: 2014-12-17 18:04 +0100
http://bitbucket.org/cffi/cffi/changeset/9267d5f13672/

Log:	Merge heads

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,10 +1,3 @@
 
 
-Next steps
-----------
-
-verify() handles "typedef ... some_integer_type", but this creates
-an opaque type that works like a struct (so we can't get the value
-out of it).
-
-accept and kill "static inline" in the cdefs
+Add other required types from stdint.h
diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -4765,7 +4765,7 @@
     CFieldObject *cf;
     Py_ssize_t offset;
 
-    if (!PyArg_ParseTuple(args, "O!O:typeof",
+    if (!PyArg_ParseTuple(args, "O!O:typeoffsetof",
                           &CTypeDescr_Type, &ct, &fieldname))
         return NULL;
 
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -208,6 +208,8 @@
 
     def _add_constants(self, key, val):
         if key in self._int_constants:
+            if self._int_constants[key] == val:
+                return     # ignore identical double declarations
             raise api.FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -235,7 +235,8 @@
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
             else:
-                converter = '_cffi_to_c_%s' % (tp.name.replace(' ', '_'),)
+                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                                                   tp.name.replace(' ', '_'))
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -274,8 +275,8 @@
         self._prnt('  if (datasize != 0) {')
         self._prnt('    if (datasize < 0)')
         self._prnt('      %s;' % errcode)
-        self._prnt('    %s = alloca(datasize);' % (tovar,))
-        self._prnt('    memset((void *)%s, 0, datasize);' % (tovar,))
+        self._prnt('    %s = alloca((size_t)datasize);' % (tovar,))
+        self._prnt('    memset((void *)%s, 0, (size_t)datasize);' % (tovar,))
         self._prnt('    if (_cffi_convert_array_from_object('
                    '(char *)%s, _cffi_type(%d), %s) < 0)' % (
             tovar, self._gettypenum(tp), fromvar))
@@ -835,12 +836,15 @@
             PyLong_FromLongLong((long long)(x)))
 
 #define _cffi_from_c_int(x, type)                                        \
-    (((type)-1) > 0 ?   /* unsigned */                                   \
-        (sizeof(type) < sizeof(long) ? PyInt_FromLong(x) :               \
-         sizeof(type) == sizeof(long) ? PyLong_FromUnsignedLong(x) :     \
-                                        PyLong_FromUnsignedLongLong(x))  \
-      : (sizeof(type) <= sizeof(long) ? PyInt_FromLong(x) :              \
-                                        PyLong_FromLongLong(x)))
+    (((type)-1) > 0 ? /* unsigned */                                     \
+        (sizeof(type) < sizeof(long) ?                                   \
+            PyInt_FromLong((long)x) :                                    \
+         sizeof(type) == sizeof(long) ?                                  \
+            PyLong_FromUnsignedLong((unsigned long)x) :                  \
+            PyLong_FromUnsignedLongLong((unsigned long long)x)) :        \
+        (sizeof(type) <= sizeof(long) ?                                  \
+            PyInt_FromLong((long)x) :                                    \
+            PyLong_FromLongLong((long long)x)))
 
 #define _cffi_to_c_int(o, type)                                          \
     (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o)        \
@@ -851,7 +855,7 @@
                                          : (type)_cffi_to_c_i32(o)) :    \
      sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o)       \
                                          : (type)_cffi_to_c_i64(o)) :    \
-     (Py_FatalError("unsupported size for type " #type), 0))
+     (Py_FatalError("unsupported size for type " #type), (type)0))
 
 #define _cffi_to_c_i8                                                    \
                  ((int(*)(PyObject *))_cffi_exports[1])
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -16,7 +16,8 @@
 class Verifier(object):
 
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
-                 ext_package=None, tag='', force_generic_engine=False, flags=None, **kwds):
+                 ext_package=None, tag='', force_generic_engine=False,
+                 source_extension='.c', flags=None, **kwds):
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
@@ -44,7 +45,7 @@
                                               k1, k2)
         suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or os.environ.get('CFFI_TMPDIR') or _caller_dir_pycache()
-        self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
+        self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension)
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
         self.ext_package = ext_package
         self._has_source = False
diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -667,7 +667,16 @@
    check.  Be sure to have other means of clearing the ``tmpdir``
    whenever you change your sources.
 
-.. versionadded:: 0.9
+.. versionadded:: 0.8.7
+   You can give C++ source code in ``ffi.verify()``::
+
+     ext = ffi.verify(r'''
+         extern "C" {
+             int somefunc(int somearg) { return real_cpp_func(somearg); }
+         }
+     ''', source_extension='.cpp', extra_compile_args=['-std=c++11'])
+
+.. versionadded:: 0.8.7
    The optional ``flags`` argument has been added, see ``man dlopen`` (ignored
    on Windows).  It defaults to ``ffi.RTLD_NOW``.
 
@@ -863,7 +872,9 @@
 
 .. versionadded:: 0.8.2
    The ``ffi.cdef()`` call takes an optional argument ``packed``: if
-   True, then all structs declared within this cdef are "packed".  This
+   True, then all structs declared within this cdef are "packed".
+   (If you need both packed and non-packed structs,
+   use several cdefs in sequence.)  This
    has a meaning similar to ``__attribute__((packed))`` in GCC.  It
    specifies that all structure fields should have an alignment of one
    byte.  (Note that the packed attribute has no effect on bit fields so
diff --git a/testing/backend_tests.py b/testing/backend_tests.py
--- a/testing/backend_tests.py
+++ b/testing/backend_tests.py
@@ -1,7 +1,7 @@
 import py
 import platform
 import sys, ctypes
-from cffi import FFI, CDefError
+from cffi import FFI, CDefError, FFIError
 from testing.support import *
 
 SIZE_OF_INT   = ctypes.sizeof(ctypes.c_int)
@@ -1564,6 +1564,12 @@
         p = ffi2.new("foo_p", [142])
         assert p.x == 142
 
+    def test_ignore_multiple_declarations_of_constant(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("#define FOO 42")
+        ffi.cdef("#define FOO 42")
+        py.test.raises(FFIError, ffi.cdef, "#define FOO 43")
+
     def test_struct_packed(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct nonpacked { char a; int b; };")
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -14,12 +14,13 @@
 else:
     if (sys.platform == 'darwin' and
           [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
+        # assume a standard clang or gcc
+        extra_compile_args = ['-Werror', '-Wconversion']
         # special things for clang
-        extra_compile_args = [
-            '-Werror', '-Qunused-arguments', '-Wno-error=shorten-64-to-32']
+        extra_compile_args.append('-Qunused-arguments')
     else:
         # assume a standard gcc
-        extra_compile_args = ['-Werror']
+        extra_compile_args = ['-Werror', '-Wconversion']
 
     class FFI(FFI):
         def verify(self, *args, **kwds):
@@ -89,11 +90,48 @@
     lib = ffi.verify('#include <math.h>', libraries=lib_m)
     assert lib.sin(1.23) == math.sin(1.23)
 
+def _Wconversion(cdef, source, **kargs):
+    if sys.platform == 'win32':
+        py.test.skip("needs GCC or Clang")
+    ffi = FFI()
+    ffi.cdef(cdef)
+    py.test.raises(VerificationError, ffi.verify, source, **kargs)
+    extra_compile_args_orig = extra_compile_args[:]
+    extra_compile_args.remove('-Wconversion')
+    try:
+        lib = ffi.verify(source, **kargs)
+    finally:
+        extra_compile_args[:] = extra_compile_args_orig
+    return lib
+
+def test_Wconversion_unsigned():
+    _Wconversion("unsigned foo(void);",
+                 "int foo(void) { return -1;}")
+
+def test_Wconversion_integer():
+    _Wconversion("short foo(void);",
+                 "long long foo(void) { return 1<<sizeof(short);}")
+
+def test_Wconversion_floating():
+    lib = _Wconversion("float sin(double);",
+                       "#include <math.h>", libraries=lib_m)
+    res = lib.sin(1.23)
+    assert res != math.sin(1.23)     # not exact, because of double->float
+    assert abs(res - math.sin(1.23)) < 1E-5
+
+def test_Wconversion_float2int():
+    _Wconversion("int sinf(float);",
+                 "#include <math.h>", libraries=lib_m)
+
+def test_Wconversion_double2int():
+    _Wconversion("int sin(double);",
+                 "#include <math.h>", libraries=lib_m)
+
 def test_rounding_1():
     ffi = FFI()
-    ffi.cdef("float sin(double x);")
+    ffi.cdef("double sinf(float x);")
     lib = ffi.verify('#include <math.h>', libraries=lib_m)
-    res = lib.sin(1.23)
+    res = lib.sinf(1.23)
     assert res != math.sin(1.23)     # not exact, because of double->float
     assert abs(res - math.sin(1.23)) < 1E-5
 
@@ -112,14 +150,21 @@
     assert lib.strlen(b"hi there!") == 9
 
 def test_strlen_approximate():
-    ffi = FFI()
-    ffi.cdef("int strlen(char *s);")
-    lib = ffi.verify("#include <string.h>")
+    lib = _Wconversion("int strlen(char *s);",
+                       "#include <string.h>")
     assert lib.strlen(b"hi there!") == 9
 
+def test_return_approximate():
+    for typename in ['short', 'int', 'long', 'long long']:
+        ffi = FFI()
+        ffi.cdef("%s foo(signed char x);" % typename)
+        lib = ffi.verify("signed char foo(signed char x) { return x;}")
+        assert lib.foo(-128) == -128
+        assert lib.foo(+127) == +127
+
 def test_strlen_array_of_char():
     ffi = FFI()
-    ffi.cdef("int strlen(char[]);")
+    ffi.cdef("size_t strlen(char[]);")
     lib = ffi.verify("#include <string.h>")
     assert lib.strlen(b"hello") == 5
 
@@ -208,8 +253,8 @@
     ffi = FFI()
     ffi.cdef('\n'.join(["%s foo_%s(%s);" % (tp, tp.replace(' ', '_'), tp)
                        for tp in typenames]))
-    lib = ffi.verify('\n'.join(["%s foo_%s(%s x) { return x+1; }" %
-                                (tp, tp.replace(' ', '_'), tp)
+    lib = ffi.verify('\n'.join(["%s foo_%s(%s x) { return (%s)(x+1); }" %
+                                (tp, tp.replace(' ', '_'), tp, tp)
                                 for tp in typenames]))
     for typename in typenames:
         foo = getattr(lib, 'foo_%s' % typename.replace(' ', '_'))
@@ -315,7 +360,7 @@
 def test_char_type():
     ffi = FFI()
     ffi.cdef("char foo(char);")
-    lib = ffi.verify("char foo(char x) { return x+1; }")
+    lib = ffi.verify("char foo(char x) { return ++x; }")
     assert lib.foo(b"A") == b"B"
     py.test.raises(TypeError, lib.foo, b"bar")
     py.test.raises(TypeError, lib.foo, "bar")
@@ -896,7 +941,7 @@
         static int foo(token_t *tk) {
             if (!tk)
                 return -42;
-            *tk += 1.601;
+            *tk += 1.601f;
             return (int)*tk;
         }
         #define TOKEN_SIZE sizeof(token_t)
@@ -991,7 +1036,7 @@
             long a;
         };
         int foo(struct foo_s s) {
-            return s.a - (int)s.b;
+            return (int)s.a - (int)s.b;
         }
     """)
     s = ffi.new("struct foo_s *", [100, 1])
@@ -1008,7 +1053,7 @@
             long a;
         };
         int foo1(struct foo_s s) {
-            return s.a - (int)s.b;
+            return (int)s.a - (int)s.b;
         }
         int (*foo)(struct foo_s s) = &foo1;
     """)
@@ -1067,7 +1112,7 @@
 def test_array_as_argument():
     ffi = FFI()
     ffi.cdef("""
-        int strlen(char string[]);
+        size_t strlen(char string[]);
     """)
     ffi.verify("#include <string.h>")
 
@@ -1079,7 +1124,7 @@
     """)
     lib = ffi.verify("""
         enum foo_e { AA, CC, BB };
-        int foo_func(enum foo_e e) { return e; }
+        int foo_func(enum foo_e e) { return (int)e; }
     """)
     assert lib.foo_func(lib.BB) == 2
     py.test.raises(TypeError, lib.foo_func, "BB")
@@ -1092,7 +1137,7 @@
     """)
     lib = ffi.verify("""
         enum foo_e { AA, CC, BB };
-        enum foo_e foo_func(int x) { return x; }
+        enum foo_e foo_func(int x) { return (enum foo_e)x; }
     """)
     assert lib.foo_func(lib.BB) == lib.BB == 2
 
@@ -1127,6 +1172,19 @@
     assert lib.AA == 0
     assert lib.BB == 2
 
+def test_typedef_enum_as_argument():
+    ffi = FFI()
+    ffi.cdef("""
+        typedef enum { AA, BB, ... } foo_t;
+        int foo_func(foo_t);
+    """)
+    lib = ffi.verify("""
+        typedef enum { AA, CC, BB } foo_t;
+        int foo_func(foo_t e) { return (int)e; }
+    """)
+    assert lib.foo_func(lib.BB) == lib.BB == 2
+    py.test.raises(TypeError, lib.foo_func, "BB")
+
 def test_typedef_enum_as_function_result():
     ffi = FFI()
     ffi.cdef("""
@@ -1135,7 +1193,7 @@
     """)
     lib = ffi.verify("""
         typedef enum { AA, CC, BB } foo_t;
-        foo_t foo_func(int x) { return x; }
+        foo_t foo_func(int x) { return (foo_t)x; }
     """)
     assert lib.foo_func(lib.BB) == lib.BB == 2
 
@@ -1676,7 +1734,7 @@
         static int c_callback(int how_many, ...) {
             va_list ap;
             /* collect the "..." arguments into the values[] array */
-            int i, *values = alloca(how_many * sizeof(int));
+            int i, *values = alloca((size_t)how_many * sizeof(int));
             va_start(ap, how_many);
             for (i=0; i<how_many; i++)
                 values[i] = va_arg(ap, int);
@@ -1717,7 +1775,7 @@
     ffi.cdef("char sum3chars(char *);")
     lib = ffi.verify("""
         char sum3chars(char *f) {
-            return f[0] + f[1] + f[2];
+            return (char)(f[0] + f[1] + f[2]);
         }
     """)
     assert lib.sum3chars((b'\x10', b'\x20', b'\x30')) == b'\x60'
@@ -1817,6 +1875,7 @@
     long tf_bl(signed char x, long c);
     unsigned long tf_bL(signed char x, unsigned long c);
     long long tf_bq(signed char x, long long c);
+    unsigned long long tf_bQ(signed char x, unsigned long long c);
     float tf_bf(signed char x, float c);
     double tf_bd(signed char x, double c);
     long double tf_bD(signed char x, long double c);
@@ -1834,20 +1893,35 @@
     double dvalue;
     long double Dvalue;
 
-    #define S(letter)  xvalue = x; letter##value = c; return rvalue;
+    typedef signed char b_t;
+    typedef unsigned char B_t;
+    typedef short h_t;
+    typedef unsigned short H_t;
+    typedef int i_t;
+    typedef unsigned int I_t;
+    typedef long l_t;
+    typedef unsigned long L_t;
+    typedef long long q_t;
+    typedef unsigned long long Q_t;
+    typedef float f_t;
+    typedef double d_t;
+    typedef long double D_t;
+    #define S(letter)  xvalue = (int)x; letter##value = (letter##_t)c;
+    #define R(letter)  return (letter##_t)rvalue;
 
-    signed char tf_bb(signed char x, signed char c) { S(i) }
-    unsigned char tf_bB(signed char x, unsigned char c) { S(i) }
-    short tf_bh(signed char x, short c) { S(i) }
-    unsigned short tf_bH(signed char x, unsigned short c) { S(i) }
-    int tf_bi(signed char x, int c) { S(i) }
-    unsigned int tf_bI(signed char x, unsigned int c) { S(i) }
-    long tf_bl(signed char x, long c) { S(i) }
-    unsigned long tf_bL(signed char x, unsigned long c) { S(i) }
-    long long tf_bq(signed char x, long long c) { S(i) }
-    float tf_bf(signed char x, float c) { S(f) }
-    double tf_bd(signed char x, double c) { S(d) }
-    long double tf_bD(signed char x, long double c) { S(D) }
+    signed char tf_bb(signed char x, signed char c) { S(i) R(b) }
+    unsigned char tf_bB(signed char x, unsigned char c) { S(i) R(B) }
+    short tf_bh(signed char x, short c) { S(i) R(h) }
+    unsigned short tf_bH(signed char x, unsigned short c) { S(i) R(H) }
+    int tf_bi(signed char x, int c) { S(i) R(i) }
+    unsigned int tf_bI(signed char x, unsigned int c) { S(i) R(I) }
+    long tf_bl(signed char x, long c) { S(i) R(l) }
+    unsigned long tf_bL(signed char x, unsigned long c) { S(i) R(L) }
+    long long tf_bq(signed char x, long long c) { S(i) R(q) }
+    unsigned long long tf_bQ(signed char x, unsigned long long c) { S(i) R(Q) }
+    float tf_bf(signed char x, float c) { S(f) R(f) }
+    double tf_bd(signed char x, double c) { S(d) R(d) }
+    long double tf_bD(signed char x, long double c) { S(D) R(D) }
     """)
     lib.rvalue = 0x7182838485868788
     for kind, cname in [('b', 'signed char'),
@@ -1859,6 +1933,7 @@
                         ('l', 'long'),
                         ('L', 'unsigned long'),
                         ('q', 'long long'),
+                        ('Q', 'unsigned long long'),
                         ('f', 'float'),
                         ('d', 'double'),
                         ('D', 'long double')]:

From noreply at buildbot.pypy.org  Wed Dec 17 18:28:09 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 17 Dec 2014 18:28:09 +0100 (CET)
Subject: [pypy-commit] pypy default: Kill an obscure feature about unrolling
 - one that generates same_as
Message-ID: <20141217172809.B3AD11D3681@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r74989:5693f914b241
Date: 2014-12-17 19:27 +0200
http://bitbucket.org/pypy/pypy/changeset/5693f914b241/

Log:	Kill an obscure feature about unrolling - one that generates same_as
	in order to pass one box twice as two unrelated boxes into the
	label() of a loop, both produced by short preamble.

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1181,7 +1181,7 @@
         i1 = getfield_gc(p0, descr=valuedescr)
         i2 = int_sub(i1, 1)
         i3 = int_add(i0, i1)
-        i4 = same_as(i2) # This same_as should be killed by backend
+        #i4 = same_as(i2) # This same_as should be killed by backend
         jump(i3, i1, i2)
         """
         expected = """
@@ -1317,7 +1317,7 @@
         p30 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p30, i28, descr=nextdescr)
         setfield_gc(p3, p30, descr=valuedescr)
-        p46 = same_as(p30) # This same_as should be killed by backend
+        #p46 = same_as(p30) # This same_as should be killed by backend
         jump(i29, p30, p3)
         """
         expected = """
@@ -2414,16 +2414,16 @@
         guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(p1, i2, descr=valuedescr)
-        i7 = same_as(i2) # This same_as should be killed by backend
-        i6 = same_as(i4)
-        jump(p1, i1, i2, i4, i6)
-        """
-        expected = """
-        [p1, i1, i2, i4, i5]
+        #i7 = same_as(i2) # This same_as should be killed by backend
+        #i6 = same_as(i4)
+        jump(p1, i1, i2, i4)
+        """
+        expected = """
+        [p1, i1, i2, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         guard_true(i4) []
         setfield_gc(p1, i2, descr=valuedescr)
-        jump(p1, i1, i2, i5, i5)
+        jump(p1, i1, i2, 1)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -2446,15 +2446,15 @@
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         escape()
-        i5 = same_as(i4)
-        jump(p1, i2, i4, i5)
-        """
-        expected = """
-        [p1, i2, i4, i5]
+        #i5 = same_as(i4)
+        jump(p1, i2, i4)
+        """
+        expected = """
+        [p1, i2, i4]
         guard_true(i4) [p1]
         setfield_gc(p1, NULL, descr=nextdescr)
         escape()
-        jump(p1, i2, i5, i5)
+        jump(p1, i2, 1)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -2476,15 +2476,15 @@
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         escape()
-        i5 = same_as(i4)
-        jump(p1, i2, i4, i5)
-        """
-        expected = """
-        [p1, i2, i4, i5]
+        #i5 = same_as(i4)
+        jump(p1, i2, i4)
+        """
+        expected = """
+        [p1, i2, i4]
         guard_true(i4) [i2, p1]
         setfield_gc(p1, NULL, descr=nextdescr)
         escape()
-        jump(p1, i2, i5, i5)
+        jump(p1, i2, 1)
         """
         self.optimize_loop(ops, expected)
 
@@ -2507,17 +2507,17 @@
         guard_true(i5) []
         i4 = int_neg(i2)
         setfield_gc(p1, i2, descr=valuedescr)
-        i8 = same_as(i2) # This same_as should be killed by backend
-        i7 = same_as(i4)
-        jump(p1, i1, i2, i4, i7)
-        """
-        expected = """
-        [p1, i1, i2, i4, i7]
+        #i8 = same_as(i2) # This same_as should be killed by backend
+        #i7 = same_as(i4)
+        jump(p1, i1, i2, i4)
+        """
+        expected = """
+        [p1, i1, i2, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i5 = int_eq(i4, 5)
         guard_true(i5) []
         setfield_gc(p1, i2, descr=valuedescr)
-        jump(p1, i1, i2, i7, i7)
+        jump(p1, i1, i2, 5)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -2735,16 +2735,16 @@
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        i101 = same_as(i4)
-        jump(p1, i2, i4, p4, i101)
-        """
-        expected = """
-        [p1, i2, i4, p4, i5]
+        #i101 = same_as(i4)
+        jump(p1, i2, i4, p4)
+        """
+        expected = """
+        [p1, i2, i4, p4]
         guard_true(i4) [p1, p4]
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        jump(p1, i2, i5, p4, i5)
+        jump(p1, i2, 1, p4)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -3620,7 +3620,7 @@
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call_assembler(i1, descr=asmdescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        i143 = same_as(i3) # Should be killed by backend
+        #i143 = same_as(i3) # Should be killed by backend
         jump(p1, i4, i3)
         '''
         self.optimize_loop(ops, ops, preamble)
@@ -3645,18 +3645,18 @@
         jump(p1, i4, i3)
         '''
         expected = '''
-        [p1, i4, i3, i5]
-        setfield_gc(p1, i5, descr=valuedescr)
-        jump(p1, i3, i5, i5)
+        [p1, i4, i3]
+        setfield_gc(p1, i3, descr=valuedescr)
+        jump(p1, i3, i3)
         '''
         preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        i148 = same_as(i3)
-        i147 = same_as(i3)
-        jump(p1, i4, i3, i148)
+        #i148 = same_as(i3)
+        #i147 = same_as(i3)
+        jump(p1, i4, i3)
         '''
         self.optimize_loop(ops, expected, preamble)
 
@@ -3670,17 +3670,17 @@
         jump(p1, i4, i3)
         '''
         expected = '''
-        [p1, i4, i3, i5]
+        [p1, i4, i3]
         setfield_gc(p1, i4, descr=valuedescr)
-        jump(p1, i3, i5, i5)
+        jump(p1, i3, i3)
         '''
         preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i1, descr=valuedescr)
-        i151 = same_as(i3)
-        jump(p1, i4, i3, i151)
+        #i151 = same_as(i3)
+        jump(p1, i4, i3)
         '''
         self.optimize_loop(ops, expected, preamble)
 
@@ -3700,14 +3700,14 @@
         escape(i1)
         escape(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        i153 = same_as(i4)
-        jump(i0, i4, i153)
+        #i153 = same_as(i4)
+        jump(i0, i4)
         '''
         expected = '''
-        [i0, i4, i5]
+        [i0, i4]
         escape(42)
         escape(i4)
-        jump(i0, i5, i5)
+        jump(i0, i4)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
@@ -3731,14 +3731,14 @@
         escape(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
         guard_no_exception() []
-        i155 = same_as(i4)
-        jump(i0, i4, i155)
+        #i155 = same_as(i4)
+        jump(i0, i4)
         '''
         expected = '''
-        [i0, i2, i3]
+        [i0, i2]
         escape(42)
         escape(i2)
-        jump(i0, i3, i3)
+        jump(i0, i2)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
@@ -5864,11 +5864,12 @@
         [p0]
         p1 = getfield_gc(p0, descr=valuedescr)
         setfield_gc(p0, p0, descr=valuedescr)
-        p4450 = same_as(p0) # Should be killed by backend
+        #p4450 = same_as(p0) # Should be killed by backend
         jump(p0)
         """
         expected = """
         [p0]
+        p1 = getfield_gc(p0, descr=valuedescr)
         setfield_gc(p0, p0, descr=valuedescr)
         jump(p0)
         """
@@ -6264,13 +6265,13 @@
         [p1, i1, i2, i3]
         escape(i3)
         i4 = int_sub(i2, i1)
-        i5 = same_as(i4)
-        jump(p1, i1, i2, i4, i5)
-        """
-        expected = """
-        [p1, i1, i2, i3, i4]
+        #i5 = same_as(i4)
+        jump(p1, i1, i2, i4)
+        """
+        expected = """
+        [p1, i1, i2, i3]
         escape(i3)
-        jump(p1, i1, i2, i4, i4)
+        jump(p1, i1, i2, i3)
         """
         self.optimize_strunicode_loop(ops, expected, preamble)
 
@@ -6290,14 +6291,15 @@
         escape(i5)
         i4 = int_sub(i2, i1)
         setfield_gc(p2, i4, descr=valuedescr)
-        i8 = same_as(i4)
-        jump(p1, i1, i2, p2, i8, i4)
-        """
-        expected = """
-        [p1, i1, i2, p2, i5, i6]
-        escape(i5)
-        setfield_gc(p2, i6, descr=valuedescr)
-        jump(p1, i1, i2, p2, i6, i6)
+        #i8 = same_as(i4)
+        jump(p1, i1, i2, p2, i4)
+        """
+        expected = """
+        [p1, i1, i2, p2, i5]
+        i311 = getfield_gc(p2, descr=valuedescr)
+        escape(i311)
+        setfield_gc(p2, i5, descr=valuedescr)
+        jump(p1, i1, i2, p2, i5)
         """
         self.optimize_strunicode_loop(ops, expected, preamble)
 
@@ -7104,19 +7106,18 @@
         p188 = getarrayitem_gc(p187, 42, descr=<GcPtrArrayDescr>)
         guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
-        p26 = same_as(p25)
-        jump(p25, p187, i184, p26)
+        jump(p25, p187, i184)
         """
         short = """
         [p1, p187, i184]
         p188 = getarrayitem_gc(p187, 42, descr=<GcPtrArrayDescr>)
         guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
-        jump(p1, p187, i184, p25)
-        """
-        expected = """
-        [p25, p187, i184, p189]
-        jump(p189, p187, i184, p189)
+        jump(p25, p187, i184)
+        """
+        expected = """
+        [p25, p187, i184]
+        jump(p25, p187, i184)
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
@@ -7225,14 +7226,16 @@
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
         setfield_gc(p1, ii, descr=otherdescr)
-        jump(p0, p1, ii2, ii, ii, ii)
-        """
-        expected = """
-        [p0, p1, ii, ii2, i1, i2]
+        jump(p0, p1, ii2, ii)
+        """
+        expected = """
+        [p0, p1, ii, ii2]
+        i1 = getfield_gc(p0, descr=valuedescr)
+        i2 = getfield_gc(p1, descr=otherdescr)
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
         setfield_gc(p1, ii, descr=otherdescr)
-        jump(p0, p1, ii2, ii, ii, ii)
+        jump(p0, p1, ii2, ii)
         """
         self.optimize_loop(ops, expected)
 
@@ -7723,10 +7726,10 @@
         jump(p1)
         """
         expected = """
-        [p0, p1]
+        [p0]
         call(p0, descr=nonwritedescr)
-        call(p1, descr=writeadescr)
-        jump(p1, p1)
+        call(p0, descr=writeadescr)
+        jump(p0)
         """
         self.optimize_loop(ops, expected)
 
@@ -7837,10 +7840,11 @@
         jump(p5, p6)
         """
         expected = """
-        [p5, p6, i10, i11]
+        [p5, p6, i10]
+        i11 = getfield_gc(p6, descr=nextdescr)
         call(i10, i11, descr=nonwritedescr)
         setfield_gc(p6, i10, descr=nextdescr)
-        jump(p5, p6, i10, i10)
+        jump(p5, p6, i10)
         """
         self.optimize_loop(ops, expected)
 
@@ -7856,11 +7860,12 @@
         jump(p5, p6)
         """
         expected = """
-        [p5, p6, i14, i12, i10]
-        i13 = int_add(i14, 7)
+        [p5, p6, i12, i10]
+        i11 = getfield_gc(p6, descr=nextdescr)
+        i13 = int_add(i11, 7)
         call(i12, i13, descr=nonwritedescr)
         setfield_gc(p6, i10, descr=nextdescr)
-        jump(p5, p6, i10, i12, i10)
+        jump(p5, p6, i12, i10)
         """
         self.optimize_loop(ops, expected)
 
@@ -7980,9 +7985,9 @@
         jump(i1, i3)
         """
         expected = """
-        [i1, i2, i6, i3]
+        [i1, i2, i6]
         call(i6, descr=nonwritedescr)
-        jump(i1, i3, i6, i3)
+        jump(i1, i2, i6)
         """
         short = """
         [i1, i2]
@@ -7990,7 +7995,7 @@
         i4 = int_add(i3, i3)
         i5 = int_add(i4, i4)
         i6 = int_add(i5, i5)
-        jump(i1, i2, i6, i3)
+        jump(i1, i3, i6)
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
@@ -8145,19 +8150,18 @@
         call(i2, descr=nonwritedescr)
         setfield_gc(p22, i1, descr=valuedescr)
         guard_nonnull_class(p18, ConstClass(node_vtable)) []
-        i10 = same_as(i1)
-        jump(p22, p18, i1, i10)
+        jump(p22, p18, i1)
         """
         short = """
         [p22, p18, i1]
-        i2 = getfield_gc(p22, descr=valuedescr)
-        jump(p22, p18, i1, i2)
-        """
-        expected = """
-        [p22, p18, i1, i2]
-        call(i2, descr=nonwritedescr)
-        setfield_gc(p22, i1, descr=valuedescr)
-        jump(p22, p18, i1, i1)
+        jump(p22, p18, i1)
+        """
+        expected = """
+        [p22, p18, i2]
+        i1 = getfield_gc(p22, descr=valuedescr)
+        call(i1, descr=nonwritedescr)
+        setfield_gc(p22, i2, descr=valuedescr)
+        jump(p22, p18, i2)
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -638,6 +638,7 @@
             for i in range(op.numargs()):
                 op.setarg(i, self.renamed(op.getarg(i)))
         if box in self.short_boxes:
+            return
             if op is None:
                 oldop = self.short_boxes[box].clone()
                 oldres = oldop.result

From noreply at buildbot.pypy.org  Wed Dec 17 18:33:24 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 17 Dec 2014 18:33:24 +0100 (CET)
Subject: [pypy-commit] pypy optresult: progress
Message-ID: <20141217173324.CDFA11D367E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74990:e42a2a263bf7
Date: 2014-12-17 19:29 +0200
http://bitbucket.org/pypy/pypy/changeset/e42a2a263bf7/

Log:	progress

diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -1,6 +1,6 @@
 from rpython.jit.metainterp.history import (ConstInt, BoxInt, ConstFloat,
     BoxFloat, TargetToken)
-from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.resoperation import rop, AbstractInputArg
 from rpython.rlib.debug import (have_debug_prints, debug_start, debug_stop,
     debug_print)
 from rpython.rlib.objectmodel import we_are_translated, compute_unique_id
@@ -132,6 +132,8 @@
             return '?'
 
     def repr_of_resop(self, op, ops_offset=None):
+        if isinstance(op, AbstractInputArg):
+            return self.repr_of_arg(op)
         if op.getopnum() == rop.DEBUG_MERGE_POINT:
             jd_sd = self.metainterp_sd.jitdrivers_sd[op.getarg(0).getint()]
             s = jd_sd.warmstate.get_location_str(op.getarglist()[3:])
@@ -196,6 +198,8 @@
             offset = ops_offset[None]
             debug_print("+%d: --end of the loop--" % offset)
 
+    def log_loop(self, loop):
+        self._log_operations(loop.inputargs, loop.operations)
 
 def int_could_be_an_address(x):
     if we_are_translated():
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -151,7 +151,11 @@
                     getop.source_op = result
                     if isinstance(result, Const):
                         optimizer.make_constant(getop, result)
-                    shortboxes.add_potential(result, getop, synthetic=True)
+                        getop.is_source_op = True
+                        shortboxes.add_potential(getop, getop, synthetic=True)
+                    else:
+                        getop.source_op = result                        
+                        shortboxes.add_potential(result, getop, synthetic=True)
                 if op.getopnum() == rop.SETARRAYITEM_GC:
                     result = op.getarg(2)
                     opnum = OpHelpers.getarrayitem_for_descr(op.getdescr())
@@ -159,6 +163,7 @@
                                          op.getdescr())
                     getop.source_op = result
                     if isinstance(result, Const):
+                        xxx
                         optimizer.make_constant(getop, result)
                     shortboxes.add_potential(result, getop, synthetic=True)
                 elif op.type != 'v':
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -1,7 +1,7 @@
 from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, maxint, is_valid_int
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp.resoperation import rop, ResOperation
-from rpython.jit.metainterp.history import BoxInt, ConstInt
+from rpython.jit.metainterp.history import ConstInt
 
 MAXINT = maxint
 MININT = -maxint - 1
@@ -238,17 +238,15 @@
     def make_guards(self, box, guards):
         if self.has_lower and self.lower > MININT:
             bound = self.lower
-            res = BoxInt()
-            op = ResOperation(rop.INT_GE, [box, ConstInt(bound)], res)
+            op = ResOperation(rop.INT_GE, [box, ConstInt(bound)])
             guards.append(op)
-            op = ResOperation(rop.GUARD_TRUE, [res], None)
+            op = ResOperation(rop.GUARD_TRUE, [op])
             guards.append(op)
         if self.has_upper and self.upper < MAXINT:
             bound = self.upper
-            res = BoxInt()
-            op = ResOperation(rop.INT_LE, [box, ConstInt(bound)], res)
+            op = ResOperation(rop.INT_LE, [box, ConstInt(bound)])
             guards.append(op)
-            op = ResOperation(rop.GUARD_TRUE, [res], None)
+            op = ResOperation(rop.GUARD_TRUE, [op])
             guards.append(op)
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -105,7 +105,7 @@
                 # once when parsing 'preamble', once when parsing 'expected'.
                 self.oparse = oparse
                 self.rd_frame_info_list, self.rd_snapshot = snapshot(fail_args)
-            def _clone_if_mutable(self):
+            def _clone_if_mutable(self, memo):
                 assert self is fdescr
                 return fdescr2
             def __repr__(self):
@@ -1809,7 +1809,7 @@
     def test_virtual_raw_malloc_force(self):
         ops = """
         [i1]
-        i2 = call('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
@@ -1817,20 +1817,20 @@
         setarrayitem_raw(i2, 1, 789, descr=rawarraydescr_float)
         label('foo') # we expect the buffer to be forced *after* the label
         escape_n(i2)
-        call('free', i2, descr=raw_free_descr)
+        call_n('free', i2, descr=raw_free_descr)
         jump(i1)
         """
         expected = """
         [i1]
         label('foo')
-        i2 = call('malloc', 20, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 20, descr=raw_malloc_descr)
         #guard_no_exception() []  # XXX should appear
         raw_store(i2, 0, i1, descr=rawarraydescr_char)
         raw_store(i2, 1, 123, descr=rawarraydescr_char)
         raw_store(i2, 2, 456, descr=rawarraydescr_char)
         raw_store(i2, 8, 789, descr=rawarraydescr_float)
         escape_n(i2)
-        call('free', i2, descr=raw_free_descr)
+        call_n('free', i2, descr=raw_free_descr)
         jump(i1)
         """
         self.optimize_loop(ops, expected)
@@ -1838,22 +1838,22 @@
     def test_virtual_raw_malloc_invalid_write_force(self):
         ops = """
         [i1]
-        i2 = call('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label('foo') # we expect the buffer to be forced *after* the label
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) # overlap!
-        call('free', i2, descr=raw_free_descr)
+        call_n('free', i2, descr=raw_free_descr)
         jump(i1)
         """
         expected = """
         [i1]
         label('foo')
-        i2 = call('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
         #guard_no_exception() []  # XXX should appear
         raw_store(i2, 0, i1, descr=rawarraydescr)
         setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char)
-        call('free', i2, descr=raw_free_descr)
+        call_n('free', i2, descr=raw_free_descr)
         jump(i1)
         """
         self.optimize_loop(ops, expected)
@@ -1861,22 +1861,22 @@
     def test_virtual_raw_malloc_invalid_read_force(self):
         ops = """
         [i1]
-        i2 = call('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, i1, descr=rawarraydescr)
         label('foo') # we expect the buffer to be forced *after* the label
-        i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char)
-        call('free', i2, descr=raw_free_descr)
+        i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr_char)
+        call_n('free', i2, descr=raw_free_descr)
         jump(i1)
         """
         expected = """
         [i1]
         label('foo')
-        i2 = call('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
         #guard_no_exception() []  # XXX should appear
         raw_store(i2, 0, i1, descr=rawarraydescr)
-        i3 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char)
-        call('free', i2, descr=raw_free_descr)
+        i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr_char)
+        call_n('free', i2, descr=raw_free_descr)
         jump(i1)
         """
         self.optimize_loop(ops, expected)
@@ -1884,15 +1884,15 @@
     def test_virtual_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
         setarrayitem_raw(i3, 0, 4242, descr=rawarraydescr) # write to the slice
-        i4 = getarrayitem_raw(i2, 0, descr=rawarraydescr_char)
+        i4 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr_char)
         i5 = int_add(i2, 1)
-        i6 = getarrayitem_raw(i5, 0, descr=rawarraydescr)
-        call('free', i2, descr=raw_free_descr)
+        i6 = getarrayitem_raw_i(i5, 0, descr=rawarraydescr)
+        call_n('free', i2, descr=raw_free_descr)
         jump(i0, i1)
         """
         expected = """
@@ -1904,13 +1904,13 @@
     def test_virtual_raw_slice_of_a_raw_slice(self):
         ops = """
         [i0, i1]
-        i2 = call('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
         guard_no_exception() []
         i3 = int_add(i2, 1) # get a slice of the original buffer
         i4 = int_add(i3, 1) # get a slice of a slice
         setarrayitem_raw(i4, 0, i1, descr=rawarraydescr_char) # write to the slice
-        i5 = getarrayitem_raw(i2, 2, descr=rawarraydescr_char)
-        call('free', i2, descr=raw_free_descr)
+        i5 = getarrayitem_raw_i(i2, 2, descr=rawarraydescr_char)
+        call_n('free', i2, descr=raw_free_descr)
         jump(i0, i5)
         """
         expected = """
@@ -1922,7 +1922,7 @@
     def test_virtual_raw_slice_force(self):
         ops = """
         [i0, i1]
-        i2 = call('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char)
         i3 = int_add(i2, 1) # get a slice of the original buffer
@@ -1935,7 +1935,7 @@
         [i0, i1]
         label('foo')
         # these ops are generated by VirtualRawBufferValue._really_force
-        i2 = call('malloc', 10, descr=raw_malloc_descr)
+        i2 = call_i('malloc', 10, descr=raw_malloc_descr)
         #guard_no_exception() []  # XXX should appear
         raw_store(i2, 0, 42, descr=rawarraydescr_char)
         raw_store(i2, 5, 4242, descr=rawarraydescr_char)
@@ -1949,10 +1949,10 @@
     def test_virtual_raw_malloc_virtualstate(self):
         ops = """
         [i0]
-        i1 = getarrayitem_raw(i0, 0, descr=rawarraydescr)
+        i1 = getarrayitem_raw_i(i0, 0, descr=rawarraydescr)
         i2 = int_add(i1, 1)
-        call('free', i0, descr=raw_free_descr)
-        i3 = call('malloc', 10, descr=raw_malloc_descr)
+        call_n('free', i0, descr=raw_free_descr)
+        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i3, 0, i2, descr=rawarraydescr)
         label('foo')
@@ -1960,11 +1960,11 @@
         """
         expected = """
         [i0]
-        i1 = getarrayitem_raw(i0, 0, descr=rawarraydescr)
+        i1 = getarrayitem_raw_i(i0, 0, descr=rawarraydescr)
         i2 = int_add(i1, 1)
-        call('free', i0, descr=raw_free_descr)
+        call_n('free', i0, descr=raw_free_descr)
         label('foo')
-        i3 = call('malloc', 10, descr=raw_malloc_descr)
+        i3 = call_i('malloc', 10, descr=raw_malloc_descr)
         #guard_no_exception() []  # XXX should appear
         raw_store(i3, 0, i2, descr=rawarraydescr)
         jump(i3)
@@ -1974,12 +1974,12 @@
     def test_virtual_raw_store_raw_load(self):
         ops = """
         [i1]
-        i0 = call('malloc', 10, descr=raw_malloc_descr)
+        i0 = call_i('malloc', 10, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 0, i1, descr=rawarraydescr)
-        i2 = raw_load(i0, 0, descr=rawarraydescr)
+        i2 = raw_load_i(i0, 0, descr=rawarraydescr)
         i3 = int_add(i1, i2)
-        call('free', i0, descr=raw_free_descr)
+        call_n('free', i0, descr=raw_free_descr)
         jump(i3)
         """
         expected = """
@@ -1992,12 +1992,12 @@
     def test_virtual_raw_store_getarrayitem_raw(self):
         ops = """
         [f1]
-        i0 = call('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
         guard_no_exception() []
         raw_store(i0, 8, f1, descr=rawarraydescr_float)
-        f2 = getarrayitem_raw(i0, 1, descr=rawarraydescr_float)
+        f2 = getarrayitem_raw_f(i0, 1, descr=rawarraydescr_float)
         f3 = float_add(f1, f2)
-        call('free', i0, descr=raw_free_descr)
+        call_n('free', i0, descr=raw_free_descr)
         jump(f3)
         """
         expected = """
@@ -2010,12 +2010,12 @@
     def test_virtual_setarrayitem_raw_raw_load(self):
         ops = """
         [f1]
-        i0 = call('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
         guard_no_exception() []
         setarrayitem_raw(i0, 1, f1, descr=rawarraydescr_float)
-        f2 = raw_load(i0, 8, descr=rawarraydescr_float)
+        f2 = raw_load_f(i0, 8, descr=rawarraydescr_float)
         f3 = float_add(f1, f2)
-        call('free', i0, descr=raw_free_descr)
+        call_n('free', i0, descr=raw_free_descr)
         jump(f3)
         """
         expected = """
@@ -2028,7 +2028,7 @@
     def test_virtual_raw_buffer_forced_but_slice_not_forced(self):
         ops = """
         [f1]
-        i0 = call('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
         guard_no_exception() []
         i1 = int_add(i0, 8)
         escape_n(i0)
@@ -2037,7 +2037,7 @@
         """
         expected = """
         [f1]
-        i0 = call('malloc', 16, descr=raw_malloc_descr)
+        i0 = call_i('malloc', 16, descr=raw_malloc_descr)
         #guard_no_exception() []  # XXX should appear
         escape_n(i0)
         i1 = int_add(i0, 8)
@@ -2263,14 +2263,18 @@
         setfield_gc(p1, i2, descr=valuedescr)
         jump(p1, i1, i2)
         """
-        expected = """
+        preamble = """
         [p1, i1, i2]
         setfield_gc(p1, i2, descr=valuedescr)
         jump(p1, i1, i2)
         """
+        expected = """
+        [p1, i1, i2]
+        jump(p1, i1, i2)
+        """
         # in this case, all setfields are removed, because we can prove
         # that in the loop it will always have the same value
-        self.optimize_loop(ops, expected)
+        self.optimize_loop(ops, expected, preamble)
 
     def test_duplicate_setfield_1(self):
         ops = """
@@ -2326,7 +2330,7 @@
         setfield_gc(p1, i1, descr=valuedescr)
         #
         # some operations on which the above setfield_gc cannot have effect
-        i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr)
+        i3 = getarrayitem_gc_pure_i(p3, 1, descr=arraydescr)
         i4 = getarrayitem_gc_i(p3, i3, descr=arraydescr)
         i5 = int_add(i3, i4)
         setarrayitem_gc(p3, 0, i5, descr=arraydescr)
@@ -2339,7 +2343,7 @@
         preamble = """
         [p1, i1, i2, p3]
         #
-        i3 = getarrayitem_gc_pure(p3, 1, descr=arraydescr)
+        i3 = getarrayitem_gc_pure_i(p3, 1, descr=arraydescr)
         i4 = getarrayitem_gc_i(p3, i3, descr=arraydescr)
         i5 = int_add(i3, i4)
         #
@@ -2370,7 +2374,7 @@
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p0, p1, descr=nextdescr)
         setfield_raw(i1, i1, descr=valuedescr)    # random op with side-effects
-        p2 = getfield_gc(p0, descr=nextdescr)
+        p2 = getfield_gc_r(p0, descr=nextdescr)
         i2 = getfield_gc_i(p2, descr=valuedescr)
         setfield_gc(p0, NULL, descr=nextdescr)
         escape_n(i2)
@@ -2410,8 +2414,8 @@
         guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(p1, i2, descr=valuedescr)
-        i7 = same_as(i2) # This same_as should be killed by backend
-        i6 = same_as(i4)
+        i7 = same_as_i(i2) # This same_as should be killed by backend
+        i6 = same_as_i(i4)
         jump(p1, i1, i2, i4, i6)
         """
         expected = """
@@ -2793,10 +2797,10 @@
         [p1]
         guard_nonnull(p1) []
         guard_class(p1, ConstClass(node_vtable2)) []
-        p2 = getfield_gc(p1, descr=nextdescr)
+        p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_nonnull(12) []
         guard_class(p2, ConstClass(node_vtable)) []
-        p3 = getfield_gc(p1, descr=otherdescr)
+        p3 = getfield_gc_r(p1, descr=otherdescr)
         guard_nonnull(12) []
         guard_class(p3, ConstClass(node_vtable)) []
         setfield_gc(p3, p2, descr=otherdescr)
@@ -2811,9 +2815,9 @@
         preamble = """
         [p1]
         guard_nonnull_class(p1, ConstClass(node_vtable2)) []
-        p2 = getfield_gc(p1, descr=nextdescr)
+        p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_class(p2, ConstClass(node_vtable)) []
-        p3 = getfield_gc(p1, descr=otherdescr)
+        p3 = getfield_gc_r(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
@@ -2842,10 +2846,10 @@
         [p1]
         guard_nonnull(p1) []
         guard_class(p1, ConstClass(node_vtable2)) []
-        p2 = getfield_gc(p1, descr=nextdescr)
+        p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_nonnull(12) []
         guard_class(p2, ConstClass(node_vtable)) []
-        p3 = getfield_gc(p1, descr=otherdescr)
+        p3 = getfield_gc_r(p1, descr=otherdescr)
         guard_nonnull(12) []
         guard_class(p3, ConstClass(node_vtable)) []
         p1a = new_with_vtable(ConstClass(node_vtable2))
@@ -2860,9 +2864,9 @@
         preamble = """
         [p1]
         guard_nonnull_class(p1, ConstClass(node_vtable2)) []
-        p2 = getfield_gc(p1, descr=nextdescr)
+        p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_class(p2, ConstClass(node_vtable)) []
-        p3 = getfield_gc(p1, descr=otherdescr)
+        p3 = getfield_gc_r(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         p3a = new_with_vtable(ConstClass(node_vtable))
@@ -2907,11 +2911,11 @@
         ops = """
         [p0]
         i0 = escape_i()
-        i2 = getfield_gc(p0, descr=valuedescr)
+        i2 = getfield_gc_i(p0, descr=valuedescr)
         i4 = int_add(i2, 1)
         setfield_gc(p0, i4, descr=valuedescr)
         guard_true(i0) []
-        i6 = getfield_gc(p0, descr=valuedescr)
+        i6 = getfield_gc_i(p0, descr=valuedescr)
         i8 = int_sub(i6, 1)
         setfield_gc(p0, i8, descr=valuedescr)
         escape_n()
@@ -2920,7 +2924,7 @@
         expected = """
         [p0]
         i0 = escape_i()
-        i2 = getfield_gc(p0, descr=valuedescr)
+        i2 = getfield_gc_i(p0, descr=valuedescr)
         i4 = int_add(i2, 1)
         setfield_gc(p0, i4, descr=valuedescr)
         guard_true(i0) []
@@ -2954,7 +2958,7 @@
     def test_invalid_loop_3(self):
         ops = """
         [p1]
-        p2 = getfield_gc(p1, descr=nextdescr)
+        p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_isnull(p2) []
         #
         p3 = new_with_vtable(ConstClass(node_vtable))
@@ -3092,8 +3096,8 @@
     def test_oois_of_itself(self):
         ops = """
         [p0]
-        p1 = getfield_gc(p0, descr=nextdescr)
-        p2 = getfield_gc(p0, descr=nextdescr)
+        p1 = getfield_gc_r(p0, descr=nextdescr)
+        p2 = getfield_gc_r(p0, descr=nextdescr)
         i1 = ptr_eq(p1, p2)
         guard_true(i1) []
         i2 = ptr_ne(p1, p2)
@@ -3102,7 +3106,7 @@
         """
         preamble = """
         [p0]
-        p1 = getfield_gc(p0, descr=nextdescr)
+        p1 = getfield_gc_r(p0, descr=nextdescr)
         jump(p0)
         """
         expected = """
@@ -3245,8 +3249,8 @@
         [p8, p11, i24]
         p26 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p26, i24, descr=adescr)
-        i34 = getfield_gc_pure(p11, descr=valuedescr)
-        i35 = getfield_gc_pure(p26, descr=adescr)
+        i34 = getfield_gc_pure_i(p11, descr=valuedescr)
+        i35 = getfield_gc_pure_i(p26, descr=adescr)
         i36 = int_add_ovf(i34, i35)
         guard_no_overflow() []
         jump(p8, p11, i35)
@@ -3260,21 +3264,21 @@
     def test_ovf_guard_in_short_preamble2(self):
         ops = """
         [p8, p11, p12]
-        p16 = getfield_gc(p8, descr=valuedescr)
-        i17 = getfield_gc(p8, descr=nextdescr)
-        i19 = getfield_gc(p16, descr=valuedescr)
+        p16 = getfield_gc_r(p8, descr=valuedescr)
+        i17 = getfield_gc_i(p8, descr=nextdescr)
+        i19 = getfield_gc_i(p16, descr=valuedescr)
         i20 = int_ge(i17, i19)
         guard_false(i20) []
-        i21 = getfield_gc(p16, descr=otherdescr)
-        i22 = getfield_gc(p16, descr=nextdescr)
+        i21 = getfield_gc_i(p16, descr=otherdescr)
+        i22 = getfield_gc_i(p16, descr=nextdescr)
         i23 = int_mul(i17, i22)
         i24 = int_add(i21, i23)
         p26 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p26, i24, descr=adescr)
         i28 = int_add(i17, 1)
         setfield_gc(p8, i28, descr=nextdescr)
-        i34 = getfield_gc_pure(p11, descr=valuedescr)
-        i35 = getfield_gc_pure(p26, descr=adescr)
+        i34 = getfield_gc_pure_i(p11, descr=valuedescr)
+        i35 = getfield_gc_pure_i(p26, descr=adescr)
         guard_nonnull(p12) []
         i36 = int_add_ovf(i34, i35)
         guard_no_overflow() []
@@ -3286,7 +3290,7 @@
         [p8, p11, i24, i39, i19, p16, i21, i34]
         i40 = int_ge(i39, i19)
         guard_false(i40) []
-        i41 = getfield_gc(p16, descr=nextdescr)
+        i41 = getfield_gc_i(p16, descr=nextdescr)
         i42 = int_mul(i39, i41)
         i43 = int_add(i21, i42)
         i44 = int_add(i39, 1)
@@ -3444,17 +3448,17 @@
     def test_residual_call_does_not_invalidate_caches(self):
         ops = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
-        i2 = call(i1, descr=nonwritedescr)
-        i3 = getfield_gc(p1, descr=valuedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        i2 = call_i(i1, descr=nonwritedescr)
+        i3 = getfield_gc_i(p1, descr=valuedescr)
         escape_n(i1)
         escape_n(i3)
         jump(p1, p2)
         """
         expected = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=valuedescr)
-        i2 = call(i1, descr=nonwritedescr)
+        i1 = getfield_gc_i(p1, descr=valuedescr)
+        i2 = call_i(i1, descr=nonwritedescr)
         escape_n(i1)
         escape_n(i1)
         jump(p1, p2)
@@ -3464,11 +3468,11 @@
     def test_residual_call_invalidate_some_caches(self):
         ops = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=adescr)
-        i2 = getfield_gc(p1, descr=bdescr)
-        i3 = call(i1, descr=writeadescr)
-        i4 = getfield_gc(p1, descr=adescr)
-        i5 = getfield_gc(p1, descr=bdescr)
+        i1 = getfield_gc_i(p1, descr=adescr)
+        i2 = getfield_gc_i(p1, descr=bdescr)
+        i3 = call_i(i1, descr=writeadescr)
+        i4 = getfield_gc_i(p1, descr=adescr)
+        i5 = getfield_gc_i(p1, descr=bdescr)
         escape_n(i1)
         escape_n(i2)
         escape_n(i4)
@@ -3477,10 +3481,10 @@
         """
         expected = """
         [p1, p2]
-        i1 = getfield_gc(p1, descr=adescr)
-        i2 = getfield_gc(p1, descr=bdescr)
-        i3 = call(i1, descr=writeadescr)
-        i4 = getfield_gc(p1, descr=adescr)
+        i1 = getfield_gc_i(p1, descr=adescr)
+        i2 = getfield_gc_i(p1, descr=bdescr)
+        i3 = call_i(i1, descr=writeadescr)
+        i4 = getfield_gc_i(p1, descr=adescr)
         escape_n(i1)
         escape_n(i2)
         escape_n(i4)
@@ -3492,11 +3496,11 @@
     def test_residual_call_invalidate_arrays(self):
         ops = """
         [p1, p2, i1]
-        p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        i3 = call(i1, descr=writeadescr)
-        p5 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p6 = getarrayitem_gc(p2, 1, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
+        i3 = call_i(i1, descr=writeadescr)
+        p5 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p6 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
         escape_n(p3)
         escape_n(p4)
         escape_n(p5)
@@ -3505,9 +3509,9 @@
         """
         expected = """
         [p1, p2, i1]
-        p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
-        i3 = call(i1, descr=writeadescr)
+        p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
+        i3 = call_i(i1, descr=writeadescr)
         escape_n(p3)
         escape_n(p4)
         escape_n(p3)
@@ -3519,12 +3523,12 @@
     def test_residual_call_invalidate_some_arrays(self):
         ops = """
         [p1, p2, i1]
-        p3 = getarrayitem_gc(p2, 0, descr=arraydescr2)
-        p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p2, 0, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
         i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
-        i3 = call(i1, descr=writearraydescr)
-        p5 = getarrayitem_gc(p2, 0, descr=arraydescr2)
-        p6 = getarrayitem_gc(p2, 1, descr=arraydescr2)
+        i3 = call_i(i1, descr=writearraydescr)
+        p5 = getarrayitem_gc_r(p2, 0, descr=arraydescr2)
+        p6 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
         i4 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         escape_n(p3)
         escape_n(p4)
@@ -3536,10 +3540,10 @@
         """
         expected = """
         [p1, p2, i1]
-        p3 = getarrayitem_gc(p2, 0, descr=arraydescr2)
-        p4 = getarrayitem_gc(p2, 1, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p2, 0, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
         i2 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
-        i3 = call(i1, descr=writearraydescr)
+        i3 = call_i(i1, descr=writearraydescr)
         i4 = getarrayitem_gc_i(p1, 1, descr=arraydescr)
         escape_n(p3)
         escape_n(p4)
@@ -3556,7 +3560,7 @@
         [p1, i1, p2, i2]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, i2, descr=adescr)
-        i3 = call(i1, descr=readadescr)
+        i3 = call_i(i1, descr=readadescr)
         setfield_gc(p1, i3, descr=valuedescr)
         setfield_gc(p2, i3, descr=adescr)
         jump(p1, i1, p2, i2)
@@ -3564,7 +3568,7 @@
         expected = """
         [p1, i1, p2, i2]
         setfield_gc(p2, i2, descr=adescr)
-        i3 = call(i1, descr=readadescr)
+        i3 = call_i(i1, descr=readadescr)
         setfield_gc(p1, i3, descr=valuedescr)
         setfield_gc(p2, i3, descr=adescr)
         jump(p1, i1, p2, i2)
@@ -4682,11 +4686,11 @@
     def test_add_sub_ovf_virtual_unroll(self):
         ops = """
         [p15]
-        i886 = getfield_gc_pure(p15, descr=valuedescr)
+        i886 = getfield_gc_pure_i(p15, descr=valuedescr)
         i888 = int_sub_ovf(i886, 1)
         guard_no_overflow() []
         escape_n(i888)
-        i4360 = getfield_gc_pure(p15, descr=valuedescr)
+        i4360 = getfield_gc_pure_i(p15, descr=valuedescr)
         i4362 = int_add_ovf(i4360, 1)
         guard_no_overflow() []
         i4360p = int_sub_ovf(i4362, 1)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -432,6 +432,8 @@
                               operations +  \
                               [ResOperation(rop.LABEL, jump_args, descr=token)]
         start_state = self._do_optimize_loop(preamble, call_pure_results)
+        import pdb
+        pdb.set_trace()
 
         assert preamble.operations[-1].getopnum() == rop.LABEL
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -305,6 +305,8 @@
         # Construct jumpargs from the virtual state
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
+        import pdb
+        pdb.set_trace()
         try:
             jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         except BadVirtualState:
@@ -635,5 +637,7 @@
         debug_print("   short boxes")
         self.short_boxes.dump(logops)
         debug_print("   exported values")
-        #self.exported_values.dump(logops)
+        for k, v in self.exported_values.iteritems():
+            debug_print("%s:%s" % (logops.repr_of_resop(k),
+                                   logops.repr_of_resop(v.box)))
         debug_stop("jit-exported-state")
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -421,8 +421,8 @@
 class VRawBufferValue(AbstractVArrayValue):
     is_about_raw = True
 
-    def __init__(self, cpu, logops, size, keybox, source_op):
-        AbstractVirtualValue.__init__(self, keybox, source_op)
+    def __init__(self, cpu, logops, size, source_op):
+        AbstractVirtualValue.__init__(self, source_op)
         # note that size is unused, because we assume that the buffer is big
         # enough to write/read everything we need. If it's not, it's undefined
         # behavior anyway, although in theory we could probably detect such
@@ -467,7 +467,7 @@
             itemvalue = self.buffer.values[i]
             itembox = itemvalue.force_box(optforce)
             op = ResOperation(rop.RAW_STORE,
-                              [self.box, ConstInt(offset), itembox], None,
+                              [self.box, ConstInt(offset), itembox],
                               descr=descr)
             optforce.emit_operation(op)
 
@@ -484,8 +484,8 @@
 class VRawSliceValue(AbstractVirtualValue):
     is_about_raw = True
 
-    def __init__(self, rawbuffer_value, offset, keybox, source_op):
-        AbstractVirtualValue.__init__(self, keybox, source_op)
+    def __init__(self, rawbuffer_value, offset, source_op):
+        AbstractVirtualValue.__init__(self, source_op)
         self.rawbuffer_value = rawbuffer_value
         self.offset = offset
 
@@ -540,15 +540,15 @@
         self.make_equal_to(source_op, vvalue)
         return vvalue
 
-    def make_virtual_raw_memory(self, size, box, source_op):
+    def make_virtual_raw_memory(self, size, source_op):
         logops = self.optimizer.loop.logops
-        vvalue = VRawBufferValue(self.optimizer.cpu, logops, size, box, source_op)
-        self.make_equal_to(box, vvalue)
+        vvalue = VRawBufferValue(self.optimizer.cpu, logops, size, source_op)
+        self.make_equal_to(source_op, vvalue)
         return vvalue
 
-    def make_virtual_raw_slice(self, rawbuffer_value, offset, box, source_op):
-        vvalue = VRawSliceValue(rawbuffer_value, offset, box, source_op)
-        self.make_equal_to(box, vvalue)
+    def make_virtual_raw_slice(self, rawbuffer_value, offset, source_op):
+        vvalue = VRawSliceValue(rawbuffer_value, offset, source_op)
+        self.make_equal_to(source_op, vvalue)
         return vvalue
 
     def optimize_GUARD_NO_EXCEPTION(self, op):
@@ -736,6 +736,7 @@
         else:
             self.emit_operation(op)
     optimize_CALL_R = optimize_CALL_N
+    optimize_CALL_I = optimize_CALL_N
 
     def do_RAW_MALLOC_VARSIZE_CHAR(self, op):
         sizebox = self.get_constant_box(op.getarg(1))
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -644,6 +644,7 @@
         #    op = op.clone(self.memo)
         #    op.is_source_op = True
         if box in self.short_boxes:
+            return # XXX avoid those corner cases
             #if op is None:
             #    xxx
             #    oldop = self.short_boxes[box]
@@ -659,15 +660,15 @@
             #value = self.optimizer.getvalue(box)
             #self.optimizer.emit_operation(ResOperation(rop.SAME_AS, [box], newbox))
             #self.optimizer.make_equal_to(newbox, value)
-            if op is None:
-                if self.short_boxes[box] is not box:
-                    xxx
-            else:
-                if self.short_boxes[box] is not op:
-                    if self.short_boxes[box] is None:
-                        self.short_boxes[box] = op
-                    else:
-                        xxx
+            #if op is None:
+            #    if self.short_boxes[box] is not box:
+            #        xxx
+            #else:
+            #    if self.short_boxes[box] is not op:
+            #        if self.short_boxes[box] is None:
+            #            self.short_boxes[box] = op
+            #        else:
+            #            xxx
         else:
             self.short_boxes[box] = op
 

From noreply at buildbot.pypy.org  Wed Dec 17 18:33:26 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 17 Dec 2014 18:33:26 +0100 (CET)
Subject: [pypy-commit] pypy optresult: merge default
Message-ID: <20141217173326.7D92D1D367E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74991:8e03e2425205
Date: 2014-12-17 19:32 +0200
http://bitbucket.org/pypy/pypy/changeset/8e03e2425205/

Log:	merge default

diff too long, truncating to 2000 out of 2146 lines

diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -1108,6 +1108,16 @@
             od.popitem()
         self.assertEqual(len(od), 0)
 
+    def test_popitem_first(self):
+        pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
+        shuffle(pairs)
+        od = OrderedDict(pairs)
+        while pairs:
+            self.assertEqual(od.popitem(last=False), pairs.pop(0))
+        with self.assertRaises(KeyError):
+            od.popitem(last=False)
+        self.assertEqual(len(od), 0)
+
     def test_pop(self):
         pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
         shuffle(pairs)
@@ -1179,7 +1189,11 @@
         od = OrderedDict(pairs)
         # yaml.dump(od) -->
         # '!!python/object/apply:__main__.OrderedDict\n- - [a, 1]\n  - [b, 2]\n'
-        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1]))
+
+        # PyPy bug fix: added [0] at the end of this line, because the
+        # test is really about the 2-tuples that need to be 2-lists
+        # inside the list of 6 of them
+        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1][0]))
 
     def test_reduce_not_too_fat(self):
         # do not save instance dictionary if not needed
@@ -1189,6 +1203,16 @@
         od.x = 10
         self.assertEqual(len(od.__reduce__()), 3)
 
+    def test_reduce_exact_output(self):
+        # PyPy: test that __reduce__() produces the exact same answer as
+        # CPython does, even though in the 'all_ordered_dicts' branch we
+        # have to emulate it.
+        pairs = [['c', 1], ['b', 2], ['d', 4]]
+        od = OrderedDict(pairs)
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,)))
+        od.x = 10
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,), {'x': 10}))
+
     def test_repr(self):
         od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)])
         self.assertEqual(repr(od),
diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt
--- a/lib-python/stdlib-upgrade.txt
+++ b/lib-python/stdlib-upgrade.txt
@@ -7,7 +7,7 @@
 
 1. check out the branch vendor/stdlib
 2. upgrade the files there
-3. update stdlib-versions.txt with the output of hg -id from the cpython repo
+3. update stdlib-version.txt with the output of hg -id from the cpython repo
 4. commit
 5. update to default/py3k
 6. create a integration branch for the new stdlib
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1175,11 +1175,9 @@
         try:
             return self.__description
         except AttributeError:
-            try:
+            if self.__statement:
                 self.__description = self.__statement._get_description()
                 return self.__description
-            except AttributeError:
-                return None
     description = property(__get_description)
 
     def __get_lastrowid(self):
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -3,6 +3,7 @@
 from __future__ import absolute_import
 import py
 
+
 class BaseTestDatetime:
     def test_repr(self):
         print datetime
@@ -210,11 +211,13 @@
             naive == aware
         assert str(e.value) == "can't compare offset-naive and offset-aware times"
 
-class TestDatetimeCPython(BaseTestDatetime):
+
+class TestDatetimeHost(BaseTestDatetime):
     def setup_class(cls):
         global datetime
         import datetime
 
+
 class TestDatetimePyPy(BaseTestDatetime):
     def setup_class(cls):
         global datetime
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -1,267 +1,285 @@
 # -*- coding: utf-8 -*-
 """Tests for _sqlite3.py"""
 
-import pytest, sys
+from __future__ import absolute_import
+import pytest
+import sys
 
-if sys.version_info < (2, 7):
-    pytest.skip("_sqlite3 requires Python 2.7")
-try:
-    import _cffi_backend
-except ImportError:
-    # On CPython, "pip install cffi".  On old PyPy's, no chance
-    pytest.skip("_sqlite3 requires _cffi_backend to be installed")
-
-from lib_pypy import _sqlite3
 
 def pytest_funcarg__con(request):
     con = _sqlite3.connect(':memory:')
     request.addfinalizer(lambda: con.close())
     return con
 
-def test_list_ddl(con):
-    """From issue996.  Mostly just looking for lack of exceptions."""
-    cursor = con.cursor()
-    cursor.execute('CREATE TABLE foo (bar INTEGER)')
-    result = list(cursor)
-    assert result == []
-    cursor.execute('INSERT INTO foo (bar) VALUES (42)')
-    result = list(cursor)
-    assert result == []
-    cursor.execute('SELECT * FROM foo')
-    result = list(cursor)
-    assert result == [(42,)]
 
-def test_connect_takes_same_positional_args_as_Connection(con):
-    from inspect import getargspec
-    clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
-    conargs = getargspec(_sqlite3.connect).args
-    assert clsargs == conargs
+class BaseTestSQLite:
+    def test_list_ddl(self, con):
+        """From issue996.  Mostly just looking for lack of exceptions."""
+        cursor = con.cursor()
+        cursor.execute('CREATE TABLE foo (bar INTEGER)')
+        result = list(cursor)
+        assert result == []
+        cursor.execute('INSERT INTO foo (bar) VALUES (42)')
+        result = list(cursor)
+        assert result == []
+        cursor.execute('SELECT * FROM foo')
+        result = list(cursor)
+        assert result == [(42,)]
 
-def test_total_changes_after_close(con):
-    con.close()
-    pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
+    def test_connect_takes_same_positional_args_as_Connection(self, con):
+        if not hasattr(_sqlite3, '_ffi'):
+            pytest.skip("only works for lib_pypy _sqlite3")
+        from inspect import getargspec
+        clsargs = getargspec(_sqlite3.Connection.__init__).args[1:]  # ignore self
+        conargs = getargspec(_sqlite3.connect).args
+        assert clsargs == conargs
 
-def test_connection_check_init():
-    class Connection(_sqlite3.Connection):
-        def __init__(self, name):
+    def test_total_changes_after_close(self, con):
+        con.close()
+        pytest.raises(_sqlite3.ProgrammingError, "con.total_changes")
+
+    def test_connection_check_init(self):
+        class Connection(_sqlite3.Connection):
+            def __init__(self, name):
+                pass
+
+        con = Connection(":memory:")
+        e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()")
+        assert '__init__' in e.value.message
+
+    def test_cursor_check_init(self, con):
+        class Cursor(_sqlite3.Cursor):
+            def __init__(self, name):
+                pass
+
+        cur = Cursor(con)
+        e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')")
+        assert '__init__' in e.value.message
+
+    def test_connection_after_close(self, con):
+        pytest.raises(TypeError, "con()")
+        con.close()
+        # raises ProgrammingError because should check closed before check args
+        pytest.raises(_sqlite3.ProgrammingError, "con()")
+
+    def test_cursor_iter(self, con):
+        cur = con.cursor()
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        next(cur)
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        con.commit()
+        next(cur)
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        with pytest.raises(_sqlite3.ProgrammingError):
+            cur.executemany('select 1', [])
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        cur.execute('create table test(ing)')
+        with pytest.raises(StopIteration):
+            next(cur)
+
+        cur.execute('select 1')
+        cur.execute('insert into test values(1)')
+        con.commit()
+        with pytest.raises(StopIteration):
+            next(cur)
+
+    def test_cursor_after_close(self, con):
+        cur = con.execute('select 1')
+        cur.close()
+        con.close()
+        pytest.raises(_sqlite3.ProgrammingError, "cur.close()")
+        # raises ProgrammingError because should check closed before check args
+        pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)")
+        pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)")
+
+    @pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')")
+    def test_connection_del(self, tmpdir):
+        """For issue1325."""
+        import os
+        import gc
+        try:
+            import resource
+        except ImportError:
+            pytest.skip("needs resource module")
+
+        limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+        try:
+            fds = 0
+            while True:
+                fds += 1
+                resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
+                try:
+                    for p in os.pipe(): os.close(p)
+                except OSError:
+                    assert fds < 100
+                else:
+                    break
+
+            def open_many(cleanup):
+                con = []
+                for i in range(3):
+                    con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
+                    if cleanup:
+                        con[i] = None
+                        gc.collect(); gc.collect()
+
+            pytest.raises(_sqlite3.OperationalError, open_many, False)
+            gc.collect(); gc.collect()
+            open_many(True)
+        finally:
+            resource.setrlimit(resource.RLIMIT_NOFILE, limit)
+
+    def test_on_conflict_rollback_executemany(self, con):
+        major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+        if (int(major), int(minor), int(micro)) < (3, 2, 2):
+            pytest.skip("requires sqlite3 version >= 3.2.2")
+        con.execute("create table foo(x, unique(x) on conflict rollback)")
+        con.execute("insert into foo(x) values (1)")
+        try:
+            con.executemany("insert into foo(x) values (?)", [[1]])
+        except _sqlite3.DatabaseError:
             pass
+        con.execute("insert into foo(x) values (2)")
+        try:
+            con.commit()
+        except _sqlite3.OperationalError:
+            pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
 
-    con = Connection(":memory:")
-    e = pytest.raises(_sqlite3.ProgrammingError, "con.cursor()")
-    assert '__init__' in e.value.message
+    def test_statement_arg_checking(self, con):
+        with pytest.raises(_sqlite3.Warning) as e:
+            con(123)
+        assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.'
+        with pytest.raises(ValueError) as e:
+            con.execute(123)
+        assert str(e.value) == 'operation parameter must be str or unicode'
+        with pytest.raises(ValueError) as e:
+            con.executemany(123, 123)
+        assert str(e.value) == 'operation parameter must be str or unicode'
+        with pytest.raises(ValueError) as e:
+            con.executescript(123)
+        assert str(e.value) == 'script argument must be unicode or string.'
 
-def test_cursor_check_init(con):
-    class Cursor(_sqlite3.Cursor):
-        def __init__(self, name):
-            pass
+    def test_statement_param_checking(self, con):
+        con.execute('create table foo(x)')
+        con.execute('insert into foo(x) values (?)', [2])
+        con.execute('insert into foo(x) values (?)', (2,))
+        class seq(object):
+            def __len__(self):
+                return 1
+            def __getitem__(self, key):
+                return 2
+        con.execute('insert into foo(x) values (?)', seq())
+        del seq.__len__
+        with pytest.raises(_sqlite3.ProgrammingError):
+            con.execute('insert into foo(x) values (?)', seq())
+        with pytest.raises(_sqlite3.ProgrammingError):
+            con.execute('insert into foo(x) values (?)', {2:2})
+        with pytest.raises(ValueError) as e:
+            con.execute('insert into foo(x) values (?)', 2)
+        assert str(e.value) == 'parameters are of unsupported type'
 
-    cur = Cursor(con)
-    e = pytest.raises(_sqlite3.ProgrammingError, "cur.execute('select 1')")
-    assert '__init__' in e.value.message
+    def test_explicit_begin(self, con):
+        con.execute('BEGIN')
+        con.execute('BEGIN ')
+        con.execute('BEGIN')
+        con.commit()
+        con.execute('BEGIN')
+        con.commit()
 
-def test_connection_after_close(con):
-    pytest.raises(TypeError, "con()")
-    con.close()
-    # raises ProgrammingError because should check closed before check args
-    pytest.raises(_sqlite3.ProgrammingError, "con()")
+    def test_row_factory_use(self, con):
+        con.row_factory = 42
+        con.execute('select 1')
 
-def test_cursor_iter(con):
-    cur = con.cursor()
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_returning_blob_must_own_memory(self, con):
+        import gc
+        con.create_function("returnblob", 0, lambda: buffer("blob"))
+        cur = con.execute("select returnblob()")
+        val = cur.fetchone()[0]
+        for i in range(5):
+            gc.collect()
+            got = (val[0], val[1], val[2], val[3])
+            assert got == ('b', 'l', 'o', 'b')
+        # in theory 'val' should be a read-write buffer
+        # but it's not right now
+        if not hasattr(_sqlite3, '_ffi'):
+            val[1] = 'X'
+            got = (val[0], val[1], val[2], val[3])
+            assert got == ('b', 'X', 'o', 'b')
 
-    cur.execute('select 1')
-    next(cur)
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_description_after_fetchall(self, con):
+        cur = con.cursor()
+        assert cur.description is None
+        cur.execute("select 42").fetchall()
+        assert cur.description is not None
 
-    cur.execute('select 1')
-    con.commit()
-    next(cur)
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_executemany_lastrowid(self, con):
+        cur = con.cursor()
+        cur.execute("create table test(a)")
+        cur.executemany("insert into test values (?)", [[1], [2], [3]])
+        assert cur.lastrowid is None
 
-    with pytest.raises(_sqlite3.ProgrammingError):
-        cur.executemany('select 1', [])
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_authorizer_bad_value(self, con):
+        def authorizer_cb(action, arg1, arg2, dbname, source):
+            return 42
+        con.set_authorizer(authorizer_cb)
+        with pytest.raises(_sqlite3.OperationalError) as e:
+            con.execute('select 123')
+        major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
+        if (int(major), int(minor), int(micro)) >= (3, 6, 14):
+            assert str(e.value) == 'authorizer malfunction'
+        else:
+            assert str(e.value) == \
+                ("illegal return value (1) from the authorization function - "
+                 "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
 
-    cur.execute('select 1')
-    cur.execute('create table test(ing)')
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_issue1573(self, con):
+        cur = con.cursor()
+        cur.execute(u'SELECT 1 as m&#233;il')
+        assert cur.description[0][0] == u"m&#233;il".encode('utf-8')
 
-    cur.execute('select 1')
-    cur.execute('insert into test values(1)')
-    con.commit()
-    with pytest.raises(StopIteration):
-        next(cur)
+    def test_adapter_exception(self, con):
+        def cast(obj):
+            raise ZeroDivisionError
 
-def test_cursor_after_close(con):
-    cur = con.execute('select 1')
-    cur.close()
-    con.close()
-    pytest.raises(_sqlite3.ProgrammingError, "cur.close()")
-    # raises ProgrammingError because should check closed before check args
-    pytest.raises(_sqlite3.ProgrammingError, "cur.execute(1,2,3,4,5)")
-    pytest.raises(_sqlite3.ProgrammingError, "cur.executemany(1,2,3,4,5)")
+        _sqlite3.register_adapter(int, cast)
+        try:
+            cur = con.cursor()
+            cur.execute("select ?", (4,))
+            val = cur.fetchone()[0]
+            # Adapter error is ignored, and parameter is passed as is.
+            assert val == 4
+            assert type(val) is int
+        finally:
+            del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
 
- at pytest.mark.skipif("not hasattr(sys, 'pypy_translation_info')")
-def test_connection_del(tmpdir):
-    """For issue1325."""
-    import os
-    import gc
-    try:
-        import resource
-    except ImportError:
-        pytest.skip("needs resource module")
 
-    limit = resource.getrlimit(resource.RLIMIT_NOFILE)
-    try:
-        fds = 0
-        while True:
-            fds += 1
-            resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
-            try:
-                for p in os.pipe(): os.close(p)
-            except OSError:
-                assert fds < 100
-            else:
-                break
-        def open_many(cleanup):
-            con = []
-            for i in range(3):
-                con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
-                if cleanup:
-                    con[i] = None
-                    gc.collect(); gc.collect()
+class TestSQLiteHost(BaseTestSQLite):
+    def setup_class(cls):
+        global _sqlite3
+        import _sqlite3
 
-        pytest.raises(_sqlite3.OperationalError, open_many, False)
-        gc.collect(); gc.collect()
-        open_many(True)
-    finally:
-        resource.setrlimit(resource.RLIMIT_NOFILE, limit)
 
-def test_on_conflict_rollback_executemany(con):
-    major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
-    if (int(major), int(minor), int(micro)) < (3, 2, 2):
-        pytest.skip("requires sqlite3 version >= 3.2.2")
-    con.execute("create table foo(x, unique(x) on conflict rollback)")
-    con.execute("insert into foo(x) values (1)")
-    try:
-        con.executemany("insert into foo(x) values (?)", [[1]])
-    except _sqlite3.DatabaseError:
-        pass
-    con.execute("insert into foo(x) values (2)")
-    try:
-        con.commit()
-    except _sqlite3.OperationalError:
-        pytest.fail("_sqlite3 knew nothing about the implicit ROLLBACK")
+class TestSQLitePyPy(BaseTestSQLite):
+    def setup_class(cls):
+        if sys.version_info < (2, 7):
+            pytest.skip("_sqlite3 requires Python 2.7")
 
-def test_statement_arg_checking(con):
-    with pytest.raises(_sqlite3.Warning) as e:
-        con(123)
-    assert str(e.value) == 'SQL is of wrong type. Must be string or unicode.'
-    with pytest.raises(ValueError) as e:
-        con.execute(123)
-    assert str(e.value) == 'operation parameter must be str or unicode'
-    with pytest.raises(ValueError) as e:
-        con.executemany(123, 123)
-    assert str(e.value) == 'operation parameter must be str or unicode'
-    with pytest.raises(ValueError) as e:
-        con.executescript(123)
-    assert str(e.value) == 'script argument must be unicode or string.'
+        try:
+            import _cffi_backend
+        except ImportError:
+            # On CPython, "pip install cffi".  On old PyPy's, no chance
+            pytest.skip("_sqlite3 requires _cffi_backend to be installed")
 
-def test_statement_param_checking(con):
-    con.execute('create table foo(x)')
-    con.execute('insert into foo(x) values (?)', [2])
-    con.execute('insert into foo(x) values (?)', (2,))
-    class seq(object):
-        def __len__(self):
-            return 1
-        def __getitem__(self, key):
-            return 2
-    con.execute('insert into foo(x) values (?)', seq())
-    del seq.__len__
-    with pytest.raises(_sqlite3.ProgrammingError):
-        con.execute('insert into foo(x) values (?)', seq())
-    with pytest.raises(_sqlite3.ProgrammingError):
-        con.execute('insert into foo(x) values (?)', {2:2})
-    with pytest.raises(ValueError) as e:
-        con.execute('insert into foo(x) values (?)', 2)
-    assert str(e.value) == 'parameters are of unsupported type'
-
-def test_explicit_begin(con):
-    con.execute('BEGIN')
-    con.execute('BEGIN ')
-    con.execute('BEGIN')
-    con.commit()
-    con.execute('BEGIN')
-    con.commit()
-
-def test_row_factory_use(con):
-    con.row_factory = 42
-    con.execute('select 1')
-
-def test_returning_blob_must_own_memory(con):
-    import gc
-    con.create_function("returnblob", 0, lambda: buffer("blob"))
-    cur = con.execute("select returnblob()")
-    val = cur.fetchone()[0]
-    for i in range(5):
-        gc.collect()
-        got = (val[0], val[1], val[2], val[3])
-        assert got == ('b', 'l', 'o', 'b')
-    # in theory 'val' should be a read-write buffer
-    # but it's not right now
-    pytest.skip("in theory 'val' should be a read-write buffer")
-    val[1] = 'X'
-    got = (val[0], val[1], val[2], val[3])
-    assert got == ('b', 'X', 'o', 'b')
-
-def test_description_after_fetchall(con):
-    cur = con.cursor()
-    cur.execute("select 42").fetchall()
-    assert cur.description is not None
-
-def test_executemany_lastrowid(con):
-    cur = con.cursor()
-    cur.execute("create table test(a)")
-    cur.executemany("insert into test values (?)", [[1], [2], [3]])
-    assert cur.lastrowid is None
-
-
-def test_authorizer_bad_value(con):
-    def authorizer_cb(action, arg1, arg2, dbname, source):
-        return 42
-    con.set_authorizer(authorizer_cb)
-    with pytest.raises(_sqlite3.OperationalError) as e:
-        con.execute('select 123')
-    major, minor, micro = _sqlite3.sqlite_version.split('.')[:3]
-    if (int(major), int(minor), int(micro)) >= (3, 6, 14):
-        assert str(e.value) == 'authorizer malfunction'
-    else:
-        assert str(e.value) == \
-            ("illegal return value (1) from the authorization function - "
-             "should be SQLITE_OK, SQLITE_IGNORE, or SQLITE_DENY")
-
-
-def test_issue1573(con):
-    cur = con.cursor()
-    cur.execute(u'SELECT 1 as m&#233;il')
-    assert cur.description[0][0] == u"m&#233;il".encode('utf-8')
-
-def test_adapter_exception(con):
-    def cast(obj):
-        raise ZeroDivisionError
-
-    _sqlite3.register_adapter(int, cast)
-    try:
-        cur = con.cursor()
-        cur.execute("select ?", (4,))
-        val = cur.fetchone()[0]
-        # Adapter error is ignored, and parameter is passed as is.
-        assert val == 4
-        assert type(val) is int
-    finally:
-        del _sqlite3.adapters[(int, _sqlite3.PrepareProtocol)]
+        global _sqlite3
+        from lib_pypy import _sqlite3
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -142,7 +142,8 @@
     part.operations.append(ResOperation(rop.LABEL, jumpargs, descr=jitcell_token))
 
     try:
-        start_state = optimize_trace(metainterp_sd, part, enable_opts)
+        start_state = optimize_trace(metainterp_sd, part, enable_opts,
+                                     export_state=True)
     except InvalidLoop:
         return None
     target_token = part.operations[0].getdescr()
@@ -170,7 +171,7 @@
 
         try:
             optimize_trace(metainterp_sd, part, enable_opts,
-                           start_state=start_state)
+                           start_state=start_state, export_state=False)
         except InvalidLoop:
             return None
 
@@ -224,7 +225,7 @@
     assert label.getopnum() == rop.LABEL
     try:
         optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
-                       start_state=start_state)
+                       start_state=start_state, export_state=False)
     except InvalidLoop:
         # Fall back on jumping to preamble
         target_token = label.getdescr()
@@ -235,7 +236,8 @@
         try:
             optimize_trace(metainterp_sd, part,
                            jitdriver_sd.warmstate.enable_opts,
-                           inline_short_preamble=False, start_state=start_state)
+                           inline_short_preamble=False, start_state=start_state,
+                           export_state=False)
         except InvalidLoop:
             return None
     assert part.operations[-1].getopnum() != rop.LABEL
@@ -807,7 +809,7 @@
         inline_short_preamble = True
     try:
         state = optimize_trace(metainterp_sd, new_trace, state.enable_opts,
-                               inline_short_preamble)
+                               inline_short_preamble, export_state=True)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -48,7 +48,8 @@
     return optimizations, unroll
 
 def optimize_trace(metainterp_sd, loop, enable_opts,
-                   inline_short_preamble=True, start_state=None):
+                   inline_short_preamble=True, start_state=None,
+                   export_state=True):
     """Optimize loop.operations to remove internal overheadish operations.
     """
 
@@ -59,7 +60,8 @@
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
             return optimize_unroll(metainterp_sd, loop, optimizations,
-                                   inline_short_preamble, start_state)
+                                   inline_short_preamble, start_state,
+                                   export_state)
         else:
             optimizer = Optimizer(metainterp_sd, loop, optimizations)
             optimizer.propagate_all_forward()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -45,7 +45,8 @@
             part.operations = operations
 
             self.add_guard_future_condition(part)
-            state = self._do_optimize_loop(part, None, state)
+            state = self._do_optimize_loop(part, None, state,
+                                           export_state=True)
             if part.operations[-1].getopnum() == rop.LABEL:
                 last_label = [part.operations.pop()]
             else:
@@ -497,7 +498,8 @@
 
 class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel):
 
-    def _do_optimize_loop(self, loop, call_pure_results, state):
+    def _do_optimize_loop(self, loop, call_pure_results, state,
+                          export_state=False):
         from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
         from rpython.jit.metainterp.optimizeopt.util import args_dict
         from rpython.jit.metainterp.optimizeopt.pure import OptPure
@@ -505,7 +507,7 @@
         self.loop = loop
         loop.call_pure_results = args_dict()
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        return optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True, state)
+        return optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True, state, export_state)
 
     def test_optimizer_renaming_boxes1(self):
         ops = """
@@ -543,8 +545,8 @@
         self.optimize_loop(ops, expected)
 
 
-class TestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin):
+class XxxTestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin):
     pass
 
-class TestOptimizerRenamingBoxesLLtype(BaseTestOptimizerRenamingBoxes, LLtypeMixin):
+class XxxTestOptimizerRenamingBoxesLLtype(BaseTestOptimizerRenamingBoxes, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -105,7 +105,7 @@
         if loop.operations[-1].getopnum() == rop.JUMP:
             loop.operations[-1].setdescr(token)
         expected = convert_old_style_to_targets(self.parse(optops), jump=True)
-        self._do_optimize_loop(loop, call_pure_results)
+        self._do_optimize_loop(loop, call_pure_results, export_state=False)
         #print '\n'.join([str(o) for o in loop.operations])
         self.assert_equal(loop, expected)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -2414,16 +2414,16 @@
         guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(p1, i2, descr=valuedescr)
-        i7 = same_as_i(i2) # This same_as should be killed by backend
-        i6 = same_as_i(i4)
-        jump(p1, i1, i2, i4, i6)
-        """
-        expected = """
-        [p1, i1, i2, i4, i5]
+        #i7 = same_as(i2) # This same_as should be killed by backend
+        #i6 = same_as(i4)
+        jump(p1, i1, i2, i4)
+        """
+        expected = """
+        [p1, i1, i2, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         guard_true(i4) []
         setfield_gc(p1, i2, descr=valuedescr)
-        jump(p1, i1, i2, i5, i5)
+        jump(p1, i1, i2, 1)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -2446,15 +2446,15 @@
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         escape_n()
-        i5 = same_as(i4)
-        jump(p1, i2, i4, i5)
-        """
-        expected = """
-        [p1, i2, i4, i5]
+        #i5 = same_as(i4)
+        jump(p1, i2, i4)
+        """
+        expected = """
+        [p1, i2, i4]
         guard_true(i4) [p1]
         setfield_gc(p1, NULL, descr=nextdescr)
         escape_n()
-        jump(p1, i2, i5, i5)
+        jump(p1, i2, 1)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -2476,15 +2476,15 @@
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         escape_n()
-        i5 = same_as(i4)
-        jump(p1, i2, i4, i5)
-        """
-        expected = """
-        [p1, i2, i4, i5]
+        #i5 = same_as(i4)
+        jump(p1, i2, i4)
+        """
+        expected = """
+        [p1, i2, i4]
         guard_true(i4) [i2, p1]
         setfield_gc(p1, NULL, descr=nextdescr)
         escape_n()
-        jump(p1, i2, i5, i5)
+        jump(p1, i2, 1)
         """
         self.optimize_loop(ops, expected)
 
@@ -2507,17 +2507,17 @@
         guard_true(i5) []
         i4 = int_neg(i2)
         setfield_gc(p1, i2, descr=valuedescr)
-        i8 = same_as(i2) # This same_as should be killed by backend
-        i7 = same_as(i4)
-        jump(p1, i1, i2, i4, i7)
-        """
-        expected = """
-        [p1, i1, i2, i4, i7]
+        #i8 = same_as(i2) # This same_as should be killed by backend
+        #i7 = same_as(i4)
+        jump(p1, i1, i2, i4)
+        """
+        expected = """
+        [p1, i1, i2, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i5 = int_eq(i4, 5)
         guard_true(i5) []
         setfield_gc(p1, i2, descr=valuedescr)
-        jump(p1, i1, i2, i7, i7)
+        jump(p1, i1, i2, 5)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -2735,16 +2735,16 @@
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        i101 = same_as(i4)
-        jump(p1, i2, i4, p4, i101)
-        """
-        expected = """
-        [p1, i2, i4, p4, i5]
+        #i101 = same_as(i4)
+        jump(p1, i2, i4, p4)
+        """
+        expected = """
+        [p1, i2, i4, p4]
         guard_true(i4) [p1, p4]
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        jump(p1, i2, i5, p4, i5)
+        jump(p1, i2, 1, p4)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -3620,7 +3620,7 @@
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call_assembler(i1, descr=asmdescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        i143 = same_as(i3) # Should be killed by backend
+        #i143 = same_as(i3) # Should be killed by backend
         jump(p1, i4, i3)
         '''
         self.optimize_loop(ops, ops, preamble)
@@ -3645,18 +3645,18 @@
         jump(p1, i4, i3)
         '''
         expected = '''
-        [p1, i4, i3, i5]
-        setfield_gc(p1, i5, descr=valuedescr)
-        jump(p1, i3, i5, i5)
+        [p1, i4, i3]
+        setfield_gc(p1, i3, descr=valuedescr)
+        jump(p1, i3, i3)
         '''
         preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        i148 = same_as(i3)
-        i147 = same_as(i3)
-        jump(p1, i4, i3, i148)
+        #i148 = same_as(i3)
+        #i147 = same_as(i3)
+        jump(p1, i4, i3)
         '''
         self.optimize_loop(ops, expected, preamble)
 
@@ -3670,17 +3670,17 @@
         jump(p1, i4, i3)
         '''
         expected = '''
-        [p1, i4, i3, i5]
+        [p1, i4, i3]
         setfield_gc(p1, i4, descr=valuedescr)
-        jump(p1, i3, i5, i5)
+        jump(p1, i3, i3)
         '''
         preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i1, descr=valuedescr)
-        i151 = same_as(i3)
-        jump(p1, i4, i3, i151)
+        #i151 = same_as(i3)
+        jump(p1, i4, i3)
         '''
         self.optimize_loop(ops, expected, preamble)
 
@@ -3700,14 +3700,14 @@
         escape_n(i1)
         escape_n(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        i153 = same_as(i4)
-        jump(i0, i4, i153)
+        #i153 = same_as(i4)
+        jump(i0, i4)
         '''
         expected = '''
-        [i0, i4, i5]
+        [i0, i4]
         escape_n(42)
         escape_n(i4)
-        jump(i0, i5, i5)
+        jump(i0, i4)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
@@ -3731,14 +3731,14 @@
         escape_n(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
         guard_no_exception() []
-        i155 = same_as(i4)
-        jump(i0, i4, i155)
+        #i155 = same_as(i4)
+        jump(i0, i4)
         '''
         expected = '''
-        [i0, i2, i3]
+        [i0, i2]
         escape_n(42)
         escape_n(i2)
-        jump(i0, i3, i3)
+        jump(i0, i2)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
@@ -5864,11 +5864,12 @@
         [p0]
         p1 = getfield_gc(p0, descr=valuedescr)
         setfield_gc(p0, p0, descr=valuedescr)
-        p4450 = same_as(p0) # Should be killed by backend
+        #p4450 = same_as(p0) # Should be killed by backend
         jump(p0)
         """
         expected = """
         [p0]
+        p1 = getfield_gc(p0, descr=valuedescr)
         setfield_gc(p0, p0, descr=valuedescr)
         jump(p0)
         """
@@ -6264,13 +6265,13 @@
         [p1, i1, i2, i3]
         escape(i3)
         i4 = int_sub(i2, i1)
-        i5 = same_as(i4)
-        jump(p1, i1, i2, i4, i5)
-        """
-        expected = """
-        [p1, i1, i2, i3, i4]
+        #i5 = same_as(i4)
+        jump(p1, i1, i2, i4)
+        """
+        expected = """
+        [p1, i1, i2, i3]
         escape(i3)
-        jump(p1, i1, i2, i4, i4)
+        jump(p1, i1, i2, i3)
         """
         self.optimize_strunicode_loop(ops, expected, preamble)
 
@@ -6290,14 +6291,15 @@
         escape(i5)
         i4 = int_sub(i2, i1)
         setfield_gc(p2, i4, descr=valuedescr)
-        i8 = same_as(i4)
-        jump(p1, i1, i2, p2, i8, i4)
-        """
-        expected = """
-        [p1, i1, i2, p2, i5, i6]
-        escape(i5)
-        setfield_gc(p2, i6, descr=valuedescr)
-        jump(p1, i1, i2, p2, i6, i6)
+        #i8 = same_as(i4)
+        jump(p1, i1, i2, p2, i4)
+        """
+        expected = """
+        [p1, i1, i2, p2, i5]
+        i311 = getfield_gc(p2, descr=valuedescr)
+        escape(i311)
+        setfield_gc(p2, i5, descr=valuedescr)
+        jump(p1, i1, i2, p2, i5)
         """
         self.optimize_strunicode_loop(ops, expected, preamble)
 
@@ -7104,19 +7106,18 @@
         p188 = getarrayitem_gc(p187, 42, descr=<GcPtrArrayDescr>)
         guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
-        p26 = same_as(p25)
-        jump(p25, p187, i184, p26)
+        jump(p25, p187, i184)
         """
         short = """
         [p1, p187, i184]
         p188 = getarrayitem_gc(p187, 42, descr=<GcPtrArrayDescr>)
         guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
-        jump(p1, p187, i184, p25)
-        """
-        expected = """
-        [p25, p187, i184, p189]
-        jump(p189, p187, i184, p189)
+        jump(p25, p187, i184)
+        """
+        expected = """
+        [p25, p187, i184]
+        jump(p25, p187, i184)
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
@@ -7225,14 +7226,16 @@
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
         setfield_gc(p1, ii, descr=otherdescr)
-        jump(p0, p1, ii2, ii, ii, ii)
-        """
-        expected = """
-        [p0, p1, ii, ii2, i1, i2]
+        jump(p0, p1, ii2, ii)
+        """
+        expected = """
+        [p0, p1, ii, ii2]
+        i1 = getfield_gc(p0, descr=valuedescr)
+        i2 = getfield_gc(p1, descr=otherdescr)
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
         setfield_gc(p1, ii, descr=otherdescr)
-        jump(p0, p1, ii2, ii, ii, ii)
+        jump(p0, p1, ii2, ii)
         """
         self.optimize_loop(ops, expected)
 
@@ -7723,10 +7726,10 @@
         jump(p1)
         """
         expected = """
-        [p0, p1]
+        [p0]
         call(p0, descr=nonwritedescr)
-        call(p1, descr=writeadescr)
-        jump(p1, p1)
+        call(p0, descr=writeadescr)
+        jump(p0)
         """
         self.optimize_loop(ops, expected)
 
@@ -7837,10 +7840,11 @@
         jump(p5, p6)
         """
         expected = """
-        [p5, p6, i10, i11]
+        [p5, p6, i10]
+        i11 = getfield_gc(p6, descr=nextdescr)
         call(i10, i11, descr=nonwritedescr)
         setfield_gc(p6, i10, descr=nextdescr)
-        jump(p5, p6, i10, i10)
+        jump(p5, p6, i10)
         """
         self.optimize_loop(ops, expected)
 
@@ -7856,11 +7860,12 @@
         jump(p5, p6)
         """
         expected = """
-        [p5, p6, i14, i12, i10]
-        i13 = int_add(i14, 7)
+        [p5, p6, i12, i10]
+        i11 = getfield_gc(p6, descr=nextdescr)
+        i13 = int_add(i11, 7)
         call(i12, i13, descr=nonwritedescr)
         setfield_gc(p6, i10, descr=nextdescr)
-        jump(p5, p6, i10, i12, i10)
+        jump(p5, p6, i12, i10)
         """
         self.optimize_loop(ops, expected)
 
@@ -7980,9 +7985,9 @@
         jump(i1, i3)
         """
         expected = """
-        [i1, i2, i6, i3]
+        [i1, i2, i6]
         call(i6, descr=nonwritedescr)
-        jump(i1, i3, i6, i3)
+        jump(i1, i2, i6)
         """
         short = """
         [i1, i2]
@@ -7990,7 +7995,7 @@
         i4 = int_add(i3, i3)
         i5 = int_add(i4, i4)
         i6 = int_add(i5, i5)
-        jump(i1, i2, i6, i3)
+        jump(i1, i3, i6)
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
@@ -8145,19 +8150,18 @@
         call(i2, descr=nonwritedescr)
         setfield_gc(p22, i1, descr=valuedescr)
         guard_nonnull_class(p18, ConstClass(node_vtable)) []
-        i10 = same_as(i1)
-        jump(p22, p18, i1, i10)
+        jump(p22, p18, i1)
         """
         short = """
         [p22, p18, i1]
-        i2 = getfield_gc(p22, descr=valuedescr)
-        jump(p22, p18, i1, i2)
-        """
-        expected = """
-        [p22, p18, i1, i2]
-        call(i2, descr=nonwritedescr)
-        setfield_gc(p22, i1, descr=valuedescr)
-        jump(p22, p18, i1, i1)
+        jump(p22, p18, i1)
+        """
+        expected = """
+        [p22, p18, i2]
+        i1 = getfield_gc(p22, descr=valuedescr)
+        call(i1, descr=nonwritedescr)
+        setfield_gc(p22, i2, descr=valuedescr)
+        jump(p22, p18, i2)
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -390,7 +390,8 @@
         assert equaloplists(optimized.operations,
                             expected.operations, False, remap, text_right)
 
-    def _do_optimize_loop(self, loop, call_pure_results, start_state=None):
+    def _do_optimize_loop(self, loop, call_pure_results, start_state=None,
+                          export_state=False):
         from rpython.jit.metainterp.optimizeopt import optimize_trace
         from rpython.jit.metainterp.optimizeopt.util import args_dict
 
@@ -406,7 +407,8 @@
             metainterp_sd.callinfocollection = self.callinfocollection
         #
         return optimize_trace(metainterp_sd, loop, self.enable_opts,
-                              start_state=start_state)
+                              start_state=start_state,
+                              export_state=export_state)
 
     def unroll_and_optimize(self, loop, call_pure_results=None):
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
@@ -431,9 +433,8 @@
         preamble.operations = [ResOperation(rop.LABEL, inputargs, descr=TargetToken(token))] + \
                               operations +  \
                               [ResOperation(rop.LABEL, jump_args, descr=token)]
-        start_state = self._do_optimize_loop(preamble, call_pure_results)
-        import pdb
-        pdb.set_trace()
+        start_state = self._do_optimize_loop(preamble, call_pure_results,
+                                             export_state=True)
 
         assert preamble.operations[-1].getopnum() == rop.LABEL
 
@@ -447,7 +448,8 @@
         assert loop.operations[0].getopnum() == rop.LABEL
         loop.inputargs = loop.operations[0].getarglist()
 
-        self._do_optimize_loop(loop, call_pure_results, start_state)
+        self._do_optimize_loop(loop, call_pure_results, start_state,
+                               export_state=False)
         extra_same_as = []
         while loop.operations[0].getopnum() != rop.LABEL:
             extra_same_as.append(loop.operations[0])
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_disable_optimizations.py b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
rename from rpython/jit/metainterp/optimizeopt/test/test_disable_optimizations.py
rename to rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -17,10 +17,11 @@
 # FIXME: Introduce some VirtualOptimizer super class instead
 
 def optimize_unroll(metainterp_sd, loop, optimizations,
-                    inline_short_preamble=True, start_state=None):
+                    inline_short_preamble=True, start_state=None,
+                    export_state=True):
     opt = UnrollOptimizer(metainterp_sd, loop, optimizations)
     opt.inline_short_preamble = inline_short_preamble
-    return opt.propagate_all_forward(start_state)
+    return opt.propagate_all_forward(start_state, export_state)
 
 
 class UnrollableOptimizer(Optimizer):
@@ -71,7 +72,7 @@
         prev = self.fix_snapshot(jump_args, snapshot.prev)
         return Snapshot(prev, new_snapshot_args)
 
-    def propagate_all_forward(self, starting_state):
+    def propagate_all_forward(self, starting_state, export_state=True):
         loop = self.optimizer.loop
         self.optimizer.clear_newoperations()
 
@@ -146,11 +147,14 @@
             self.close_bridge(start_label)
 
         self.optimizer.flush()
-        KillHugeIntBounds(self.optimizer).apply()
+        if export_state:
+            KillHugeIntBounds(self.optimizer).apply()
 
         loop.operations = self.optimizer.get_newoperations()
-        final_state = self.export_state(stop_label)
-        final_state.dump(self.optimizer.metainterp_sd)
+        if export_state:
+            final_state = self.export_state(stop_label)
+        else:
+            final_state = None
         loop.operations.append(stop_label)
         return final_state
 
@@ -179,8 +183,7 @@
             for box in self.inputargs:
                 self.boxes_created_this_iteration[box] = None
 
-        short_boxes = ShortBoxes(self.optimizer, inputargs,
-                                 self.boxes_created_this_iteration)
+        short_boxes = ShortBoxes(self.optimizer, inputargs)
 
         self.optimizer.clear_newoperations()
         for i in range(len(original_jump_args)):
@@ -317,7 +320,9 @@
         jumpop.initarglist(jumpargs)
 
         # Inline the short preamble at the end of the loop
-        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values,
+                                                         self.optimizer,
+                                                         keyboxes=True)
         assert len(short_inputargs) == len(jmp_to_short_args)
         args = {}
         for i in range(len(short_inputargs)):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -587,12 +587,11 @@
 
 
 class ShortBoxes(object):
-    def __init__(self, optimizer, surviving_boxes, available_boxes=None):
+    def __init__(self, optimizer, surviving_boxes):
         self.potential_ops = {}
         self.alternatives = {}
         self.synthetic = {}
         self.optimizer = optimizer
-        self.available_boxes = available_boxes
         self.assumed_classes = {}
 
         assert surviving_boxes is not None
@@ -679,8 +678,6 @@
             return
         if op in self.short_boxes_in_production:
             raise BoxNotProducable
-        if self.available_boxes is not None and op not in self.available_boxes:
-            raise BoxNotProducable
         self.short_boxes_in_production[op] = None
 
         if op in self.potential_ops:
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -330,6 +330,20 @@
     keepalive_until_here(newp)
     return newp
 
+ at jit.dont_look_inside
+ at specialize.ll()
+def ll_arrayclear(p):
+    # Equivalent to memset(array, 0).  Only for GcArray(primitive-type) for now.
+    from rpython.rlib.objectmodel import keepalive_until_here
+
+    length = len(p)
+    ARRAY = lltype.typeOf(p).TO
+    offset = llmemory.itemoffsetof(ARRAY, 0)
+    dest_addr = llmemory.cast_ptr_to_adr(p) + offset
+    llmemory.raw_memclear(dest_addr, llmemory.sizeof(ARRAY.OF) * length)
+    keepalive_until_here(p)
+
+
 def no_release_gil(func):
     func._dont_inline_ = True
     func._no_release_gil_ = True
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -158,6 +158,16 @@
     assert a2[2].x == 3
     assert a2[2].y == 15
 
+def test_ll_arrayclear():
+    TYPE = lltype.GcArray(lltype.Signed)
+    a1 = lltype.malloc(TYPE, 10)
+    for i in range(10):
+        a1[i] = 100 + i
+    rgc.ll_arrayclear(a1)
+    assert len(a1) == 10
+    for i in range(10):
+        assert a1[i] == 0
+
 def test__contains_gcptr():
     assert not rgc._contains_gcptr(lltype.Signed)
     assert not rgc._contains_gcptr(
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -669,6 +669,7 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
+DictRepr.ll_newdict = staticmethod(ll_newdict)
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -692,26 +693,6 @@
     pass
 
 
-def rtype_r_dict(hop, i_force_non_null=None):
-    r_dict = hop.r_result
-    if not r_dict.custom_eq_hash:
-        raise TyperError("r_dict() call does not return an r_dict instance")
-    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
-    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
-    if i_force_non_null is not None:
-        assert i_force_non_null == 2
-        hop.inputarg(lltype.Void, arg=2)
-    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    hop.exception_cannot_occur()
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
-    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
-        hop.genop('setfield', [v_result, cname, v_eqfn])
-    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
-        hop.genop('setfield', [v_result, cname, v_hashfn])
-    return v_result
-
 # ____________________________________________________________
 #
 #  Iteration.
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -28,8 +28,8 @@
 #    }
 #
 #    struct dicttable {
-#        int num_items;
-#        int num_used_items;
+#        int num_live_items;
+#        int num_ever_used_items;
 #        int resize_counter;
 #        {byte, short, int, long} *indexes;
 #        dictentry *entries;
@@ -72,6 +72,8 @@
         'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr)
                              and DICTVALUE._needsgc()),
         }
+    if getattr(ll_eq_function, 'no_direct_compare', False):
+        entrymeths['no_direct_compare'] = True
 
     # * the key
     entryfields.append(("key", DICTKEY))
@@ -113,8 +115,8 @@
     DICTENTRY = lltype.Struct("odictentry", *entryfields)
     DICTENTRYARRAY = lltype.GcArray(DICTENTRY,
                                     adtmeths=entrymeths)
-    fields =          [ ("num_items", lltype.Signed),
-                        ("num_used_items", lltype.Signed),
+    fields =          [ ("num_live_items", lltype.Signed),
+                        ("num_ever_used_items", lltype.Signed),
                         ("resize_counter", lltype.Signed),
                         ("indexes", llmemory.GCREF),
                         ("lookup_function_no", lltype.Signed),
@@ -156,7 +158,7 @@
 
     def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue,
                  custom_eq_hash=None, force_non_null=False):
-        assert not force_non_null
+        #assert not force_non_null
         self.rtyper = rtyper
         self.finalized = False
         self.DICT = lltype.GcForwardReference()
@@ -217,7 +219,7 @@
         #dictobj = getattr(dictobj, '__self__', dictobj)
         if dictobj is None:
             return lltype.nullptr(self.DICT)
-        if not isinstance(dictobj, (dict, objectmodel.r_ordereddict)):
+        if not isinstance(dictobj, (dict, objectmodel.r_dict)):
             raise TypeError("expected a dict: %r" % (dictobj,))
         try:
             key = Constant(dictobj)
@@ -231,8 +233,8 @@
             if r_key.lowleveltype == llmemory.Address:
                 raise TypeError("No prebuilt dicts of address keys")
             r_value = self.value_repr
-            if isinstance(dictobj, objectmodel.r_ordereddict):
-                
+            if isinstance(dictobj, objectmodel.r_dict):
+
                 if self.r_rdict_eqfn.lowleveltype != lltype.Void:
                     l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq)
                     l_dict.fnkeyeq = l_fn
@@ -416,6 +418,7 @@
 TYPE_LONG  = lltype.Unsigned
 
 def ll_malloc_indexes_and_choose_lookup(d, n):
+    # keep in sync with ll_clear_indexes() below
     if n <= 256:
         d.indexes = lltype.cast_opaque_ptr(llmemory.GCREF,
                                            lltype.malloc(DICTINDEX_BYTE.TO, n,
@@ -437,6 +440,16 @@
                                                          zero=True))
         d.lookup_function_no = FUNC_LONG
 
+def ll_clear_indexes(d, n):
+    if n <= 256:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_BYTE, d.indexes))
+    elif n <= 65536:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_SHORT, d.indexes))
+    elif IS_64BIT and n <= 2 ** 32:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_INT, d.indexes))
+    else:
+        rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_LONG, d.indexes))
+
 def ll_call_insert_clean_function(d, hash, i):
     DICT = lltype.typeOf(d).TO
     if d.lookup_function_no == FUNC_BYTE:
@@ -492,11 +505,11 @@
     return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
-    return d.num_items
+    return d.num_live_items
 
 def ll_dict_bool(d):
     # check if a dict is True, allowing for None
-    return bool(d) and d.num_items != 0
+    return bool(d) and d.num_live_items != 0
 
 def ll_dict_getitem(d, key):
     index = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
@@ -519,43 +532,64 @@
         entry = d.entries[i]
         entry.value = value
     else:
-        if len(d.entries) == d.num_used_items:
-            if ll_dict_grow(d):
-                ll_call_insert_clean_function(d, hash, d.num_used_items)
-        entry = d.entries[d.num_used_items]
+        reindexed = False
+        if len(d.entries) == d.num_ever_used_items:
+            try:
+                reindexed = ll_dict_grow(d)
+            except:
+                _ll_dict_rescue(d)
+                raise
+        rc = d.resize_counter - 3
+        if rc <= 0:
+            try:
+                ll_dict_resize(d)
+                reindexed = True
+            except:
+                _ll_dict_rescue(d)
+                raise
+            rc = d.resize_counter - 3
+            ll_assert(rc > 0, "ll_dict_resize failed?")
+        if reindexed:
+            ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
+        #
+        d.resize_counter = rc
+        entry = d.entries[d.num_ever_used_items]
         entry.key = key
         entry.value = value
         if hasattr(ENTRY, 'f_hash'):
             entry.f_hash = hash
         if hasattr(ENTRY, 'f_valid'):
             entry.f_valid = True
-        d.num_used_items += 1
-        d.num_items += 1
-        rc = d.resize_counter - 3
-        if rc <= 0:
-            ll_dict_resize(d)
-            rc = d.resize_counter - 3
-            ll_assert(rc > 0, "ll_dict_resize failed?")
-        d.resize_counter = rc
+        d.num_ever_used_items += 1
+        d.num_live_items += 1
+
+ at jit.dont_look_inside
+def _ll_dict_rescue(d):
+    # MemoryError situation!  The 'indexes' contains an invalid entry
+    # at this point.  But we can call ll_dict_reindex() with the
+    # following arguments, ensuring no further malloc occurs.
+    ll_dict_reindex(d, _ll_len_of_d_indexes(d))
+_ll_dict_rescue._dont_inline_ = True
 
 def _ll_dict_insertclean(d, key, value, hash):
     ENTRY = lltype.typeOf(d.entries).TO.OF
-    ll_call_insert_clean_function(d, hash, d.num_used_items)
-    entry = d.entries[d.num_used_items]
+    ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
+    entry = d.entries[d.num_ever_used_items]
     entry.key = key
     entry.value = value
     if hasattr(ENTRY, 'f_hash'):
         entry.f_hash = hash
     if hasattr(ENTRY, 'f_valid'):
         entry.f_valid = True
-    d.num_used_items += 1
-    d.num_items += 1
+    d.num_ever_used_items += 1
+    d.num_live_items += 1
     rc = d.resize_counter - 3
     d.resize_counter = rc
 
 def _ll_len_of_d_indexes(d):
     # xxx Haaaack: returns len(d.indexes).  Works independently of
     # the exact type pointed to by d, using a forced cast...
+    # Must only be called by @jit.dont_look_inside functions.
     return len(rffi.cast(DICTINDEX_BYTE, d.indexes))
 
 def _overallocate_entries_len(baselen):
@@ -575,7 +609,11 @@
 
 @jit.dont_look_inside
 def ll_dict_grow(d):
-    if d.num_items < d.num_used_items // 4:
+    if d.num_live_items < d.num_ever_used_items // 2:
+        # At least 50% of the allocated entries are dead, so perform a
+        # compaction. If ll_dict_remove_deleted_items detects that over
+        # 75% of allocated entries are dead, then it will also shrink the
+        # memory allocated at the same time as doing a compaction.
         ll_dict_remove_deleted_items(d)
         return True
 
@@ -594,16 +632,25 @@
     return False
 
 def ll_dict_remove_deleted_items(d):
-    new_allocated = _overallocate_entries_len(d.num_items)
-    if new_allocated < len(d.entries) // 2:
+    if d.num_live_items < len(d.entries) // 4:
+        # At least 75% of the allocated entries are dead, so shrink the memory
+        # allocated as well as doing a compaction.
+        new_allocated = _overallocate_entries_len(d.num_live_items)
         newitems = lltype.malloc(lltype.typeOf(d).TO.entries.TO, new_allocated)
     else:
         newitems = d.entries
+        # The loop below does a lot of writes into 'newitems'.  It's a better
+        # idea to do a single gc_writebarrier rather than activating the
+        # card-by-card logic (worth 11% in microbenchmarks).
+        from rpython.rtyper.lltypesystem.lloperation import llop
+        llop.gc_writebarrier(lltype.Void, newitems)
     #
-    ENTRY = lltype.typeOf(d).TO.entries.TO.OF
+    ENTRIES = lltype.typeOf(d).TO.entries.TO
+    ENTRY = ENTRIES.OF
     isrc = 0
     idst = 0
-    while isrc < len(d.entries):
+    isrclimit = d.num_ever_used_items
+    while isrc < isrclimit:
         if d.entries.valid(isrc):
             src = d.entries[isrc]
             dst = newitems[idst]
@@ -616,9 +663,21 @@
                 dst.f_valid = True
             idst += 1
         isrc += 1
-    d.entries = newitems
-    assert d.num_items == idst
-    d.num_used_items = idst
+    assert d.num_live_items == idst
+    d.num_ever_used_items = idst
+    if ((ENTRIES.must_clear_key or ENTRIES.must_clear_value) and
+            d.entries == newitems):
+        # must clear the extra entries: they may contain valid pointers
+        # which would create a temporary memory leak
+        while idst < isrclimit:
+            entry = newitems[idst]
+            if ENTRIES.must_clear_key:
+                entry.key = lltype.nullptr(ENTRY.key.TO)
+            if ENTRIES.must_clear_value:
+                entry.value = lltype.nullptr(ENTRY.value.TO)
+            idst += 1
+    else:
+        d.entries = newitems
 
     ll_dict_reindex(d, _ll_len_of_d_indexes(d))
 
@@ -632,7 +691,7 @@
 @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
 def _ll_dict_del(d, index):
     d.entries.mark_deleted(index)
-    d.num_items -= 1
+    d.num_live_items -= 1
     # clear the key and the value if they are GC pointers
     ENTRIES = lltype.typeOf(d.entries).TO
     ENTRY = ENTRIES.OF
@@ -641,32 +700,37 @@
         entry.key = lltype.nullptr(ENTRY.key.TO)
     if ENTRIES.must_clear_value:
         entry.value = lltype.nullptr(ENTRY.value.TO)
-    #
-    # The rest is commented out: like CPython we no longer shrink the
-    # dictionary here.  It may shrink later if we try to append a number
-    # of new items to it.  Unsure if this behavior was designed in
-    # CPython or is accidental.  A design reason would be that if you
-    # delete all items in a dictionary (e.g. with a series of
-    # popitem()), then CPython avoids shrinking the table several times.
-    #num_entries = len(d.entries)
-    #if num_entries > DICT_INITSIZE and d.num_items <= num_entries / 4:
-    #    ll_dict_resize(d)
-    # A previous xxx: move the size checking and resize into a single
-    # call which is opaque to the JIT when the dict isn't virtual, to
-    # avoid extra branches.
+
+    if index == d.num_ever_used_items - 1:
+        # The last element of the ordereddict has been deleted. Instead of
+        # simply marking the item as dead, we can safely reuse it. Since it's
+        # also possible that there are more dead items immediately behind the
+        # last one, we reclaim all the dead items at the end of the ordereditem
+        # at the same point.
+        i = d.num_ever_used_items - 2
+        while i >= 0 and not d.entries.valid(i):
+            i -= 1
+        j = i + 1
+        assert j >= 0
+        d.num_ever_used_items = j
+
+    # If the dictionary is at least 87.5% dead items, then consider shrinking
+    # it.
+    if d.num_live_items + DICT_INITSIZE <= len(d.entries) / 8:
+        ll_dict_resize(d)
 
 def ll_dict_resize(d):
     # make a 'new_size' estimate and shrink it if there are many
     # deleted entry markers.  See CPython for why it is a good idea to
     # quadruple the dictionary size as long as it's not too big.
-    # (Quadrupling comes from '(d.num_items + d.num_items + 1) * 2'
-    # as long as num_items is not too large.)
-    num_extra = min(d.num_items + 1, 30000)
+    # (Quadrupling comes from '(d.num_live_items + d.num_live_items + 1) * 2'
+    # as long as num_live_items is not too large.)
+    num_extra = min(d.num_live_items + 1, 30000)
     _ll_dict_resize_to(d, num_extra)
 ll_dict_resize.oopspec = 'odict.resize(d)'
 
 def _ll_dict_resize_to(d, num_extra):
-    new_estimate = (d.num_items + num_extra) * 2
+    new_estimate = (d.num_live_items + num_extra) * 2
     new_size = DICT_INITSIZE
     while new_size <= new_estimate:
         new_size *= 2
@@ -677,13 +741,17 @@
         ll_dict_reindex(d, new_size)
 
 def ll_dict_reindex(d, new_size):
-    ll_malloc_indexes_and_choose_lookup(d, new_size)
-    d.resize_counter = new_size * 2 - d.num_items * 3
+    if bool(d.indexes) and _ll_len_of_d_indexes(d) == new_size:
+        ll_clear_indexes(d, new_size)   # hack: we can reuse the same array
+    else:
+        ll_malloc_indexes_and_choose_lookup(d, new_size)
+    d.resize_counter = new_size * 2 - d.num_live_items * 3
     assert d.resize_counter > 0
     #
     entries = d.entries
     i = 0
-    while i < d.num_used_items:
+    ibound = d.num_ever_used_items
+    while i < ibound:
         if entries.valid(i):
             hash = entries.hash(i)
             ll_call_insert_clean_function(d, hash, i)
@@ -758,7 +826,7 @@
     else:
         # pristine entry -- lookup failed
         if store_flag == FLAG_STORE:
-            indexes[i] = rffi.cast(T, d.num_used_items + VALID_OFFSET)
+            indexes[i] = rffi.cast(T, d.num_ever_used_items + VALID_OFFSET)
         elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
             return ll_kill_something(d, T)
         return -1
@@ -775,7 +843,7 @@
             if store_flag == FLAG_STORE:
                 if deletedslot == -1:
                     deletedslot = intmask(i)
-                indexes[deletedslot] = rffi.cast(T, d.num_used_items +
+                indexes[deletedslot] = rffi.cast(T, d.num_ever_used_items +
                                                  VALID_OFFSET)
             elif d.paranoia and store_flag == FLAG_DELETE_TRY_HARD:
                 return ll_kill_something(d, T)
@@ -835,10 +903,11 @@
     d = DICT.allocate()
     d.entries = _ll_empty_array(DICT)
     ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
-    d.num_items = 0
-    d.num_used_items = 0
+    d.num_live_items = 0
+    d.num_ever_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
+OrderedDictRepr.ll_newdict = staticmethod(ll_newdict)
 
 def ll_newdict_size(DICT, orig_length_estimate):
     length_estimate = (orig_length_estimate // 2) * 3
@@ -848,8 +917,8 @@
     d = DICT.allocate()
     d.entries = DICT.entries.TO.allocate(orig_length_estimate)
     ll_malloc_indexes_and_choose_lookup(d, n)
-    d.num_items = 0
-    d.num_used_items = 0
+    d.num_live_items = 0
+    d.num_ever_used_items = 0
     d.resize_counter = n * 2
     return d
 
@@ -864,23 +933,6 @@
     pass
 
 
-def rtype_r_dict(hop):
-    r_dict = hop.r_result
-    if not r_dict.custom_eq_hash:
-        raise TyperError("r_dict() call does not return an r_dict instance")
-    v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
-    v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
-    cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    hop.exception_cannot_occur()
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
-    if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyeq')
-        hop.genop('setfield', [v_result, cname, v_eqfn])
-    if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void:
-        cname = hop.inputconst(lltype.Void, 'fnkeyhash')
-        hop.genop('setfield', [v_result, cname, v_hashfn])
-    return v_result
-
 # ____________________________________________________________
 #
 #  Iteration.
@@ -916,7 +968,7 @@
         entries = dict.entries
         index = iter.index
         assert index >= 0
-        entries_len = dict.num_used_items
+        entries_len = dict.num_ever_used_items
         while index < entries_len:
             nextindex = index + 1
             if entries.valid(index):
@@ -951,15 +1003,15 @@
     newdict = DICT.allocate()
     newdict.entries = DICT.entries.TO.allocate(len(dict.entries))
 
-    newdict.num_items = dict.num_items
-    newdict.num_used_items = dict.num_used_items
+    newdict.num_live_items = dict.num_live_items
+    newdict.num_ever_used_items = dict.num_ever_used_items
     if hasattr(DICT, 'fnkeyeq'):
         newdict.fnkeyeq = dict.fnkeyeq
     if hasattr(DICT, 'fnkeyhash'):
         newdict.fnkeyhash = dict.fnkeyhash
 
     i = 0
-    while i < newdict.num_used_items:
+    while i < newdict.num_ever_used_items:
         d_entry = newdict.entries[i]
         entry = dict.entries[i]
         ENTRY = lltype.typeOf(newdict.entries).TO.OF
@@ -976,14 +1028,14 @@
 ll_dict_copy.oopspec = 'odict.copy(dict)'
 
 def ll_dict_clear(d):
-    if d.num_used_items == 0:
+    if d.num_ever_used_items == 0:
         return
     DICT = lltype.typeOf(d).TO
     old_entries = d.entries
     d.entries = _ll_empty_array(DICT)
     ll_malloc_indexes_and_choose_lookup(d, DICT_INITSIZE)
-    d.num_items = 0
-    d.num_used_items = 0
+    d.num_live_items = 0
+    d.num_ever_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     # old_entries.delete() XXX
 ll_dict_clear.oopspec = 'odict.clear(d)'
@@ -991,9 +1043,9 @@
 def ll_dict_update(dic1, dic2):
     if dic1 == dic2:
         return
-    ll_prepare_dict_update(dic1, dic2.num_items)
+    ll_prepare_dict_update(dic1, dic2.num_live_items)
     i = 0
-    while i < dic2.num_used_items:
+    while i < dic2.num_ever_used_items:
         entries = dic2.entries
         if entries.valid(i):
             entry = entries[i]
@@ -1012,7 +1064,13 @@
     #      (d.resize_counter - 1) // 3 = room left in d
     #  so, if num_extra == 1, we need d.resize_counter > 3
     #      if num_extra == 2, we need d.resize_counter > 6  etc.
-    jit.conditional_call(d.resize_counter <= num_extra * 3,
+    # Note however a further hack: if num_extra <= d.num_live_items,
+    # we avoid calling _ll_dict_resize_to here.  This is to handle
+    # the case where dict.update() actually has a lot of collisions.
+    # If num_extra is much greater than d.num_live_items the conditional_call
+    # will trigger anyway, which is really the goal.
+    x = num_extra - d.num_live_items
+    jit.conditional_call(d.resize_counter <= x * 3,
                          _ll_dict_resize_to, d, num_extra)
 
 # this is an implementation of keys(), values() and items()
@@ -1028,9 +1086,9 @@
 
 def _make_ll_keys_values_items(kind):
     def ll_kvi(LIST, dic):
-        res = LIST.ll_newlist(dic.num_items)
+        res = LIST.ll_newlist(dic.num_live_items)
         entries = dic.entries
-        dlen = dic.num_used_items
+        dlen = dic.num_ever_used_items
         items = res.ll_items()
         i = 0
         p = 0
@@ -1064,24 +1122,24 @@
     return i != -1
 
 def _ll_getnextitem(dic):
-    if dic.num_items == 0:
+    if dic.num_live_items == 0:
         raise KeyError
 
     entries = dic.entries
 
     while True:
-        i = dic.num_used_items - 1
+        i = dic.num_ever_used_items - 1
         if entries.valid(i):
             break
-        dic.num_used_items -= 1
+        dic.num_ever_used_items -= 1
 
     key = entries[i].key
     index = dic.lookup_function(dic, key, entries.hash(i),
                                 FLAG_DELETE_TRY_HARD)
     # if the lookup function returned me a random strange thing,
     # don't care about deleting the item
-    if index == dic.num_used_items - 1:
-        dic.num_used_items -= 1
+    if index == dic.num_ever_used_items - 1:
+        dic.num_ever_used_items -= 1
     else:
         assert index != -1
     return index
diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py
--- a/rpython/rtyper/rbuiltin.py
+++ b/rpython/rtyper/rbuiltin.py
@@ -724,15 +724,16 @@
     raise TyperError("hasattr is only suported on a constant")
 
 @typer_for(annmodel.SomeOrderedDict.knowntype)
+ at typer_for(objectmodel.r_dict)
 @typer_for(objectmodel.r_ordereddict)
-def rtype_ordered_dict(hop):
-    from rpython.rtyper.lltypesystem.rordereddict import ll_newdict
-
+def rtype_dict_constructor(hop, i_force_non_null=None):
+    # 'i_force_non_null' is ignored here; if it has any effect, it
+    # has already been applied to 'hop.r_result'
     hop.exception_cannot_occur()
     r_dict = hop.r_result
     cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
-    if hasattr(r_dict, 'r_dict_eqfn'):
+    v_result = hop.gendirectcall(r_dict.ll_newdict, cDICT)
+    if r_dict.custom_eq_hash:
         v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0)
         v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1)
         if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void:
@@ -743,9 +744,6 @@
             hop.genop('setfield', [v_result, cname, v_hashfn])
     return v_result
 
-from rpython.rtyper.lltypesystem.rdict import rtype_r_dict
-typer_for(objectmodel.r_dict)(rtype_r_dict)
-
 # _________________________________________________________________
 # weakrefs
 
diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py
--- a/rpython/rtyper/rdict.py
+++ b/rpython/rtyper/rdict.py
@@ -57,11 +57,10 @@
 
 
 def rtype_newdict(hop):
-    from rpython.rtyper.lltypesystem.rdict import ll_newdict
     hop.inputargs()    # no arguments expected
     r_dict = hop.r_result
     cDICT = hop.inputconst(lltype.Void, r_dict.DICT)
-    v_result = hop.gendirectcall(ll_newdict, cDICT)
+    v_result = hop.gendirectcall(r_dict.ll_newdict, cDICT)
     return v_result
 
 
diff --git a/rpython/rtyper/test/test_rordereddict.py b/rpython/rtyper/test/test_rordereddict.py
--- a/rpython/rtyper/test/test_rordereddict.py
+++ b/rpython/rtyper/test/test_rordereddict.py
@@ -71,7 +71,7 @@
             for j in range(i):
                 assert rordereddict.ll_dict_getitem(ll_d, llstr(str(j))) == j
             rordereddict.ll_dict_setitem(ll_d, llstr(str(i)), i)
-        assert ll_d.num_items == 20
+        assert ll_d.num_live_items == 20
         for i in range(20):
             assert rordereddict.ll_dict_getitem(ll_d, llstr(str(i))) == i
 
@@ -84,7 +84,7 @@
             rordereddict.ll_dict_setitem(ll_d, llstr(str(i)), i)
             if i % 2 != 0:
                 rordereddict.ll_dict_delitem(ll_d, llstr(str(i)))
-        assert ll_d.num_items == 10
+        assert ll_d.num_live_items == 10
         for i in range(0, 20, 2):
             assert rordereddict.ll_dict_getitem(ll_d, llstr(str(i))) == i
 
@@ -129,7 +129,7 @@
         for i in range(40):
             rordereddict.ll_dict_setitem(ll_d, lls, i)
             rordereddict.ll_dict_delitem(ll_d, lls)
-        assert ll_d.num_used_items <= 10
+        assert ll_d.num_ever_used_items <= 10
 
     def test_dict_iteration(self):
         DICT = self._get_str_dict()
@@ -190,7 +190,7 @@
         rordereddict.ll_dict_setitem(ll_d, llstr("j"), 1)
         rordereddict.ll_dict_setitem(ll_d, llstr("l"), 1)
         rordereddict.ll_dict_clear(ll_d)
-        assert ll_d.num_items == 0
+        assert ll_d.num_live_items == 0
 
     def test_get(self):
         DICT = self._get_str_dict()
@@ -251,6 +251,16 @@

From noreply at buildbot.pypy.org  Wed Dec 17 18:34:09 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 17 Dec 2014 18:34:09 +0100 (CET)
Subject: [pypy-commit] pypy optresult: kill pdb
Message-ID: <20141217173409.15DC91D3681@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74992:0edebff04429
Date: 2014-12-17 19:33 +0200
http://bitbucket.org/pypy/pypy/changeset/0edebff04429/

Log:	kill pdb

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -308,8 +308,6 @@
         # Construct jumpargs from the virtual state
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        import pdb
-        pdb.set_trace()
         try:
             jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         except BadVirtualState:

From noreply at buildbot.pypy.org  Wed Dec 17 18:47:53 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Dec 2014 18:47:53 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: Quick fix for ll_arraycopy()
 on an array of structs
Message-ID: <20141217174753.D87C81C01C3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r74993:901068cd72f6
Date: 2014-12-17 17:47 +0000
http://bitbucket.org/pypy/pypy/changeset/901068cd72f6/

Log:	Quick fix for ll_arraycopy() on an array of structs

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -483,11 +483,14 @@
         self.emit_operation(op)
 
     def _optimize_CALL_ARRAYCOPY(self, op):
+        length = self.get_constant_box(op.getarg(5))
+        if length and length.getint() == 0:
+            return True # 0-length arraycopy
+
         source_value = self.getvalue(op.getarg(1))
         dest_value = self.getvalue(op.getarg(2))
         source_start_box = self.get_constant_box(op.getarg(3))
         dest_start_box = self.get_constant_box(op.getarg(4))
-        length = self.get_constant_box(op.getarg(5))
         extrainfo = op.getdescr().get_extra_info()
         if (source_start_box and dest_start_box
             and length and (dest_value.is_virtual() or length.getint() <= 8) and
@@ -498,6 +501,8 @@
             dest_start = dest_start_box.getint()
             # XXX fish fish fish
             arraydescr = extrainfo.write_descrs_arrays[0]
+            if arraydescr.is_array_of_structs():
+                return False       # not supported right now
             for index in range(length.getint()):
                 if source_value.is_virtual():
                     assert isinstance(source_value, VArrayValue)
@@ -527,8 +532,6 @@
                                          descr=arraydescr)
                     self.emit_operation(newop)
             return True
-        if length and length.getint() == 0:
-            return True # 0-length arraycopy
         return False
 
     def optimize_CALL_PURE(self, op):

From noreply at buildbot.pypy.org  Wed Dec 17 18:57:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Dec 2014 18:57:52 +0100 (CET)
Subject: [pypy-commit] pypy default: Quick fix for ll_arraycopy() on an
	array of structs
Message-ID: <20141217175752.562B01C01C3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74994:341027e99d33
Date: 2014-12-17 17:47 +0000
http://bitbucket.org/pypy/pypy/changeset/341027e99d33/

Log:	Quick fix for ll_arraycopy() on an array of structs

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -483,11 +483,14 @@
         self.emit_operation(op)
 
     def _optimize_CALL_ARRAYCOPY(self, op):
+        length = self.get_constant_box(op.getarg(5))
+        if length and length.getint() == 0:
+            return True # 0-length arraycopy
+
         source_value = self.getvalue(op.getarg(1))
         dest_value = self.getvalue(op.getarg(2))
         source_start_box = self.get_constant_box(op.getarg(3))
         dest_start_box = self.get_constant_box(op.getarg(4))
-        length = self.get_constant_box(op.getarg(5))
         extrainfo = op.getdescr().get_extra_info()
         if (source_start_box and dest_start_box
             and length and (dest_value.is_virtual() or length.getint() <= 8) and
@@ -498,6 +501,8 @@
             dest_start = dest_start_box.getint()
             # XXX fish fish fish
             arraydescr = extrainfo.write_descrs_arrays[0]
+            if arraydescr.is_array_of_structs():
+                return False       # not supported right now
             for index in range(length.getint()):
                 if source_value.is_virtual():
                     assert isinstance(source_value, VArrayValue)
@@ -527,8 +532,6 @@
                                          descr=arraydescr)
                     self.emit_operation(newop)
             return True
-        if length and length.getint() == 0:
-            return True # 0-length arraycopy
         return False
 
     def optimize_CALL_PURE(self, op):

From noreply at buildbot.pypy.org  Wed Dec 17 18:57:53 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Dec 2014 18:57:53 +0100 (CET)
Subject: [pypy-commit] pypy default: Add tests for 341027e99d33.
Message-ID: <20141217175753.93C531C01C3@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r74995:41c42127ea0c
Date: 2014-12-17 17:57 +0000
http://bitbucket.org/pypy/pypy/changeset/41c42127ea0c/

Log:	Add tests for 341027e99d33.

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -999,6 +999,40 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct_arraycopy(self):
+        ops = """
+        [f0, f1]
+        p0 = new_array_clear(3, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=compleximagdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=complexrealdescr)
+        call(0, p0, p0, 0, 2, 1, descr=complexarraycopydescr)
+        f2 = getinteriorfield_gc(p0, 2, descr=complexrealdescr)
+        f3 = getinteriorfield_gc(p0, 2, descr=compleximagdescr)
+        escape(f2)
+        escape(f3)
+        finish(1)
+        """
+        expected = """
+        [f0, f1]
+        escape(f1)
+        escape(f0)
+        finish(1)
+        """
+        self.optimize_loop(ops, ops)
+        py.test.skip("XXX missing optimization: ll_arraycopy(array-of-structs)")
+
+    def test_nonvirtual_array_of_struct_arraycopy(self):
+        ops = """
+        [p0]
+        call(0, p0, p0, 0, 2, 1, descr=complexarraycopydescr)
+        f2 = getinteriorfield_gc(p0, 2, descr=compleximagdescr)
+        f3 = getinteriorfield_gc(p0, 2, descr=complexrealdescr)
+        escape(f2)
+        escape(f3)
+        finish(1)
+        """
+        self.optimize_loop(ops, ops)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -223,6 +223,10 @@
     complexarraydescr = cpu.arraydescrof(complexarray)
     complexrealdescr = cpu.interiorfielddescrof(complexarray, "real")
     compleximagdescr = cpu.interiorfielddescrof(complexarray, "imag")
+    complexarraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+            EffectInfo([], [complexarraydescr], [], [], [complexarraydescr], [],
+                       EffectInfo.EF_CANNOT_RAISE,
+                       oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     rawarraydescr = cpu.arraydescrof(lltype.Array(lltype.Signed,
                                                   hints={'nolength': True}))

From noreply at buildbot.pypy.org  Wed Dec 17 19:38:51 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Dec 2014 19:38:51 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: Add some more @jits
Message-ID: <20141217183851.D683F1C04B4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r74996:24a23a64f250
Date: 2014-12-17 19:37 +0100
http://bitbucket.org/pypy/pypy/changeset/24a23a64f250/

Log:	Add some more @jits

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -41,6 +41,7 @@
 #
 #
 
+ at jit.look_inside_iff(lambda d, key, hash, flag: jit.isvirtual(d))
 def ll_call_lookup_function(d, key, hash, flag):
     fun = d.lookup_function_no
     if fun == FUNC_BYTE:
@@ -449,6 +450,7 @@
     else:
         rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_LONG, d.indexes))
 
+ at jit.dont_look_inside
 def ll_call_insert_clean_function(d, hash, i):
     if d.lookup_function_no == FUNC_BYTE:
         ll_dict_store_clean(d, hash, i, TYPE_BYTE)
@@ -582,6 +584,7 @@
 _ll_dict_rescue._dont_inline_ = True
 
 def _ll_dict_insertclean(d, key, value, hash):
+    # never translated
     ENTRY = lltype.typeOf(d.entries).TO.OF
     ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
     entry = d.entries[d.num_ever_used_items]

From noreply at buildbot.pypy.org  Wed Dec 17 19:38:53 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 17 Dec 2014 19:38:53 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: merge heads
Message-ID: <20141217183853.1AF6E1C04B4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r74997:7ef08cb6ae5a
Date: 2014-12-17 18:38 +0000
http://bitbucket.org/pypy/pypy/changeset/7ef08cb6ae5a/

Log:	merge heads

diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -41,6 +41,7 @@
 #
 #
 
+ at jit.look_inside_iff(lambda d, key, hash, flag: jit.isvirtual(d))
 def ll_call_lookup_function(d, key, hash, flag):
     fun = d.lookup_function_no
     if fun == FUNC_BYTE:
@@ -449,6 +450,7 @@
     else:
         rgc.ll_arrayclear(lltype.cast_opaque_ptr(DICTINDEX_LONG, d.indexes))
 
+ at jit.dont_look_inside
 def ll_call_insert_clean_function(d, hash, i):
     if d.lookup_function_no == FUNC_BYTE:
         ll_dict_store_clean(d, hash, i, TYPE_BYTE)
@@ -582,6 +584,7 @@
 _ll_dict_rescue._dont_inline_ = True
 
 def _ll_dict_insertclean(d, key, value, hash):
+    # never translated
     ENTRY = lltype.typeOf(d.entries).TO.OF
     ll_call_insert_clean_function(d, hash, d.num_ever_used_items)
     entry = d.entries[d.num_ever_used_items]

From noreply at buildbot.pypy.org  Wed Dec 17 21:04:48 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 17 Dec 2014 21:04:48 +0100 (CET)
Subject: [pypy-commit] pypy optresult: fix the parsing stuff,
	some low hanging fruit
Message-ID: <20141217200448.3D6EC1D290A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r74998:6f3c6ecaa793
Date: 2014-12-17 22:04 +0200
http://bitbucket.org/pypy/pypy/changeset/6f3c6ecaa793/

Log:	fix the parsing stuff, some low hanging fruit

diff too long, truncating to 2000 out of 3207 lines

diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -83,9 +83,7 @@
         # replace CALL_PURE with just CALL
         args = op.getarglist()
         opnum = OpHelpers.call_for_descr(op.getdescr())
-        newop = ResOperation(opnum, args, op.getdescr())
-        newop.source_op = op
-        self.getvalue(op).box = newop
+        newop = self.optimizer.replace_op_with(op, opnum)
         self.emit_operation(newop)
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
     optimize_CALL_PURE_F = optimize_CALL_PURE_I
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -420,7 +420,8 @@
             if val.box.same_constant(CONST_0):
                 self.last_emitted_operation = REMOVED
                 return
-            op = op.copy_and_change(rop.CALL, args=op.getarglist()[1:])
+            opnum = OpHelpers.call_for_type(op)
+            op = op.copy_and_change(opnum, args=op.getarglist()[1:])
         self.emit_operation(op)
 
     def _optimize_nullness(self, op, box, expect_nonnull):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -781,7 +781,7 @@
         py.test.skip("implement me")
         ops = """
         []
-        i0 = escape()
+        i0 = escape_i()
         i7 = uint_lt(i0, i0)
         guard_false(i7) []
         i8 = uint_le(i0, i0)
@@ -794,7 +794,7 @@
         """
         expected = """
         []
-        i0 = escape()
+        i0 = escape_i()
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -2263,18 +2263,14 @@
         setfield_gc(p1, i2, descr=valuedescr)
         jump(p1, i1, i2)
         """
-        preamble = """
+        expected = """
         [p1, i1, i2]
         setfield_gc(p1, i2, descr=valuedescr)
         jump(p1, i1, i2)
         """
-        expected = """
-        [p1, i1, i2]
-        jump(p1, i1, i2)
-        """
         # in this case, all setfields are removed, because we can prove
         # that in the loop it will always have the same value
-        self.optimize_loop(ops, expected, preamble)
+        self.optimize_loop(ops, expected)
 
     def test_duplicate_setfield_1(self):
         ops = """
@@ -2614,7 +2610,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 0, p2, descr=arraydescr2)
-        p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         escape_n(p3)
         jump(p1, p3)
         """
@@ -2655,9 +2651,9 @@
         setarrayitem_gc(p1, i1, p2, descr=arraydescr2)
         setarrayitem_gc(p1, 0, p3, descr=arraydescr2)
         setarrayitem_gc(p1, 1, p4, descr=arraydescr2)
-        p5 = getarrayitem_gc(p1, i1, descr=arraydescr2)
-        p6 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p7 = getarrayitem_gc(p1, 1, descr=arraydescr2)
+        p5 = getarrayitem_gc_r(p1, i1, descr=arraydescr2)
+        p6 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p7 = getarrayitem_gc_r(p1, 1, descr=arraydescr2)
         escape_n(p5)
         escape_n(p6)
         escape_n(p7)
@@ -2668,7 +2664,7 @@
         setarrayitem_gc(p1, i1, p2, descr=arraydescr2)
         setarrayitem_gc(p1, 0, p3, descr=arraydescr2)
         setarrayitem_gc(p1, 1, p4, descr=arraydescr2)
-        p5 = getarrayitem_gc(p1, i1, descr=arraydescr2)
+        p5 = getarrayitem_gc_r(p1, i1, descr=arraydescr2)
         escape_n(p5)
         escape_n(p3)
         escape_n(p4)
@@ -2679,9 +2675,9 @@
     def test_getarrayitem_pure_does_not_invalidate(self):
         ops = """
         [p1, p2]
-        p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        i4 = getfield_gc_pure(ConstPtr(myptr), descr=valuedescr)
-        p5 = getarrayitem_gc(p1, 0, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        i4 = getfield_gc_pure_i(ConstPtr(myptr), descr=valuedescr)
+        p5 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         escape_n(p3)
         escape_n(i4)
         escape_n(p5)
@@ -2689,7 +2685,7 @@
         """
         expected = """
         [p1, p2]
-        p3 = getarrayitem_gc(p1, 0, descr=arraydescr2)
+        p3 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         escape_n(p3)
         escape_n(5)
         escape_n(p3)
@@ -2702,8 +2698,8 @@
         [p1, p2, p3, p4, i1]
         setarrayitem_gc(p1, 0, p3, descr=arraydescr2)
         setarrayitem_gc(p2, 1, p4, descr=arraydescr2)
-        p5 = getarrayitem_gc(p1, 0, descr=arraydescr2)
-        p6 = getarrayitem_gc(p2, 1, descr=arraydescr2)
+        p5 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
+        p6 = getarrayitem_gc_r(p2, 1, descr=arraydescr2)
         escape_n(p5)
         escape_n(p6)
         jump(p1, p2, p3, p4, i1)
@@ -2751,7 +2747,7 @@
     def test_bug_1(self):
         ops = """
         [i0, p1]
-        p4 = getfield_gc(p1, descr=nextdescr)
+        p4 = getfield_gc_r(p1, descr=nextdescr)
         guard_nonnull(p4) []
         escape_n(p4)
         #
@@ -2773,7 +2769,7 @@
     def test_bug_2(self):
         ops = """
         [i0, p1]
-        p4 = getarrayitem_gc(p1, 0, descr=arraydescr2)
+        p4 = getarrayitem_gc_r(p1, 0, descr=arraydescr2)
         guard_nonnull(p4) []
         escape_n(p4)
         #
@@ -3580,7 +3576,7 @@
         [p1, i1, p2, i2]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, i2, descr=adescr)
-        i3 = call(i1, descr=writeadescr)
+        i3 = call_i(i1, descr=writeadescr)
         setfield_gc(p1, i3, descr=valuedescr)
         setfield_gc(p2, i3, descr=adescr)
         jump(p1, i1, p2, i2)
@@ -3588,7 +3584,7 @@
         expected = """
         [p1, i1, p2, i2]
         setfield_gc(p2, i2, descr=adescr)
-        i3 = call(i1, descr=writeadescr)
+        i3 = call_i(i1, descr=writeadescr)
         setfield_gc(p1, i3, descr=valuedescr)
         setfield_gc(p2, i3, descr=adescr)
         jump(p1, i1, p2, i2)
@@ -3600,7 +3596,7 @@
         [p1, i1, p2, i2]
         setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p2, i2, descr=adescr)
-        i3 = call(i1, descr=plaincalldescr)
+        i3 = call_i(i1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
         setfield_gc(p2, i3, descr=adescr)
         jump(p1, i1, p2, i2)
@@ -3611,14 +3607,14 @@
         ops = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
-        i3 = call_assembler(i1, descr=asmdescr)
+        i3 = call_assembler_i(i1, descr=asmdescr)
         setfield_gc(p1, i3, descr=valuedescr)
         jump(p1, i4, i3)
         '''
         preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
-        i3 = call_assembler(i1, descr=asmdescr)
+        i3 = call_assembler_i(i1, descr=asmdescr)
         setfield_gc(p1, i3, descr=valuedescr)
         #i143 = same_as(i3) # Should be killed by backend
         jump(p1, i4, i3)
@@ -3629,7 +3625,7 @@
         ops = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
-        i3 = call_assembler(i1, descr=asmdescr)
+        i3 = call_assembler_i(i1, descr=asmdescr)
         setfield_gc(p1, i1, descr=valuedescr)
         jump(p1, i4, i3)
         '''
@@ -3640,7 +3636,7 @@
         ops = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
-        i3 = call_pure(p1, descr=plaincalldescr)
+        i3 = call_pure_i(p1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
         jump(p1, i4, i3)
         '''
@@ -3652,7 +3648,7 @@
         preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
-        i3 = call(p1, descr=plaincalldescr)
+        i3 = call_i(p1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
         #i148 = same_as(i3)
         #i147 = same_as(i3)
@@ -3665,7 +3661,7 @@
         ops = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
-        i3 = call_pure(p1, descr=plaincalldescr)
+        i3 = call_pure_i(p1, descr=plaincalldescr)
         setfield_gc(p1, i1, descr=valuedescr)
         jump(p1, i4, i3)
         '''
@@ -3677,7 +3673,7 @@
         preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
-        i3 = call(p1, descr=plaincalldescr)
+        i3 = call_i(p1, descr=plaincalldescr)
         setfield_gc(p1, i1, descr=valuedescr)
         #i151 = same_as(i3)
         jump(p1, i4, i3)
@@ -3691,15 +3687,15 @@
         [i0, i1, i2]
         escape_n(i1)
         escape_n(i2)
-        i3 = call_pure(123456, 4, 5, 6, descr=plaincalldescr)
-        i4 = call_pure(123456, 4, i0, 6, descr=plaincalldescr)
+        i3 = call_pure_i(123456, 4, 5, 6, descr=plaincalldescr)
+        i4 = call_pure_i(123456, 4, i0, 6, descr=plaincalldescr)
         jump(i0, i3, i4)
         '''
         preamble = '''
         [i0, i1, i2]
         escape_n(i1)
         escape_n(i2)
-        i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
+        i4 = call_i(123456, 4, i0, 6, descr=plaincalldescr)
         #i153 = same_as(i4)
         jump(i0, i4)
         '''
@@ -3719,9 +3715,9 @@
         [i0, i1, i2]
         escape_n(i1)
         escape_n(i2)
-        i3 = call_pure(123456, 4, 5, 6, descr=plaincalldescr)
+        i3 = call_pure_i(123456, 4, 5, 6, descr=plaincalldescr)
         guard_no_exception() []
-        i4 = call_pure(123456, 4, i0, 6, descr=plaincalldescr)
+        i4 = call_pure_i(123456, 4, i0, 6, descr=plaincalldescr)
         guard_no_exception() []
         jump(i0, i3, i4)
         '''
@@ -3729,7 +3725,7 @@
         [i0, i1, i2]
         escape_n(i1)
         escape_n(i2)
-        i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
+        i4 = call_i(123456, 4, i0, 6, descr=plaincalldescr)
         guard_no_exception() []
         #i155 = same_as(i4)
         jump(i0, i4)
@@ -3747,7 +3743,7 @@
         #      both in the preamble and in the peeled loop
         ops = '''
         [p1, i1, i2]
-        p2 = call_pure(0, p1, i1, i2, descr=strslicedescr)
+        p2 = call_pure_i(0, p1, i1, i2, descr=strslicedescr)
         escape_n(p2)
         jump(p1, i1, i2)
         '''
@@ -3819,7 +3815,7 @@
         #
         p2 = virtual_ref(p1, 3)
         setfield_gc(p0, p2, descr=nextdescr)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=nextdescr)
@@ -3834,7 +3830,7 @@
         setfield_gc(p2, p3, descr=virtualtokendescr)
         setfield_gc(p0, p2, descr=nextdescr)
         #
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
@@ -3859,7 +3855,7 @@
         #
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=nextdescr)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced(descr=fdescr) [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=nextdescr)
@@ -3874,7 +3870,7 @@
         setfield_gc(p2, p3, descr=virtualtokendescr)
         setfield_gc(p0, p2, descr=nextdescr)
         #
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced(descr=fdescr2) [p2, i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
@@ -3905,7 +3901,7 @@
         #
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=refdescr)
-        call(i1, descr=nonwritedescr)
+        call_n(i1, descr=nonwritedescr)
         guard_no_exception(descr=fdescr) [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=refdescr)
@@ -3915,7 +3911,7 @@
         preamble = """
         [p0, i1]
         p3 = force_token()
-        call(i1, descr=nonwritedescr)
+        call_n(i1, descr=nonwritedescr)
         guard_no_exception(descr=fdescr) [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         escape_n()
@@ -3924,7 +3920,7 @@
         expected = """
         [p0, i1]
         p3 = force_token()
-        call(i1, descr=nonwritedescr)
+        call_n(i1, descr=nonwritedescr)
         guard_no_exception(descr=fdescr2) [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         escape_n()
@@ -3950,7 +3946,7 @@
         p2 = virtual_ref(p1, 7)
         escape_n(p2)
         virtual_ref_finish(p2, p1)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() []
         jump(i1)
         """
@@ -3964,7 +3960,7 @@
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p2, NULL, descr=virtualtokendescr)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() []
         jump(i1)
         """
@@ -3976,7 +3972,7 @@
         p2 = virtual_ref(p1, 23)
         escape_n(p2)
         virtual_ref_finish(p2, p1)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
         jump(i1, p1)
         """
@@ -3989,7 +3985,7 @@
         escape_n(p2)
         setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p2, NULL, descr=virtualtokendescr)
-        call_may_force(i1, descr=mayforcevirtdescr)
+        call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
         jump(i1, p1)
         """
@@ -4004,7 +4000,7 @@
         setarrayitem_gc(p1, 1, 1, descr=arraydescr)
         p2 = new_array(3, descr=arraydescr)
         setarrayitem_gc(p2, 1, 3, descr=arraydescr)
-        call(0, p1, p2, 1, 1, 2, descr=arraycopydescr)
+        call_n(0, p1, p2, 1, 1, 2, descr=arraycopydescr)
         i2 = getarrayitem_gc_i(p2, 1, descr=arraydescr)
         jump(i2)
         '''
@@ -4021,7 +4017,7 @@
         p2 = new_array(3, descr=arraydescr)
         setarrayitem_gc(p1, 0, i0, descr=arraydescr)
         setarrayitem_gc(p2, 0, 3, descr=arraydescr)
-        call(0, p1, p2, 1, 1, 2, descr=arraycopydescr)
+        call_n(0, p1, p2, 1, 1, 2, descr=arraycopydescr)
         i2 = getarrayitem_gc_i(p2, 0, descr=arraydescr)
         jump(i2)
         '''
@@ -4038,7 +4034,7 @@
         p2 = new_array(3, descr=arraydescr)
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
         setarrayitem_gc(p2, 2, 13, descr=arraydescr)
-        call(0, p1, p2, 0, 0, 3, descr=arraycopydescr)
+        call_n(0, p1, p2, 0, 0, 3, descr=arraycopydescr)
         escape_n(p2)
         jump()
         '''
@@ -4057,7 +4053,7 @@
         ops = '''
         [p1]
         p0 = new_array(0, descr=arraydescr)
-        call(0, p0, p1, 0, 0, 0, descr=arraycopydescr)
+        call_n(0, p0, p1, 0, 0, 0, descr=arraycopydescr)
         jump(p1)
         '''
         expected = '''
@@ -4074,7 +4070,7 @@
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
         setarrayitem_gc(p2, 2, 13, descr=arraydescr)
         escape_n(p2)
-        call(0, p1, p2, 0, 0, 3, descr=arraycopydescr)
+        call_n(0, p1, p2, 0, 0, 3, descr=arraycopydescr)
         escape_n(p2)
         jump()
         '''
@@ -4099,7 +4095,7 @@
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
         setarrayitem_gc(p2, 2, 13, descr=arraydescr)
         escape_n(p2)
-        call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+        call_n(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         escape_n(p2)
         jump()
         '''
@@ -4110,7 +4106,7 @@
         escape_n(p2)
         p1 = new_array(10, descr=arraydescr)
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
-        call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+        call_n(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         escape_n(p2)
         jump()
         '''
@@ -4713,7 +4709,7 @@
     def test_framestackdepth_overhead(self):
         ops = """
         [p0, i22]
-        i1 = getfield_gc(p0, descr=valuedescr)
+        i1 = getfield_gc_i(p0, descr=valuedescr)
         i2 = int_gt(i1, i22)
         guard_false(i2) []
         i3 = int_add(i1, 1)
@@ -4738,8 +4734,8 @@
     def test_setgetfield_raw(self):
         ops = """
         [p4, p7, i30]
-        p16 = getfield_gc(p4, descr=valuedescr)
-        p17 = getarrayitem_gc(p4, 1, descr=arraydescr)
+        p16 = getfield_gc_r(p4, descr=valuedescr)
+        p17 = getarrayitem_gc_r(p4, 1, descr=arraydescr)
         guard_value(p16, ConstPtr(myptr), descr=<Guard3>) []
         i1 = getfield_raw(p7, descr=nextdescr)
         i2 = int_add(i1, i30)
@@ -4760,10 +4756,10 @@
     def test_setgetarrayitem_raw(self):
         ops = """
         [p4, p7, i30]
-        p16 = getfield_gc(p4, descr=valuedescr)
+        p16 = getfield_gc_r(p4, descr=valuedescr)
         guard_value(p16, ConstPtr(myptr), descr=<Guard3>) []
-        p17 = getarrayitem_gc(p4, 1, descr=arraydescr)
-        i1 = getarrayitem_raw(p7, 1, descr=arraydescr)
+        p17 = getarrayitem_gc_r(p4, 1, descr=arraydescr)
+        i1 = getarrayitem_raw_i(p7, 1, descr=arraydescr)
         i2 = int_add(i1, i30)
         setarrayitem_raw(p7, 1, 7, descr=arraydescr)
         setarrayitem_raw(p7, 1, i2, descr=arraydescr)
@@ -4771,7 +4767,7 @@
         """
         expected = """
         [p4, p7, i30]
-        i1 = getarrayitem_raw(p7, 1, descr=arraydescr)
+        i1 = getarrayitem_raw_i(p7, 1, descr=arraydescr)
         i2 = int_add(i1, i30)
         setarrayitem_raw(p7, 1, 7, descr=arraydescr)
         setarrayitem_raw(p7, 1, i2, descr=arraydescr)
@@ -4782,14 +4778,14 @@
     def test_pure(self):
         ops = """
         [p42]
-        p53 = getfield_gc(ConstPtr(myptr), descr=nextdescr)
-        p59 = getfield_gc_pure(p53, descr=valuedescr)
-        i61 = call(1, p59, descr=nonwritedescr)
+        p53 = getfield_gc_r(ConstPtr(myptr), descr=nextdescr)
+        p59 = getfield_gc_pure_r(p53, descr=valuedescr)
+        i61 = call_i(1, p59, descr=nonwritedescr)
         jump(p42)
         """
         expected = """
         [p42, p59]
-        i61 = call(1, p59, descr=nonwritedescr)
+        i61 = call_i(1, p59, descr=nonwritedescr)
         jump(p42, p59)
 
         """
@@ -4801,7 +4797,7 @@
         ops = """
         [p3]
         p1 = escape_r()
-        p2 = getfield_gc_pure(p1, descr=nextdescr)
+        p2 = getfield_gc_pure_r(p1, descr=nextdescr)
         setfield_gc(p1, p3, descr=nextdescr)
         jump(p3)
         """
@@ -4811,7 +4807,7 @@
         ops = """
         [p3]
         p1 = escape_r()
-        p2 = getfield_gc_pure(p1, descr=nextdescr)
+        p2 = getfield_gc_pure_r(p1, descr=nextdescr)
         setfield_gc(p1, p3, descr=otherdescr)
         escape_n(p2)
         jump(p3)
@@ -4819,7 +4815,7 @@
         expected = """
         [p3]
         p1 = escape_r()
-        p2 = getfield_gc_pure(p1, descr=nextdescr)
+        p2 = getfield_gc_pure_r(p1, descr=nextdescr)
         setfield_gc(p1, p3, descr=otherdescr)
         escape_n(p2)
         jump(p3)
@@ -4830,7 +4826,7 @@
         ops = """
         []
         p1 = escape_r()
-        p2 = getfield_gc_pure(p1, descr=nextdescr)
+        p2 = getfield_gc_pure_r(p1, descr=nextdescr)
         p3 = escape_r()
         setfield_gc(p3, p1, descr=nextdescr)
         jump()
@@ -4840,34 +4836,34 @@
     def test_getfield_guard_const(self):
         ops = """
         [p0]
-        p20 = getfield_gc(p0, descr=nextdescr)
+        p20 = getfield_gc_r(p0, descr=nextdescr)
         guard_nonnull(p20) []
         guard_class(p20, ConstClass(node_vtable)) []
         guard_class(p20, ConstClass(node_vtable)) []
-        p23 = getfield_gc(p20, descr=valuedescr)
+        p23 = getfield_gc_r(p20, descr=valuedescr)
         guard_isnull(p23) []
         guard_class(p20, ConstClass(node_vtable)) []
         guard_value(p20, ConstPtr(myptr)) []
 
-        p37 = getfield_gc(p0, descr=nextdescr)
+        p37 = getfield_gc_r(p0, descr=nextdescr)
         guard_nonnull(p37) []
         guard_class(p37, ConstClass(node_vtable)) []
         guard_class(p37, ConstClass(node_vtable)) []
-        p40 = getfield_gc(p37, descr=valuedescr)
+        p40 = getfield_gc_r(p37, descr=valuedescr)
         guard_isnull(p40) []
         guard_class(p37, ConstClass(node_vtable)) []
         guard_value(p37, ConstPtr(myptr)) []
 
-        p64 = call_may_force(p23, p40, descr=plaincalldescr)
+        p64 = call_may_force_r(p23, p40, descr=plaincalldescr)
         jump(p0)
         """
         expected = """
         [p0]
-        p20 = getfield_gc(p0, descr=nextdescr)
+        p20 = getfield_gc_r(p0, descr=nextdescr)
         guard_value(p20, ConstPtr(myptr)) []
-        p23 = getfield_gc(p20, descr=valuedescr)
+        p23 = getfield_gc_r(p20, descr=valuedescr)
         guard_isnull(p23) []
-        p64 = call_may_force(NULL, NULL, descr=plaincalldescr)
+        p64 = call_may_force_r(NULL, NULL, descr=plaincalldescr)
         jump(p0)
         """
         self.optimize_loop(ops, expected, expected)
@@ -4875,34 +4871,34 @@
     def test_getfield_guard_const_preamble(self):
         ops = """
         [p0]
-        p01 = getfield_gc(p0, descr=nextdescr)
-        p02 = getfield_gc(p01, descr=valuedescr)
+        p01 = getfield_gc_r(p0, descr=nextdescr)
+        p02 = getfield_gc_r(p01, descr=valuedescr)
         guard_value(p01, ConstPtr(myptr)) []
-        p11 = getfield_gc(p0, descr=nextdescr)
-        p12 = getfield_gc(p11, descr=valuedescr)
+        p11 = getfield_gc_r(p0, descr=nextdescr)
+        p12 = getfield_gc_r(p11, descr=valuedescr)
         guard_value(p11, ConstPtr(myptr)) []
-        p64 = call_may_force(p02, p12, descr=plaincalldescr)
-
-        p21 = getfield_gc(p0, descr=nextdescr)
-        p22 = getfield_gc(p21, descr=valuedescr)
+        p64 = call_may_force_r(p02, p12, descr=plaincalldescr)
+
+        p21 = getfield_gc_r(p0, descr=nextdescr)
+        p22 = getfield_gc_r(p21, descr=valuedescr)
         guard_value(p21, ConstPtr(myptr)) []
-        p31 = getfield_gc(p0, descr=nextdescr)
-        p32 = getfield_gc(p31, descr=valuedescr)
+        p31 = getfield_gc_r(p0, descr=nextdescr)
+        p32 = getfield_gc_r(p31, descr=valuedescr)
         guard_value(p31, ConstPtr(myptr)) []
-        p65 = call_may_force(p22, p32, descr=plaincalldescr)
+        p65 = call_may_force_r(p22, p32, descr=plaincalldescr)
         jump(p0)
         """
         expected = """
         [p0]
-        p01 = getfield_gc(p0, descr=nextdescr)
-        p02 = getfield_gc(p01, descr=valuedescr)
+        p01 = getfield_gc_r(p0, descr=nextdescr)
+        p02 = getfield_gc_r(p01, descr=valuedescr)
         guard_value(p01, ConstPtr(myptr)) []
-        p64 = call_may_force(p02, p02, descr=plaincalldescr)
-
-        p21 = getfield_gc(p0, descr=nextdescr)
-        p22 = getfield_gc(p21, descr=valuedescr)
+        p64 = call_may_force_r(p02, p02, descr=plaincalldescr)
+
+        p21 = getfield_gc_r(p0, descr=nextdescr)
+        p22 = getfield_gc_r(p21, descr=valuedescr)
         guard_value(p21, ConstPtr(myptr)) []
-        p65 = call_may_force(p22, p22, descr=plaincalldescr)
+        p65 = call_may_force_r(p22, p22, descr=plaincalldescr)
         jump(p0)
         """
         self.optimize_loop(ops, expected, expected)
@@ -4946,7 +4942,7 @@
     def test_bound_and(self):
         ops = """
         []
-        i0 = escape()
+        i0 = escape_i()
         i1 = int_and(i0, 255)
         i2 = int_lt(i1, 500)
         guard_true(i2) []
@@ -4976,7 +4972,7 @@
         """
         expected = """
         []
-        i0 = escape()
+        i0 = escape_i()
         i1 = int_and(i0, 255)
         i12 = int_lt(i1, 100)
         guard_true(i12) []
@@ -5351,8 +5347,8 @@
     def test_bound_eq(self):
         ops = """
         []
-        i0 = escape()
-        i1 = escape()
+        i0 = escape_i()
+        i1 = escape_i()
         i2 = int_le(i0, 4)
         guard_true(i2) []
         i3 = int_eq(i0, i1)
@@ -5363,8 +5359,8 @@
         """
         expected = """
         []
-        i0 = escape()
-        i1 = escape()
+        i0 = escape_i()
+        i1 = escape_i()
         i2 = int_le(i0, 4)
         guard_true(i2) []
         i3 = int_eq(i0, i1)
@@ -5376,19 +5372,19 @@
     def test_bound_eq_const(self):
         ops = """
         []
-        i0 = escape()
+        i0 = escape_i()
         i1 = int_eq(i0, 7)
         guard_true(i1) []
         i2 = int_add(i0, 3)
-        escape(i2)
+        escape_n(i2)
         jump()
         """
         expected = """
         []
-        i0 = escape()
+        i0 = escape_i()
         i1 = int_eq(i0, 7)
         guard_true(i1) []
-        escape(10)
+        escape_n(10)
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -5615,7 +5611,7 @@
     def test_bound_backpropagate_int_signext(self):
         ops = """
         []
-        i0 = escape()
+        i0 = escape_i()
         i1 = int_signext(i0, 1)
         i2 = int_eq(i0, i1)
         guard_true(i2) []
@@ -5627,7 +5623,7 @@
         """
         expected = """
         []
-        i0 = escape()
+        i0 = escape_i()
         i1 = int_signext(i0, 1)
         i2 = int_eq(i0, i1)
         guard_true(i2) []
@@ -5638,7 +5634,7 @@
     def test_bound_backpropagate_int_signext_2(self):
         ops = """
         []
-        i0 = escape()
+        i0 = escape_i()
         i1 = int_signext(i0, 1)
         i2 = int_eq(i0, i1)
         guard_true(i2) []
@@ -5761,9 +5757,9 @@
         guard_no_overflow() []
         i4 = int_mul_ovf(i0, i1)
         guard_no_overflow() []
-        escape(i2)
-        escape(i3)
-        escape(i4)
+        escape_n(i2)
+        escape_n(i3)
+        escape_n(i4)
         i24 = int_mul_ovf(i10, i11)
         guard_no_overflow() []
         i23 = int_sub_ovf(i10, i11)
@@ -5774,9 +5770,9 @@
         """
         expected = """
         [i0, i1, i10, i11, i2, i3, i4]
-        escape(i2)
-        escape(i3)
-        escape(i4)
+        escape_n(i2)
+        escape_n(i3)
+        escape_n(i4)
         i24 = int_mul_ovf(i10, i11)
         guard_no_overflow() []
         i23 = int_sub_ovf(i10, i11)
@@ -5804,12 +5800,12 @@
         guard_value(i4, 3) []
         guard_class(p9, ConstClass(node_vtable)) []
         guard_class(p9, ConstClass(node_vtable)) []
-        p22 = getfield_gc(p9, descr=inst_w_seq)
+        p22 = getfield_gc_r(p9, descr=inst_w_seq)
         guard_nonnull(p22) []
-        i23 = getfield_gc(p9, descr=inst_index)
-        p24 = getfield_gc(p22, descr=inst_w_list)
+        i23 = getfield_gc_i(p9, descr=inst_index)
+        p24 = getfield_gc_r(p22, descr=inst_w_list)
         guard_isnull(p24) []
-        i25 = getfield_gc(p22, descr=inst_length)
+        i25 = getfield_gc_i(p22, descr=inst_length)
         i26 = int_ge(i23, i25)
         guard_true(i26) []
         setfield_gc(p9, ConstPtr(myptr), descr=inst_w_seq)
@@ -5823,16 +5819,16 @@
         setfield_gc(p75, 0, descr=inst_index)
         guard_class(p75, ConstClass(node_vtable)) []
         guard_class(p75, ConstClass(node_vtable)) []
-        p79 = getfield_gc(p75, descr=inst_w_seq)
+        p79 = getfield_gc_r(p75, descr=inst_w_seq)
         guard_nonnull(p79) []
-        i80 = getfield_gc(p75, descr=inst_index)
-        p81 = getfield_gc(p79, descr=inst_w_list)
+        i80 = getfield_gc_i(p75, descr=inst_index)
+        p81 = getfield_gc_r(p79, descr=inst_w_list)
         guard_isnull(p81) []
-        i82 = getfield_gc(p79, descr=inst_length)
+        i82 = getfield_gc_i(p79, descr=inst_length)
         i83 = int_ge(i80, i82)
         guard_false(i83) []
-        i84 = getfield_gc(p79, descr=inst_start)
-        i85 = getfield_gc(p79, descr=inst_step)
+        i84 = getfield_gc_i(p79, descr=inst_start)
+        i85 = getfield_gc_i(p79, descr=inst_step)
         i86 = int_mul(i80, i85)
         i87 = int_add(i84, i86)
         i91 = int_add(i80, 1)
@@ -5854,7 +5850,7 @@
     def test_let_getfield_kill_setfields(self):
         ops = """
         [p0]
-        p1 = getfield_gc(p0, descr=valuedescr)
+        p1 = getfield_gc_r(p0, descr=valuedescr)
         setfield_gc(p0, p1, descr=valuedescr)
         setfield_gc(p0, p1, descr=valuedescr)
         setfield_gc(p0, p0, descr=valuedescr)
@@ -5862,14 +5858,14 @@
         """
         preamble = """
         [p0]
-        p1 = getfield_gc(p0, descr=valuedescr)
+        p1 = getfield_gc_r(p0, descr=valuedescr)
         setfield_gc(p0, p0, descr=valuedescr)
         #p4450 = same_as(p0) # Should be killed by backend
         jump(p0)
         """
         expected = """
         [p0]
-        p1 = getfield_gc(p0, descr=valuedescr)
+        p1 = getfield_gc_r(p0, descr=valuedescr)
         setfield_gc(p0, p0, descr=valuedescr)
         jump(p0)
         """
@@ -5878,7 +5874,7 @@
     def test_let_getfield_kill_chained_setfields(self):
         ops = """
         [p0]
-        p1 = getfield_gc(p0, descr=valuedescr)
+        p1 = getfield_gc_r(p0, descr=valuedescr)
         setfield_gc(p0, p0, descr=valuedescr)
         setfield_gc(p0, p1, descr=valuedescr)
         setfield_gc(p0, p1, descr=valuedescr)
@@ -5886,7 +5882,7 @@
         """
         preamble = """
         [p0]
-        p1 = getfield_gc(p0, descr=valuedescr)
+        p1 = getfield_gc_r(p0, descr=valuedescr)
         jump(p0)
         """
         expected = """
@@ -5899,7 +5895,7 @@
         # FXIME: Can this occur?
         ops = """
         [p0, p1, pinv]
-        i1 = getfield_gc(pinv, descr=valuedescr)
+        i1 = getfield_gc_i(pinv, descr=valuedescr)
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, i1, descr=nextdescr)
         """
@@ -5910,7 +5906,7 @@
         []
         p0 = new_with_vtable(ConstClass(intobj_noimmut_vtable))
         setfield_gc(p0, 42, descr=noimmut_intval)
-        escape(p0)
+        escape_n(p0)
         jump()
         """
         self.optimize_loop(ops, ops)
@@ -5920,7 +5916,7 @@
         [i0]
         p0 = new_with_vtable(ConstClass(intobj_immut_vtable))
         setfield_gc(p0, i0, descr=immut_intval)
-        escape(p0)
+        escape_n(p0)
         jump(i0)
         """
         self.optimize_loop(ops, ops)
@@ -5929,7 +5925,7 @@
         ops = """
         []
         p0 = new_with_vtable(ConstClass(intobj_immut_vtable))
-        escape(p0)
+        escape_n(p0)
         jump()
         """
         self.optimize_loop(ops, ops)
@@ -5939,7 +5935,7 @@
         []
         p0 = new_with_vtable(ConstClass(intobj_immut_vtable))
         setfield_gc(p0, 1242, descr=immut_intval)
-        escape(p0)
+        escape_n(p0)
         jump()
         """
         from rpython.rtyper.lltypesystem import lltype, llmemory
@@ -5951,7 +5947,7 @@
                                                    IntObj1242())
         expected = """
         []
-        escape(ConstPtr(intobj1242))
+        escape_n(ConstPtr(intobj1242))
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -5960,7 +5956,7 @@
         [p1]
         p0 = new_with_vtable(ConstClass(ptrobj_immut_vtable))
         setfield_gc(p0, p1, descr=immut_ptrval)
-        escape(p0)
+        escape_n(p0)
         jump(p1)
         """
         self.optimize_loop(ops, ops)
@@ -5971,7 +5967,7 @@
         p1 = new_with_vtable(ConstClass(intobj_immut_vtable))
         setfield_gc(p1, 1242, descr=immut_intval)
         setfield_gc(p0, p1, descr=immut_ptrval)
-        escape(p0)
+        escape_n(p0)
         jump()
         """
         class PtrObj1242(object):
@@ -5986,7 +5982,7 @@
                                                    PtrObj1242())
         expected = """
         []
-        escape(ConstPtr(ptrobj1242))
+        escape_n(ConstPtr(ptrobj1242))
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -5994,14 +5990,14 @@
     def test_bug_unroll_with_immutables(self):
         ops = """
         [p0]
-        i2 = getfield_gc_pure(p0, descr=immut_intval)
+        i2 = getfield_gc_pure_i(p0, descr=immut_intval)
         p1 = new_with_vtable(ConstClass(intobj_immut_vtable))
         setfield_gc(p1, 1242, descr=immut_intval)
         jump(p1)
         """
         preamble = """
         [p0]
-        i2 = getfield_gc_pure(p0, descr=immut_intval)
+        i2 = getfield_gc_pure_i(p0, descr=immut_intval)
         jump()
         """
         expected = """
@@ -6015,7 +6011,7 @@
         []
         p0 = new_with_vtable(ConstClass(ptrobj_immut_vtable))
         setfield_gc(p0, p0, descr=immut_ptrval)
-        escape(p0)
+        escape_n(p0)
         jump()
         """
         from rpython.rtyper.lltypesystem import lltype, llmemory
@@ -6031,7 +6027,7 @@
                                                    PtrObjSelf())
         expected = """
         []
-        escape(ConstPtr(ptrobjself))
+        escape_n(ConstPtr(ptrobjself))
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -6042,7 +6038,7 @@
         p1 = new_with_vtable(ConstClass(ptrobj_immut_vtable))
         setfield_gc(p0, p1, descr=immut_ptrval)
         setfield_gc(p1, p0, descr=immut_ptrval)
-        escape(p0)
+        escape_n(p0)
         jump()
         """
         class PtrObjSelf2(object):
@@ -6060,7 +6056,7 @@
                                                     PtrObjSelf2())
         expected = """
         []
-        escape(ConstPtr(ptrobjself2))
+        escape_n(ConstPtr(ptrobjself2))
         jump()
         """
         self.optimize_loop(ops, expected)
@@ -6119,7 +6115,7 @@
     def test_str_concat_1(self):
         ops = """
         [p1, p2]
-        p3 = call(0, p1, p2, descr=strconcatdescr)
+        p3 = call_r(0, p1, p2, descr=strconcatdescr)
         jump(p2, p3)
         """
         preamble = """
@@ -6149,7 +6145,7 @@
         p1 = newstr(2)
         strsetitem(p1, 0, i0)
         strsetitem(p1, 1, i1)
-        p3 = call(0, p1, p2, descr=strconcatdescr)
+        p3 = call_r(0, p1, p2, descr=strconcatdescr)
         jump(i1, i0, p3)
         """
         expected = """
@@ -6170,7 +6166,7 @@
         p1 = newstr(2)
         strsetitem(p1, 0, i0)
         strsetitem(p1, 1, i1)
-        p3 = call(0, p2, p1, descr=strconcatdescr)
+        p3 = call_r(0, p2, p1, descr=strconcatdescr)
         jump(i1, i0, p3)
         """
         expected = """
@@ -6190,8 +6186,8 @@
     def test_str_concat_str_str_str(self):
         ops = """
         [p1, p2, p3]
-        p4 = call(0, p1, p2, descr=strconcatdescr)
-        p5 = call(0, p4, p3, descr=strconcatdescr)
+        p4 = call_r(0, p1, p2, descr=strconcatdescr)
+        p5 = call_r(0, p4, p3, descr=strconcatdescr)
         jump(p2, p3, p5)
         """
         preamble = """
@@ -6223,7 +6219,7 @@
     def test_str_concat_str_cstr1(self):
         ops = """
         [p2]
-        p3 = call(0, p2, s"x", descr=strconcatdescr)
+        p3 = call_r(0, p2, s"x", descr=strconcatdescr)
         jump(p3)
         """
         expected = """
@@ -6242,13 +6238,13 @@
         []
         p1 = same_as(s"ab")
         p2 = same_as(s"cde")
-        p3 = call(0, p1, p2, descr=strconcatdescr)
-        escape(p3)
+        p3 = call_r(0, p1, p2, descr=strconcatdescr)
+        escape_n(p3)
         jump()
         """
         expected = """
         []
-        escape(s"abcde")
+        escape_n(s"abcde")
         jump()
         """
         self.optimize_strunicode_loop(ops, expected, expected)
@@ -6256,21 +6252,21 @@
     def test_str_slice_len_surviving1(self):
         ops = """
         [p1, i1, i2, i3]
-        escape(i3)
-        p2 = call(0, p1, i1, i2, descr=strslicedescr)
+        escape_n(i3)
+        p2 = call_r(0, p1, i1, i2, descr=strslicedescr)
         i4 = strlen(p2)
         jump(p1, i1, i2, i4)
         """
         preamble = """
         [p1, i1, i2, i3]
-        escape(i3)
+        escape_n(i3)
         i4 = int_sub(i2, i1)
         #i5 = same_as(i4)
         jump(p1, i1, i2, i4)
         """
         expected = """
         [p1, i1, i2, i3]
-        escape(i3)
+        escape_n(i3)
         jump(p1, i1, i2, i3)
         """
         self.optimize_strunicode_loop(ops, expected, preamble)
@@ -6278,17 +6274,17 @@
     def test_str_slice_len_surviving2(self):
         ops = """
         [p1, i1, i2, p2]
-        i5 = getfield_gc(p2, descr=valuedescr)
-        escape(i5)
-        p3 = call(0, p1, i1, i2, descr=strslicedescr)
+        i5 = getfield_gc_i(p2, descr=valuedescr)
+        escape_n(i5)
+        p3 = call_r(0, p1, i1, i2, descr=strslicedescr)
         i4 = strlen(p3)
         setfield_gc(p2, i4, descr=valuedescr)
         jump(p1, i1, i2, p2)
         """
         preamble = """
         [p1, i1, i2, p2]
-        i5 = getfield_gc(p2, descr=valuedescr)
-        escape(i5)
+        i5 = getfield_gc_i(p2, descr=valuedescr)
+        escape_n(i5)
         i4 = int_sub(i2, i1)
         setfield_gc(p2, i4, descr=valuedescr)
         #i8 = same_as(i4)
@@ -6296,8 +6292,8 @@
         """
         expected = """
         [p1, i1, i2, p2, i5]
-        i311 = getfield_gc(p2, descr=valuedescr)
-        escape(i311)
+        i311 = getfield_gc_i(p2, descr=valuedescr)
+        escape_n(i311)
         setfield_gc(p2, i5, descr=valuedescr)
         jump(p1, i1, i2, p2, i5)
         """
@@ -6306,7 +6302,7 @@
     def test_str_slice_1(self):
         ops = """
         [p1, i1, i2]
-        p2 = call(0, p1, i1, i2, descr=strslicedescr)
+        p2 = call_r(0, p1, i1, i2, descr=strslicedescr)
         jump(p2, i1, i2)
         """
         preamble = """
@@ -6327,7 +6323,7 @@
     def test_str_slice_2(self):
         ops = """
         [p1, i2]
-        p2 = call(0, p1, 0, i2, descr=strslicedescr)
+        p2 = call_r(0, p1, 0, i2, descr=strslicedescr)
         jump(p2, i2)
         """
         expected = """
@@ -6341,8 +6337,8 @@
     def test_str_slice_3(self):
         ops = """
         [p1, i1, i2, i3, i4]
-        p2 = call(0, p1, i1, i2, descr=strslicedescr)
-        p3 = call(0, p2, i3, i4, descr=strslicedescr)
+        p2 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        p3 = call_r(0, p2, i3, i4, descr=strslicedescr)
         jump(p3, i1, i2, i3, i4)
         """
         preamble = """
@@ -6365,9 +6361,9 @@
     def test_str_slice_getitem1(self):
         ops = """
         [p1, i1, i2, i3]
-        p2 = call(0, p1, i1, i2, descr=strslicedescr)
+        p2 = call_r(0, p1, i1, i2, descr=strslicedescr)
         i4 = strgetitem(p2, i3)
-        escape(i4)
+        escape_n(i4)
         jump(p1, i1, i2, i3)
         """
         preamble = """
@@ -6375,12 +6371,12 @@
         i6 = int_sub(i2, i1)      # killed by the backend
         i5 = int_add(i1, i3)
         i4 = strgetitem(p1, i5)
-        escape(i4)
+        escape_n(i4)
         jump(p1, i1, i2, i3, i4)
         """
         expected = """
         [p1, i1, i2, i3, i4]
-        escape(i4)
+        escape_n(i4)
         jump(p1, i1, i2, i3, i4)
         """
         self.optimize_strunicode_loop(ops, expected, preamble)
@@ -6391,14 +6387,14 @@
         p1 = newstr(2)
         strsetitem(p1, 0, i3)
         strsetitem(p1, 1, i4)
-        p2 = call(0, p1, 1, 2, descr=strslicedescr)
+        p2 = call_r(0, p1, 1, 2, descr=strslicedescr)
         i5 = strgetitem(p2, 0)
-        escape(i5)
+        escape_n(i5)
         jump(i3, i4)
         """
         expected = """
         [i3, i4]
-        escape(i4)
+        escape_n(i4)
         jump(i3, i4)
         """
         self.optimize_strunicode_loop(ops, expected, expected)
@@ -6406,8 +6402,8 @@
     def test_str_slice_concat(self):
         ops = """
         [p1, i1, i2, p2]
-        p3 = call(0, p1, i1, i2, descr=strslicedescr)
-        p4 = call(0, p3, p2, descr=strconcatdescr)
+        p3 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        p4 = call_r(0, p3, p2, descr=strconcatdescr)
         jump(p4, i1, i2, p2)
         """
         preamble = """
@@ -6514,8 +6510,8 @@
     def test_str_equal_noop1(self):
         ops = """
         [p1, p2]
-        i0 = call(0, p1, p2, descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p1, p2, descr=strequaldescr)
+        escape_n(i0)
         jump(p1, p2)
         """
         self.optimize_strunicode_loop_extradescrs(ops, ops, ops)
@@ -6523,9 +6519,9 @@
     def test_str_equal_noop2(self):
         ops = """
         [p1, p2, p3]
-        p4 = call(0, p1, p2, descr=strconcatdescr)
-        i0 = call(0, p3, p4, descr=strequaldescr)
-        escape(i0)
+        p4 = call_r(0, p1, p2, descr=strconcatdescr)
+        i0 = call_i(0, p3, p4, descr=strequaldescr)
+        escape_n(i0)
         jump(p1, p2, p3)
         """
         preamble = """
@@ -6536,8 +6532,8 @@
         p4 = newstr(i3)
         copystrcontent(p1, p4, 0, 0, i1)
         copystrcontent(p2, p4, 0, i1, i2)
-        i0 = call(0, p3, p4, descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p3, p4, descr=strequaldescr)
+        escape_n(i0)
         jump(p1, p2, p3, i3, i1, i2)
         """
         expected = """
@@ -6545,8 +6541,8 @@
         p4 = newstr(i3)
         copystrcontent(p1, p4, 0, 0, i1)
         copystrcontent(p2, p4, 0, i1, i2)
-        i0 = call(0, p3, p4, descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p3, p4, descr=strequaldescr)
+        escape_n(i0)
         jump(p1, p2, p3, i3, i1, i2)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected,
@@ -6555,22 +6551,22 @@
     def test_str_equal_slice1(self):
         ops = """
         [p1, i1, i2, p3]
-        p4 = call(0, p1, i1, i2, descr=strslicedescr)
-        i0 = call(0, p4, p3, descr=strequaldescr)
-        escape(i0)
+        p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        i0 = call_i(0, p4, p3, descr=strequaldescr)
+        escape_n(i0)
         jump(p1, i1, i2, p3)
         """
         preamble = """
         [p1, i1, i2, p3]
         i3 = int_sub(i2, i1)
-        i0 = call(0, p1, i1, i3, p3, descr=streq_slice_checknull_descr)
-        escape(i0)
+        i0 = call_i(0, p1, i1, i3, p3, descr=streq_slice_checknull_descr)
+        escape_n(i0)
         jump(p1, i1, i2, p3, i3)
         """
         expected = """
         [p1, i1, i2, p3, i3]
-        i0 = call(0, p1, i1, i3, p3, descr=streq_slice_checknull_descr)
-        escape(i0)
+        i0 = call_i(0, p1, i1, i3, p3, descr=streq_slice_checknull_descr)
+        escape_n(i0)
         jump(p1, i1, i2, p3, i3)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected,
@@ -6579,22 +6575,22 @@
     def test_str_equal_slice2(self):
         ops = """
         [p1, i1, i2, p3]
-        p4 = call(0, p1, i1, i2, descr=strslicedescr)
-        i0 = call(0, p3, p4, descr=strequaldescr)
-        escape(i0)
+        p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        i0 = call_i(0, p3, p4, descr=strequaldescr)
+        escape_n(i0)
         jump(p1, i1, i2, p3)
         """
         preamble = """
         [p1, i1, i2, p3]
         i4 = int_sub(i2, i1)
-        i0 = call(0, p1, i1, i4, p3, descr=streq_slice_checknull_descr)
-        escape(i0)
+        i0 = call_i(0, p1, i1, i4, p3, descr=streq_slice_checknull_descr)
+        escape_n(i0)
         jump(p1, i1, i2, p3, i4)
         """
         expected = """
         [p1, i1, i2, p3, i4]
-        i0 = call(0, p1, i1, i4, p3, descr=streq_slice_checknull_descr)
-        escape(i0)
+        i0 = call_i(0, p1, i1, i4, p3, descr=streq_slice_checknull_descr)
+        escape_n(i0)
         jump(p1, i1, i2, p3, i4)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected,
@@ -6604,23 +6600,23 @@
         ops = """
         [p1, i1, i2, p3]
         guard_nonnull(p3) []
-        p4 = call(0, p1, i1, i2, descr=strslicedescr)
-        i0 = call(0, p3, p4, descr=strequaldescr)
-        escape(i0)
+        p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        i0 = call_i(0, p3, p4, descr=strequaldescr)
+        escape_n(i0)
         jump(p1, i1, i2, p3)
         """
         expected = """
         [p1, i1, i2, p3, i4]
-        i0 = call(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
-        escape(i0)
+        i0 = call_i(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
+        escape_n(i0)
         jump(p1, i1, i2, p3, i4)
         """
         preamble = """
         [p1, i1, i2, p3]
         guard_nonnull(p3) []
         i4 = int_sub(i2, i1)
-        i0 = call(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
-        escape(i0)
+        i0 = call_i(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
+        escape_n(i0)
         jump(p1, i1, i2, p3, i4)
         """
         self.optimize_strunicode_loop_extradescrs(ops,
@@ -6629,22 +6625,22 @@
     def test_str_equal_slice4(self):
         ops = """
         [p1, i1, i2]
-        p3 = call(0, p1, i1, i2, descr=strslicedescr)
-        i0 = call(0, p3, s"x", descr=strequaldescr)
-        escape(i0)
+        p3 = call_r(0, p1, i1, i2, descr=strslicedescr)
+        i0 = call_i(0, p3, s"x", descr=strequaldescr)
+        escape_n(i0)
         jump(p1, i1, i2)
         """
         preamble = """
         [p1, i1, i2]
         i3 = int_sub(i2, i1)
-        i0 = call(0, p1, i1, i3, 120, descr=streq_slice_char_descr)
-        escape(i0)
+        i0 = call_i(0, p1, i1, i3, 120, descr=streq_slice_char_descr)
+        escape_n(i0)
         jump(p1, i1, i2, i3)
         """
         expected = """
         [p1, i1, i2, i3]
-        i0 = call(0, p1, i1, i3, 120, descr=streq_slice_char_descr)
-        escape(i0)
+        i0 = call_i(0, p1, i1, i3, 120, descr=streq_slice_char_descr)
+        escape_n(i0)
         jump(p1, i1, i2, i3)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected,
@@ -6653,24 +6649,24 @@
     def test_str_equal_slice5(self):
         ops = """
         [p1, i1, i2, i3]
-        p4 = call(0, p1, i1, i2, descr=strslicedescr)
+        p4 = call_r(0, p1, i1, i2, descr=strslicedescr)
         p5 = newstr(1)
         strsetitem(p5, 0, i3)
-        i0 = call(0, p5, p4, descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p5, p4, descr=strequaldescr)
+        escape_n(i0)
         jump(p1, i1, i2, i3)
         """
         preamble = """
         [p1, i1, i2, i3]
         i4 = int_sub(i2, i1)
-        i0 = call(0, p1, i1, i4, i3, descr=streq_slice_char_descr)
-        escape(i0)
+        i0 = call_i(0, p1, i1, i4, i3, descr=streq_slice_char_descr)
+        escape_n(i0)
         jump(p1, i1, i2, i3, i4)
         """
         expected = """
         [p1, i1, i2, i3, i4]
-        i0 = call(0, p1, i1, i4, i3, descr=streq_slice_char_descr)
-        escape(i0)
+        i0 = call_i(0, p1, i1, i4, i3, descr=streq_slice_char_descr)
+        escape_n(i0)
         jump(p1, i1, i2, i3, i4)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected,
@@ -6679,19 +6675,19 @@
     def test_str_equal_none1(self):
         ops = """
         [p1]
-        i0 = call(0, p1, NULL, descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p1, NULL, descr=strequaldescr)
+        escape_n(i0)
         jump(p1)
         """
         preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i0)
         """
         expected = """
         [p1, i0]
-        escape(i0)
+        escape_n(i0)
         jump(p1, i0)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
@@ -6699,19 +6695,19 @@
     def test_str_equal_none2(self):
         ops = """
         [p1]
-        i0 = call(0, NULL, p1, descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, NULL, p1, descr=strequaldescr)
+        escape_n(i0)
         jump(p1)
         """
         preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i0)
         """
         expected = """
         [p1, i0]
-        escape(i0)
+        escape_n(i0)
         jump(p1, i0)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
@@ -6720,21 +6716,21 @@
         ops = """
         [p1]
         guard_nonnull(p1) []
-        i0 = call(0, p1, s"hello world", descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p1, s"hello world", descr=strequaldescr)
+        escape_n(i0)
         jump(p1)
         """
         expected = """
         [p1]
-        i0 = call(0, p1, s"hello world", descr=streq_nonnull_descr)
-        escape(i0)
+        i0 = call_i(0, p1, s"hello world", descr=streq_nonnull_descr)
+        escape_n(i0)
         jump(p1)
         """
         preamble = """
         [p1]
         guard_nonnull(p1) []
-        i0 = call(0, p1, s"hello world", descr=streq_nonnull_descr)
-        escape(i0)
+        i0 = call_i(0, p1, s"hello world", descr=streq_nonnull_descr)
+        escape_n(i0)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
@@ -6743,13 +6739,13 @@
         ops = """
         [p1]
         guard_nonnull(p1) []
-        i0 = call(0, p1, s"", descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p1, s"", descr=strequaldescr)
+        escape_n(i0)
         jump(p1)
         """
         expected = """
         [p1, i0]
-        escape(i0)
+        escape_n(i0)
         jump(p1, i0)
         """
         preamble = """
@@ -6757,7 +6753,7 @@
         guard_nonnull(p1) []
         i1 = strlen(p1)
         i0 = int_eq(i1, 0)
-        escape(i0)
+        escape_n(i0)
         jump(p1, i0)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
@@ -6766,21 +6762,21 @@
         ops = """
         [p1]
         guard_nonnull(p1) []
-        i0 = call(0, p1, s"x", descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p1, s"x", descr=strequaldescr)
+        escape_n(i0)
         jump(p1)
         """
         expected = """
         [p1]
-        i0 = call(0, p1, 120, descr=streq_nonnull_char_descr)
-        escape(i0)
+        i0 = call_i(0, p1, 120, descr=streq_nonnull_char_descr)
+        escape_n(i0)
         jump(p1)
         """
         preamble = """
         [p1]
         guard_nonnull(p1) []
-        i0 = call(0, p1, 120, descr=streq_nonnull_char_descr)
-        escape(i0)
+        i0 = call_i(0, p1, 120, descr=streq_nonnull_char_descr)
+        escape_n(i0)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
@@ -6788,9 +6784,9 @@
     def test_str_equal_nonnull4(self):
         ops = """
         [p1, p2]
-        p4 = call(0, p1, p2, descr=strconcatdescr)
-        i0 = call(0, s"hello world", p4, descr=strequaldescr)
-        escape(i0)
+        p4 = call_r(0, p1, p2, descr=strconcatdescr)
+        i0 = call_i(0, s"hello world", p4, descr=strequaldescr)
+        escape_n(i0)
         jump(p1, p2)
         """
         preamble = """
@@ -6801,8 +6797,8 @@
         p4 = newstr(i3)
         copystrcontent(p1, p4, 0, 0, i1)
         copystrcontent(p2, p4, 0, i1, i2)
-        i0 = call(0, s"hello world", p4, descr=streq_nonnull_descr)
-        escape(i0)
+        i0 = call_i(0, s"hello world", p4, descr=streq_nonnull_descr)
+        escape_n(i0)
         jump(p1, p2, i3, i1, i2)
         """
         expected = """
@@ -6810,8 +6806,8 @@
         p4 = newstr(i3)
         copystrcontent(p1, p4, 0, 0, i1)
         copystrcontent(p2, p4, 0, i1, i2)
-        i0 = call(0, s"hello world", p4, descr=streq_nonnull_descr)
-        escape(i0)
+        i0 = call_i(0, s"hello world", p4, descr=streq_nonnull_descr)
+        escape_n(i0)
         jump(p1, p2, i3, i1, i2)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
@@ -6820,13 +6816,13 @@
         ops = """
         [i1]
         p1 = newstr(0)
-        i0 = call(0, p1, s"", descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p1, s"", descr=strequaldescr)
+        escape_n(i0)
         jump(i1)
         """
         expected = """
         [i1]
-        escape(1)
+        escape_n(1)
         jump(i1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
@@ -6836,19 +6832,19 @@
         [i1]
         p1 = newstr(1)
         strsetitem(p1, 0, i1)
-        i0 = call(0, p1, s"x", descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p1, s"x", descr=strequaldescr)
+        escape_n(i0)
         jump(i1)
         """
         preamble = """
         [i1]
         i0 = int_eq(i1, 120)     # ord('x')
-        escape(i0)
+        escape_n(i0)
         jump(i1, i0)
         """
         expected = """
         [i1, i0]
-        escape(i0)
+        escape_n(i0)
         jump(i1, i0)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
@@ -6860,19 +6856,19 @@
         strsetitem(p1, 0, i1)
         p2 = newstr(1)
         strsetitem(p2, 0, i2)
-        i0 = call(0, p1, p2, descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p1, p2, descr=strequaldescr)
+        escape_n(i0)
         jump(i1, i2)
         """
         preamble = """
         [i1, i2]
         i0 = int_eq(i1, i2)
-        escape(i0)
+        escape_n(i0)
         jump(i1, i2, i0)
         """
         expected = """
         [i1, i2, i0]
-        escape(i0)
+        escape_n(i0)
         jump(i1, i2, i0)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
@@ -6883,8 +6879,8 @@
         p1 = newstr(2)
         strsetitem(p1, 0, i1)
         strsetitem(p1, 1, i2)
-        i0 = call(0, p1, s"xy", descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, p1, s"xy", descr=strequaldescr)
+        escape_n(i0)
         jump(i1, i2)
         """
         expected = """
@@ -6892,8 +6888,8 @@
         p1 = newstr(2)
         strsetitem(p1, 0, i1)
         strsetitem(p1, 1, i2)
-        i0 = call(0, p1, s"xy", descr=streq_lengthok_descr)
-        escape(i0)
+        i0 = call_i(0, p1, s"xy", descr=streq_lengthok_descr)
+        escape_n(i0)
         jump(i1, i2)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
@@ -6901,14 +6897,14 @@
     def test_str_equal_chars3(self):
         ops = """
         [p1]
-        i0 = call(0, s"x", p1, descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, s"x", p1, descr=strequaldescr)
+        escape_n(i0)
         jump(p1)
         """
         expected = """
         [p1]
-        i0 = call(0, p1, 120, descr=streq_checknull_char_descr)
-        escape(i0)
+        i0 = call_i(0, p1, 120, descr=streq_checknull_char_descr)
+        escape_n(i0)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
@@ -6918,13 +6914,13 @@
         [i1]
         p1 = newstr(1)
         strsetitem(p1, 0, i1)
-        i0 = call(0, s"xy", p1, descr=strequaldescr)
-        escape(i0)
+        i0 = call_i(0, s"xy", p1, descr=strequaldescr)
+        escape_n(i0)
         jump(i1)
         """
         expected = """
         [i1]
-        escape(0)
+        escape_n(0)
         jump(i1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
@@ -6932,16 +6928,16 @@
     def test_str2unicode_constant(self):
         ops = """
         []
-        escape(1213)
-        p0 = call(0, "xy", descr=s2u_descr)      # string -> unicode
+        escape_n(1213)
+        p0 = call_r(0, "xy", descr=s2u_descr)      # string -> unicode
         guard_no_exception() []
-        escape(p0)
+        escape_n(p0)
         jump()
         """
         expected = """
         []
-        escape(1213)
-        escape(u"xy")
+        escape_n(1213)
+        escape_n(u"xy")
         jump()
         """
         self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
@@ -6949,9 +6945,9 @@
     def test_str2unicode_nonconstant(self):
         ops = """
         [p0]
-        p1 = call(0, p0, descr=s2u_descr)      # string -> unicode
+        p1 = call_r(0, p0, descr=s2u_descr)      # string -> unicode
         guard_no_exception() []
-        escape(p1)
+        escape_n(p1)
         jump(p1)
         """
         self.optimize_strunicode_loop_extradescrs(ops, ops, ops)
@@ -6962,14 +6958,14 @@
     def test_record_known_class(self):
         ops = """
         [p0]
-        p1 = getfield_gc(p0, descr=nextdescr)
+        p1 = getfield_gc_r(p0, descr=nextdescr)
         record_known_class(p1, ConstClass(node_vtable))
         guard_class(p1, ConstClass(node_vtable)) []
         jump(p1)
         """
         expected = """
         [p0]
-        p1 = getfield_gc(p0, descr=nextdescr)
+        p1 = getfield_gc_r(p0, descr=nextdescr)
         jump(p1)
         """
         self.optimize_loop(ops, expected)
@@ -6979,14 +6975,14 @@
         [p0, p1, i0]
         quasiimmut_field(p0, descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i1 = getfield_gc_pure(p0, descr=quasifielddescr)
-        escape(i1)
+        i1 = getfield_gc_pure_i(p0, descr=quasifielddescr)
+        escape_n(i1)
         jump(p1, p0, i1)
         """
         expected = """
         [p0, p1, i0]
-        i1 = getfield_gc_pure(p0, descr=quasifielddescr)
-        escape(i1)
+        i1 = getfield_gc_pure_i(p0, descr=quasifielddescr)
+        escape_n(i1)
         jump(p1, p0, i1)
         """
         self.optimize_loop(ops, expected)
@@ -6996,14 +6992,14 @@
         []
         quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i1 = getfield_gc_pure(ConstPtr(quasiptr), descr=quasifielddescr)
-        escape(i1)
+        i1 = getfield_gc_pure_i(ConstPtr(quasiptr), descr=quasifielddescr)
+        escape_n(i1)
         jump()
         """
         expected = """
         []
         guard_not_invalidated() []
-        escape(-4247)
+        escape_n(-4247)
         jump()
         """
         self.optimize_loop(ops, expected, expected)
@@ -7030,14 +7026,14 @@
         ops = """
         [i0]
         guard_not_invalidated() []
-        call_may_force(i0, descr=mayforcevirtdescr)
+        call_may_force_n(i0, descr=mayforcevirtdescr)
         guard_not_invalidated() []
         jump(i0)
         """
         expected = """
         [i0]
         guard_not_invalidated() []
-        call_may_force(i0, descr=mayforcevirtdescr)
+        call_may_force_n(i0, descr=mayforcevirtdescr)
         guard_not_invalidated() []
         jump(i0)
         """
@@ -7048,22 +7044,22 @@
         [i0a, i0b]
         quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i1 = getfield_gc_pure(ConstPtr(quasiptr), descr=quasifielddescr)
-        call_may_force(i0b, descr=mayforcevirtdescr)
+        i1 = getfield_gc_pure_i(ConstPtr(quasiptr), descr=quasifielddescr)
+        call_may_force_n(i0b, descr=mayforcevirtdescr)
         quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i2 = getfield_gc_pure(ConstPtr(quasiptr), descr=quasifielddescr)
-        i3 = escape(i1)
-        i4 = escape(i2)
+        i2 = getfield_gc_pure_i(ConstPtr(quasiptr), descr=quasifielddescr)
+        i3 = escape_i(i1)
+        i4 = escape_i(i2)
         jump(i3, i4)
         """
         expected = """
         [i0a, i0b]
         guard_not_invalidated() []
-        call_may_force(i0b, descr=mayforcevirtdescr)
+        call_may_force_n(i0b, descr=mayforcevirtdescr)
         guard_not_invalidated() []
-        i3 = escape(-4247)
-        i4 = escape(-4247)
+        i3 = escape_i(-4247)
+        i4 = escape_i(-4247)
         jump(i3, i4)
         """
         self.optimize_loop(ops, expected)
@@ -7075,20 +7071,20 @@
         setfield_gc(p, 421, descr=quasifielddescr)
         quasiimmut_field(p, descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i1 = getfield_gc_pure(p, descr=quasifielddescr)
-        call_may_force(i0b, descr=mayforcevirtdescr)
+        i1 = getfield_gc_pure_i(p, descr=quasifielddescr)
+        call_may_force_n(i0b, descr=mayforcevirtdescr)
         quasiimmut_field(p, descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i2 = getfield_gc_pure(p, descr=quasifielddescr)
-        i3 = escape(i1)
-        i4 = escape(i2)
+        i2 = getfield_gc_pure_i(p, descr=quasifielddescr)
+        i3 = escape_i(i1)
+        i4 = escape_i(i2)
         jump(i3, i4)
         """
         expected = """
         [i0a, i0b]
-        call_may_force(i0b, descr=mayforcevirtdescr)
-        i3 = escape(421)
-        i4 = escape(421)
+        call_may_force_n(i0b, descr=mayforcevirtdescr)
+        i3 = escape_i(421)
+        i4 = escape_i(421)
         jump(i3, i4)
         """
         self.optimize_loop(ops, expected)
@@ -7096,23 +7092,23 @@
     def test_constant_getfield1(self):
         ops = """
         [p1, p187, i184]
-        p188 = getarrayitem_gc(p187, 42, descr=<GcPtrArrayDescr>)
+        p188 = getarrayitem_gc_r(p187, 42, descr=<GcPtrArrayDescr>)
         guard_value(p188, ConstPtr(myptr)) []
-        p25 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
+        p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         jump(p25, p187, i184)
         """
         preamble = """
         [p1, p187, i184]
-        p188 = getarrayitem_gc(p187, 42, descr=<GcPtrArrayDescr>)
+        p188 = getarrayitem_gc_r(p187, 42, descr=<GcPtrArrayDescr>)
         guard_value(p188, ConstPtr(myptr)) []
-        p25 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
+        p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         jump(p25, p187, i184)
         """
         short = """
         [p1, p187, i184]
-        p188 = getarrayitem_gc(p187, 42, descr=<GcPtrArrayDescr>)
+        p188 = getarrayitem_gc_r(p187, 42, descr=<GcPtrArrayDescr>)
         guard_value(p188, ConstPtr(myptr)) []
-        p25 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
+        p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         jump(p25, p187, i184)
         """
         expected = """
@@ -7124,15 +7120,15 @@
     def test_constant_getfield1bis(self):
         ops = """
         [p1, p187, i184]
-        p188 = getarrayitem_gc(p187, 42, descr=<GcPtrArrayDescr>)
+        p188 = getarrayitem_gc_r(p187, 42, descr=<GcPtrArrayDescr>)
         guard_value(p188, ConstPtr(myptr)) []
-        p25 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
-        p26 = call(p25, descr=nonwritedescr)
+        p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
+        p26 = call_r(p25, descr=nonwritedescr)
         jump(p26, p187, i184)
         """
         expected = """
         [p24, p187, i184, p25]
-        p26 = call(p25, descr=nonwritedescr)
+        p26 = call_r(p25, descr=nonwritedescr)
         jump(p26, p187, i184, p25)
         """
         self.optimize_loop(ops, expected)
@@ -7140,7 +7136,7 @@
     def test_constant_getfield2(self):
         ops = """
         [p19]
-        p22 = getfield_gc(p19, descr=otherdescr)
+        p22 = getfield_gc_r(p19, descr=otherdescr)
         guard_value(p19, ConstPtr(myptr)) []
         jump(p19)
         """
@@ -7153,14 +7149,14 @@
     def test_constant_getfield3(self):
         ops = """
         [p19, p20, p21]
-        p22 = getfield_gc(p19, descr=otherdescr)
+        p22 = getfield_gc_r(p19, descr=otherdescr)
         guard_value(p19, ConstPtr(myptr)) []
-        p23 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
+        p23 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         jump(p20, p21, p21)
         """
         expected = """
         [p20, p21]
-        p22 = getfield_gc(p20, descr=otherdescr)
+        p22 = getfield_gc_r(p20, descr=otherdescr)
         guard_value(p20, ConstPtr(myptr)) []
         jump(p21, p21)
         """
@@ -7169,14 +7165,14 @@
     def test_constant_getfield4(self):
         ops = """
         [p19, p20, p21]
-        p22 = getfield_gc(p19, descr=otherdescr)
-        p23 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
+        p22 = getfield_gc_r(p19, descr=otherdescr)
+        p23 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         guard_value(p19, ConstPtr(myptr)) []
         jump(p20, p21, p21)
         """
         expected = """
         [p20, p21]
-        p22 = getfield_gc(p20, descr=otherdescr)
+        p22 = getfield_gc_r(p20, descr=otherdescr)
         guard_value(p20, ConstPtr(myptr)) []
         jump(p21, p21)
         """
@@ -7185,18 +7181,18 @@
     def test_constnats_among_virtual_fileds(self):
         ops = """
         [p19, p20, p21]
-        p1 = getfield_gc(p20, descr=valuedescr)
-        p2 = getfield_gc(p1, descr=otherdescr)
+        p1 = getfield_gc_r(p20, descr=valuedescr)
+        p2 = getfield_gc_r(p1, descr=otherdescr)
         pv = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(pv, p19, descr=valuedescr)
-        p22 = getfield_gc(p19, descr=otherdescr)
+        p22 = getfield_gc_r(p19, descr=otherdescr)
         guard_value(p19, ConstPtr(myptr)) []
-        p23 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
+        p23 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         jump(p21, pv, p21)
         """
         expected = """
         [p20]
-        p22 = getfield_gc(p20, descr=otherdescr)
+        p22 = getfield_gc_r(p20, descr=otherdescr)
         guard_value(p20, ConstPtr(myptr)) []
         jump(ConstPtr(myptr))
         """
@@ -7210,19 +7206,19 @@
         # results of the two getfields as separate boxes.
         ops = """
         [p0, p1, ii, ii2]
-        i1 = getfield_gc(p0, descr=valuedescr)
-        i2 = getfield_gc(p1, descr=otherdescr)
+        i1 = getfield_gc_i(p0, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=otherdescr)
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
         setfield_gc(p1, ii, descr=otherdescr)
-        i4 = getfield_gc(p0, descr=valuedescr)
-        i5 = getfield_gc(p1, descr=otherdescr)
+        i4 = getfield_gc_i(p0, descr=valuedescr)
+        i5 = getfield_gc_i(p1, descr=otherdescr)
         jump(p0, p1, ii2, ii)
         """
         preamble = """
         [p0, p1, ii, ii2]
-        i1 = getfield_gc(p0, descr=valuedescr)
-        i2 = getfield_gc(p1, descr=otherdescr)
+        i1 = getfield_gc_i(p0, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=otherdescr)
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
         setfield_gc(p1, ii, descr=otherdescr)
@@ -7230,8 +7226,8 @@
         """
         expected = """
         [p0, p1, ii, ii2]
-        i1 = getfield_gc(p0, descr=valuedescr)
-        i2 = getfield_gc(p1, descr=otherdescr)
+        i1 = getfield_gc_i(p0, descr=valuedescr)
+        i2 = getfield_gc_i(p1, descr=otherdescr)
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
         setfield_gc(p1, ii, descr=otherdescr)
@@ -7242,8 +7238,8 @@
     def test_dont_specialize_on_boxes_equal(self):
         ops = """

From noreply at buildbot.pypy.org  Wed Dec 17 23:03:36 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Dec 2014 23:03:36 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: correctly handle exec in tuple
 form (cpython issue21591)
Message-ID: <20141217220336.5A1BF1C0715@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r74999:2e4e36c84077
Date: 2014-12-17 17:00 -0500
http://bitbucket.org/pypy/pypy/changeset/2e4e36c84077/

Log:	correctly handle exec in tuple form (cpython issue21591)

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -277,10 +277,16 @@
         globs = None
         locs = None
         to_execute = self.handle_expr(exec_node.children[1])
-        if child_count >= 4:
+        if isinstance(to_execute, ast.Tuple) and child_count < 4 and \
+                len(to_execute.elts) in {2, 3}:
+            globs = to_execute.elts[1]
+            if len(to_execute.elts) == 3:
+                locs = to_execute.elts[2]
+            to_execute = to_execute.elts[0]
+        elif child_count >= 4:
             globs = self.handle_expr(exec_node.children[3])
-        if child_count == 6:
-            locs = self.handle_expr(exec_node.children[5])
+            if child_count == 6:
+                locs = self.handle_expr(exec_node.children[5])
         return ast.Exec(to_execute, globs, locs, exec_node.lineno,
                         exec_node.column)
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1619,13 +1619,6 @@
     def prepare_exec(f, prog, globals, locals, compile_flags, builtin, codetype):
         """Manipulate parameters to exec statement to (codeobject, dict, dict).
         """
-        if (globals is None and locals is None and
-            isinstance(prog, tuple) and
-            (len(prog) == 2 or len(prog) == 3)):
-            globals = prog[1]
-            if len(prog) == 3:
-                locals = prog[2]
-            prog = prog[0]
         if globals is None:
             globals = f.f_globals
             if locals is None:
diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py
--- a/pypy/interpreter/test/test_exec.py
+++ b/pypy/interpreter/test/test_exec.py
@@ -18,8 +18,7 @@
     ''')
 
 
-class AppTestExecStmt: 
-
+class AppTestExecStmt:
     def test_string(self):
         g = {}
         l = {}
@@ -30,7 +29,7 @@
         g = {}
         exec "a = 3" in g
         assert g['a'] == 3
-        
+
     def test_builtinsupply(self):
         g = {}
         exec "pass" in g
@@ -63,7 +62,7 @@
         l = {}
         exec ("a = 3", g, l)
         assert l['a'] == 3
-        
+
     def test_tupleglobals(self):
         g = {}
         exec ("a = 3", g)
@@ -207,9 +206,8 @@
     def test_exec_and_name_lookups(self):
         ns = {}
         exec """def f():
-        exec 'x=1' in locals()
-        return x
-""" in ns
+            exec 'x=1' in locals()
+            return x""" in ns
 
         f = ns['f']
 
@@ -246,3 +244,18 @@
         exec c
         assert len(x) == 6
         assert ord(x[0]) == 0x0439
+
+    def test_nested_qualified_exec(self):
+        code = ["""
+def g():
+    def f():
+        if True:
+            exec "" in {}, {}
+        """, """
+def g():
+    def f():
+        if True:
+            exec("", {}, {})
+        """]
+        for c in code:
+            compile(c, "<code>", "exec")

From noreply at buildbot.pypy.org  Wed Dec 17 23:15:57 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Wed, 17 Dec 2014 23:15:57 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: try to fix translation
Message-ID: <20141217221557.E96161C08E7@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75000:8d5bf221300a
Date: 2014-12-17 17:15 -0500
http://bitbucket.org/pypy/pypy/changeset/8d5bf221300a/

Log:	try to fix translation

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -277,12 +277,13 @@
         globs = None
         locs = None
         to_execute = self.handle_expr(exec_node.children[1])
-        if isinstance(to_execute, ast.Tuple) and child_count < 4 and \
-                len(to_execute.elts) in {2, 3}:
-            globs = to_execute.elts[1]
-            if len(to_execute.elts) == 3:
-                locs = to_execute.elts[2]
-            to_execute = to_execute.elts[0]
+        if child_count < 4:
+            if isinstance(to_execute, ast.Tuple) and \
+                    (len(to_execute.elts) == 2 or len(to_execute.elts) == 3):
+                globs = to_execute.elts[1]
+                if len(to_execute.elts) == 3:
+                    locs = to_execute.elts[2]
+                to_execute = to_execute.elts[0]
         elif child_count >= 4:
             globs = self.handle_expr(exec_node.children[3])
             if child_count == 6:

From noreply at buildbot.pypy.org  Thu Dec 18 00:38:20 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Dec 2014 00:38:20 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: SSL: Add support for npn_protocols
Message-ID: <20141217233821.0A4B31C0715@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75001:c62ae9b1f120
Date: 2014-12-16 18:40 +0100
http://bitbucket.org/pypy/pypy/changeset/c62ae9b1f120/

Log:	SSL: Add support for npn_protocols

diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py
--- a/lib-python/3/test/test_ssl.py
+++ b/lib-python/3/test/test_ssl.py
@@ -1830,9 +1830,9 @@
                         outdata = s.read()
                         if outdata != indata.lower():
                             self.fail(
-                                "While sending with <<{name:s}>> bad data "
-                                "<<{outdata:r}>> ({nout:d}) received; "
-                                "expected <<{indata:r}>> ({nin:d})\n".format(
+                                "While sending with <<{name}>> bad data "
+                                "<<{outdata!r}>> ({nout}) received; "
+                                "expected <<{indata!r}>> ({nin})\n".format(
                                     name=meth_name, outdata=outdata[:20],
                                     nout=len(outdata),
                                     indata=indata[:20], nin=len(indata)
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1,9 +1,10 @@
 import weakref
 
 from rpython.rlib import rpoll, rsocket
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import intmask, widen, r_uint
 from rpython.rlib.ropenssl import *
 from rpython.rlib.rposix import get_errno, set_errno
+from rpython.rlib.rweakref import RWeakValueDictionary
 from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.interpreter.baseobjspace import W_Root
@@ -69,7 +70,7 @@
 constants["OP_NO_TLSv1"] = SSL_OP_NO_TLSv1
 constants["HAS_SNI"] = HAS_SNI
 constants["HAS_ECDH"] = True  # To break the test suite
-constants["HAS_NPN"] = True  # To break the test suite
+constants["HAS_NPN"] = HAS_NPN
 constants["HAS_TLS_UNIQUE"] = True  # To break the test suite
 
 # OpenSSL version
@@ -95,6 +96,53 @@
                                       space.wrap(errno), space.wrap(msg))
     return OperationError(w_errtype, w_exception)
 
+class SSLNpnProtocols(object):
+
+    def __init__(self, ctx, protos):
+        self.protos = protos
+        self.buf, self.pinned, self.is_raw = rffi.get_nonmovingbuffer(protos)
+        NPN_STORAGE.set(r_uint(rffi.cast(rffi.UINT, self.buf)), self)
+
+        # set both server and client callbacks, because the context
+        # can be used to create both types of sockets
+        libssl_SSL_CTX_set_next_protos_advertised_cb(
+            ctx, self.advertiseNPN_cb, self.buf)
+        libssl_SSL_CTX_set_next_proto_select_cb(
+            ctx, self.selectNPN_cb, self.buf)
+
+    def __del__(self):
+        rffi.free_nonmovingbuffer(
+            self.protos, self.buf, self.pinned, self.is_raw)    
+
+    @staticmethod
+    def advertiseNPN_cb(s, data_ptr, len_ptr, args):
+        npn = NPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        if npn and npn.protos:
+            data_ptr[0] = npn.buf
+            len_ptr[0] = rffi.cast(rffi.UINT, len(npn.protos))
+        else:
+            data_ptr[0] = lltype.nullptr(rffi.CCHARP.TO)
+            len_ptr[0] = rffi.cast(rffi.UINT, 0)
+
+        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
+
+    @staticmethod
+    def selectNPN_cb(s, out_ptr, outlen_ptr, server, server_len, args):
+        npn = NPN_STORAGE.get(r_uint(rffi.cast(rffi.UINT, args)))
+        if npn and npn.protos:
+            client = npn.buf
+            client_len = len(npn.protos)
+        else:
+            client = lltype.nullptr(rffi.CCHARP.TO)
+            client_len = 0            
+
+        libssl_SSL_select_next_proto(out_ptr, outlen_ptr,
+                                     server, server_len,
+                                     client, client_len)
+        return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK)
+
+NPN_STORAGE = RWeakValueDictionary(r_uint, SSLNpnProtocols)
+
 
 class SSLContext(W_Root):
     ctx = lltype.nullptr(SSL_CTX.TO)
@@ -277,6 +325,13 @@
         if ret != 1:
             raise _ssl_seterror(space, None, -1)
 
+    @unwrap_spec(protos='bufferstr')
+    def set_npn_protocols_w(self, space, protos):
+        if not HAS_NPN:
+            raise oefmt(space.w_NotImplementedError,
+                        "The NPN extension requires OpenSSL 1.0.1 or later.")
+
+        self.npn_protocols = SSLNpnProtocols(self.ctx, protos)
 
 SSLContext.typedef = TypeDef(
     "_SSLContext",
@@ -291,6 +346,7 @@
     load_verify_locations = interp2app(SSLContext.load_verify_locations_w),
     session_stats = interp2app(SSLContext.session_stats_w),
     set_default_verify_paths=interp2app(SSLContext.set_default_verify_paths_w),
+    _set_npn_protocols=interp2app(SSLContext.set_npn_protocols_w),
 )
 
     
@@ -640,6 +696,15 @@
             else:
                 return _decode_certificate(space, self.peer_cert)
 
+    def selected_npn_protocol(self, space):
+        with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as out_ptr:
+            with lltype.scoped_alloc(rffi.UINTP.TO, 1) as len_ptr:
+                libssl_SSL_get0_next_proto_negotiated(self.ssl,
+                                                      out_ptr, len_ptr)
+                if out_ptr[0]:
+                    return space.wrap(
+                        rffi.charpsize2str(out_ptr[0], widen(len_ptr[0])))
+
 def _decode_certificate(space, certificate, verbose=False):
     w_retval = space.newdict()
 
@@ -850,6 +915,7 @@
     shutdown = interp2app(SSLSocket.shutdown),
     cipher = interp2app(SSLSocket.cipher),
     peer_certificate = interp2app(SSLSocket.peer_certificate),
+    selected_npn_protocol = interp2app(SSLSocket.selected_npn_protocol),
 )
 
 
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -181,6 +181,15 @@
         self.s.close()
         del ss; gc.collect()
         
+    def test_npn_protocol(self):
+        import socket, _ssl, gc
+        ctx = _ssl._SSLContext()
+        ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2')
+        ss = ctx._wrap_socket(self.s, True,
+                              server_hostname="svn.python.org")
+        self.s.close()
+        del ss; gc.collect()
+
 
 class AppTestConnectedSSL_Timeout(AppTestConnectedSSL):
     # Same tests, with a socket timeout
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -80,6 +80,7 @@
     SSL_OP_SINGLE_DH_USE = rffi_platform.ConstantInteger(
         "SSL_OP_SINGLE_DH_USE")
     HAS_SNI = rffi_platform.Defined("SSL_CTRL_SET_TLSEXT_HOSTNAME")
+    HAS_NPN = rffi_platform.Defined("OPENSSL_NPN_NEGOTIATED")
     SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE")
     SSL_VERIFY_PEER = rffi_platform.ConstantInteger("SSL_VERIFY_PEER")
     SSL_VERIFY_FAIL_IF_NO_PEER_CERT = rffi_platform.ConstantInteger("SSL_VERIFY_FAIL_IF_NO_PEER_CERT")
@@ -99,6 +100,7 @@
         "SSL_RECEIVED_SHUTDOWN")
     SSL_MODE_AUTO_RETRY = rffi_platform.ConstantInteger("SSL_MODE_AUTO_RETRY")
     SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER = rffi_platform.ConstantInteger("SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER")
+    SSL_TLSEXT_ERR_OK = rffi_platform.ConstantInteger("SSL_TLSEXT_ERR_OK")
 
     NID_subject_alt_name = rffi_platform.ConstantInteger("NID_subject_alt_name")
     GEN_DIRNAME = rffi_platform.ConstantInteger("GEN_DIRNAME")
@@ -312,6 +314,24 @@
 ssl_external('PEM_read_bio_X509_AUX',
              [BIO, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP], X509)
 
+if HAS_NPN:
+    SSL_NEXT_PROTOS_ADV_CB = lltype.Ptr(lltype.FuncType(
+        [SSL, rffi.CCHARPP, rffi.UINTP, rffi.VOIDP], rffi.INT))
+    ssl_external('SSL_CTX_set_next_protos_advertised_cb',
+                 [SSL_CTX, SSL_NEXT_PROTOS_ADV_CB, rffi.VOIDP], lltype.Void)
+    SSL_NEXT_PROTOS_SEL_CB = lltype.Ptr(lltype.FuncType(
+        [SSL, rffi.CCHARPP, rffi.UCHARP, rffi.CCHARP, rffi.UINT, rffi.VOIDP],
+        rffi.INT))
+    ssl_external('SSL_CTX_set_next_proto_select_cb',
+                 [SSL_CTX, SSL_NEXT_PROTOS_SEL_CB, rffi.VOIDP], lltype.Void)
+    ssl_external(
+        'SSL_select_next_proto', [rffi.CCHARPP, rffi.UCHARP,
+                                  rffi.CCHARP, rffi.UINT,
+                                  rffi.CCHARP, rffi.UINT], rffi.INT)
+    ssl_external(
+        'SSL_get0_next_proto_negotiated', [
+            SSL, rffi.CCHARPP, rffi.UINTP], lltype.Void)
+
 EVP_MD_CTX = rffi.COpaquePtr('EVP_MD_CTX', compilation_info=eci)
 EVP_MD     = lltype.Ptr(EVP_MD_st)
 

From noreply at buildbot.pypy.org  Thu Dec 18 00:38:22 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Dec 2014 00:38:22 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: SSL: implement tls_unique_cb()
Message-ID: <20141217233822.713B61C0715@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75002:e8613bc29b37
Date: 2014-12-16 19:12 +0100
http://bitbucket.org/pypy/pypy/changeset/e8613bc29b37/

Log:	SSL: implement tls_unique_cb()

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -705,6 +705,27 @@
                     return space.wrap(
                         rffi.charpsize2str(out_ptr[0], widen(len_ptr[0])))
 
+    def tls_unique_cb(self, space):
+        """Returns the 'tls-unique' channel binding data, as defined by RFC 5929.
+        If the TLS handshake is not yet complete, None is returned"""
+
+        # In case of 'tls-unique' it will be 12 bytes for TLS, 36
+        # bytes for older SSL, but let's be safe
+        CB_MAXLEN = 128
+
+        with lltype.scoped_alloc(rffi.CCHARP.TO, CB_MAXLEN) as buf:
+            if (libssl_SSL_session_reused(self.ssl) ^ 
+                (self.socket_side == PY_SSL_CLIENT)):
+                # if session is resumed XOR we are the client
+                length = libssl_SSL_get_finished(self.ssl, buf, CB_MAXLEN)
+            else:
+                # if a new session XOR we are the server
+                length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN)
+            
+            if length > 0:
+                return space.wrapbytes(rffi.charpsize2str(buf, intmask(length)))
+
+
 def _decode_certificate(space, certificate, verbose=False):
     w_retval = space.newdict()
 
@@ -916,6 +937,7 @@
     cipher = interp2app(SSLSocket.cipher),
     peer_certificate = interp2app(SSLSocket.peer_certificate),
     selected_npn_protocol = interp2app(SSLSocket.selected_npn_protocol),
+    tls_unique_cb = interp2app(SSLSocket.tls_unique_cb),
 )
 
 
@@ -947,6 +969,7 @@
         libssl_SSL_set_connect_state(ss.ssl)
     else:
         libssl_SSL_set_accept_state(ss.ssl)
+    ss.socket_side = side
 
     ss.w_socket = weakref.ref(w_sock)
     return ss
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -190,6 +190,14 @@
         self.s.close()
         del ss; gc.collect()
 
+    def test_tls_unique_cb(self):
+        import ssl, sys, gc
+        ss = ssl.wrap_socket(self.s)
+        ss.do_handshake()
+        assert isinstance(ss.get_channel_binding(), bytes)
+        self.s.close()
+        del ss; gc.collect()
+
 
 class AppTestConnectedSSL_Timeout(AppTestConnectedSSL):
     # Same tests, with a socket timeout
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -242,6 +242,9 @@
 ssl_external('SSL_get_shutdown', [SSL], rffi.INT)
 ssl_external('SSL_set_read_ahead', [SSL, rffi.INT], lltype.Void)
 ssl_external('SSL_set_tlsext_host_name', [SSL, rffi.CCHARP], rffi.INT, macro=True)
+ssl_external('SSL_session_reused', [SSL], rffi.INT, macro=True)
+ssl_external('SSL_get_finished', [SSL, rffi.CCHARP, rffi.SIZE_T], rffi.SIZE_T)
+ssl_external('SSL_get_peer_finished', [SSL, rffi.CCHARP, rffi.SIZE_T], rffi.SIZE_T)
 
 ssl_external('SSL_get_peer_certificate', [SSL], X509)
 ssl_external('X509_get_subject_name', [X509], X509_NAME)

From noreply at buildbot.pypy.org  Thu Dec 18 00:38:23 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Dec 2014 00:38:23 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: SSL: add compression() method
Message-ID: <20141217233823.B8BE41C0715@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75003:5efb69f6c4d5
Date: 2014-12-18 00:28 +0100
http://bitbucket.org/pypy/pypy/changeset/5efb69f6c4d5/

Log:	SSL: add compression() method

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -705,6 +705,19 @@
                     return space.wrap(
                         rffi.charpsize2str(out_ptr[0], widen(len_ptr[0])))
 
+    def compression(self, space):
+        # if OPENSSL_NO_COMP
+        #     return None
+        if not self.ssl:
+            return None
+        comp_method = libssl_SSL_get_current_compression(self.ssl)
+        if not comp_method or widen(comp_method[0].c_type) == NID_undef:
+            return None
+        short_name = libssl_OBJ_nid2sn(comp_method[0].c_type)
+        if not short_name:
+            return None
+        return space.wrap(rffi.charp2str(short_name))
+
     def tls_unique_cb(self, space):
         """Returns the 'tls-unique' channel binding data, as defined by RFC 5929.
         If the TLS handshake is not yet complete, None is returned"""
@@ -937,6 +950,7 @@
     cipher = interp2app(SSLSocket.cipher),
     peer_certificate = interp2app(SSLSocket.peer_certificate),
     selected_npn_protocol = interp2app(SSLSocket.selected_npn_protocol),
+    compression = interp2app(SSLSocket.compression),
     tls_unique_cb = interp2app(SSLSocket.tls_unique_cb),
 )
 
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -198,6 +198,14 @@
         self.s.close()
         del ss; gc.collect()
 
+    def test_compression(self):
+        import ssl, sys, gc
+        ss = ssl.wrap_socket(self.s)
+        ss.do_handshake()
+        assert ss.compression() in [None, 'ZLIB', 'RLE']
+        self.s.close()
+        del ss; gc.collect()
+
 
 class AppTestConnectedSSL_Timeout(AppTestConnectedSSL):
     # Same tests, with a socket timeout
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -102,6 +102,7 @@
     SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER = rffi_platform.ConstantInteger("SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER")
     SSL_TLSEXT_ERR_OK = rffi_platform.ConstantInteger("SSL_TLSEXT_ERR_OK")
 
+    NID_undef = rffi_platform.ConstantInteger("NID_undef")
     NID_subject_alt_name = rffi_platform.ConstantInteger("NID_subject_alt_name")
     GEN_DIRNAME = rffi_platform.ConstantInteger("GEN_DIRNAME")
     GEN_EMAIL = rffi_platform.ConstantInteger("GEN_EMAIL")
@@ -149,6 +150,10 @@
          ('name', rffi.CCHARP),
          ])
 
+    COMP_METHOD_st = rffi_platform.Struct(
+        'COMP_METHOD',
+        [('type', rffi.INT),
+        ])
 
 for k, v in rffi_platform.configure(CConfig).items():
     globals()[k] = v
@@ -170,6 +175,7 @@
 GENERAL_NAMES = rffi.COpaquePtr('GENERAL_NAMES')
 GENERAL_NAME = rffi.CArrayPtr(GENERAL_NAME_st)
 OBJ_NAME = rffi.CArrayPtr(OBJ_NAME_st)
+COMP_METHOD = rffi.CArrayPtr(COMP_METHOD_st)
 
 HAVE_OPENSSL_RAND = OPENSSL_VERSION_NUMBER >= 0x0090500f
 HAVE_SSL_CTX_CLEAR_OPTIONS = OPENSSL_VERSION_NUMBER >= 0x009080df
@@ -283,12 +289,16 @@
 ssl_external('GENERAL_NAME_print', [BIO, GENERAL_NAME], rffi.INT)
 ssl_external('pypy_GENERAL_NAME_dirn', [GENERAL_NAME], X509_NAME,
              macro=True)
+ssl_external('OBJ_nid2sn',
+             [rffi.INT], rffi.CCHARP)
 
 ssl_external('SSL_get_current_cipher', [SSL], SSL_CIPHER)
 ssl_external('SSL_CIPHER_get_name', [SSL_CIPHER], rffi.CCHARP)
 ssl_external('SSL_CIPHER_get_version', [SSL_CIPHER], rffi.CCHARP)
 ssl_external('SSL_CIPHER_get_bits', [SSL_CIPHER, rffi.INTP], rffi.INT)
 
+ssl_external('SSL_get_current_compression', [SSL], COMP_METHOD)
+
 ssl_external('ERR_get_error', [], rffi.INT)
 ssl_external('ERR_peek_last_error', [], rffi.INT)
 ssl_external('ERR_error_string', [rffi.ULONG, rffi.CCHARP], rffi.CCHARP)

From noreply at buildbot.pypy.org  Thu Dec 18 00:38:24 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Dec 2014 00:38:24 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Be sure to initialize all fields in the
 socket object, even if __init__ is not called.
Message-ID: <20141217233824.EFE731C0715@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75004:eea8f92e03d4
Date: 2014-12-18 00:36 +0100
http://bitbucket.org/pypy/pypy/changeset/eea8f92e03d4/

Log:	Be sure to initialize all fields in the socket object, even if
	__init__ is not called.

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -144,6 +144,7 @@
 
     def descr_new(space, w_subtype, __args__):
         sock = space.allocate_instance(W_Socket, w_subtype)
+        W_Socket.__init__(sock, RSocket.empty_rsocket())
         return space.wrap(sock)
 
     @unwrap_spec(family=int, type=int, proto=int,
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -589,7 +589,7 @@
         finally:
             os.chdir(oldcwd)
 
-    def test_subclass(self):
+    def test_subclass_init(self):
         # Socket is not created in __new__, but in __init__.
         import socket
         class Socket_IPV6(socket.socket):
@@ -597,6 +597,18 @@
                 socket.socket.__init__(self, family=socket.AF_INET6)
         assert Socket_IPV6().family == socket.AF_INET6
 
+    def test_subclass_noinit(self):
+        from _socket import socket
+        class MySock(socket):
+            def __init__(self, *args):
+                pass  # don't call super
+        s = MySock()
+        assert s.type == 0
+        assert s.proto == 0
+        assert s.family == 0
+        assert s.fileno() < 0
+        raises(OSError, s.bind, ('localhost', 0))
+
     def test_dealloc_warn(self):
         import _socket
         import gc
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -497,6 +497,10 @@
     """RPython-level socket object.
     """
     fd = _c.INVALID_SOCKET
+    family = 0
+    type = 0
+    proto = 0
+    timeout = -1.0
 
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0,
                  fd=_c.INVALID_SOCKET):
@@ -512,6 +516,11 @@
         self.proto = proto
         self.timeout = defaults.timeout
 
+    @staticmethod
+    def empty_rsocket():
+        rsocket = instantiate(RSocket)
+        return rsocket
+
     @rgc.must_be_light_finalizer
     def __del__(self):
         fd = self.fd

From noreply at buildbot.pypy.org  Thu Dec 18 00:53:30 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 18 Dec 2014 00:53:30 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Don't assume __builtins__ is a
	dict.
Message-ID: <20141217235330.9D97A1C08E7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75005:aabd37028202
Date: 2014-12-17 15:53 -0800
http://bitbucket.org/pypy/pypy/changeset/aabd37028202/

Log:	Don't assume __builtins__ is a dict.

diff --git a/lib-python/2.7/test/_mock_backport.py b/lib-python/2.7/test/_mock_backport.py
--- a/lib-python/2.7/test/_mock_backport.py
+++ b/lib-python/2.7/test/_mock_backport.py
@@ -23,7 +23,7 @@
 
 __version__ = '1.0'
 
-
+import __builtin__
 import inspect
 import pprint
 import sys
@@ -32,7 +32,7 @@
 from functools import wraps, partial
 
 
-_builtins = {name for name in __builtins__ if not name.startswith('_')}
+_builtins = {name for name in dir(__builtin__) if not name.startswith('_')}
 
 BaseExceptions = (BaseException,)
 if 'java' in sys.platform:

From noreply at buildbot.pypy.org  Thu Dec 18 00:53:32 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 18 Dec 2014 00:53:32 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: merged upstream
Message-ID: <20141217235332.0BBFF1C08E7@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75006:abfb4eb09565
Date: 2014-12-17 15:53 -0800
http://bitbucket.org/pypy/pypy/changeset/abfb4eb09565/

Log:	merged upstream

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -277,10 +277,17 @@
         globs = None
         locs = None
         to_execute = self.handle_expr(exec_node.children[1])
-        if child_count >= 4:
+        if child_count < 4:
+            if isinstance(to_execute, ast.Tuple) and \
+                    (len(to_execute.elts) == 2 or len(to_execute.elts) == 3):
+                globs = to_execute.elts[1]
+                if len(to_execute.elts) == 3:
+                    locs = to_execute.elts[2]
+                to_execute = to_execute.elts[0]
+        elif child_count >= 4:
             globs = self.handle_expr(exec_node.children[3])
-        if child_count == 6:
-            locs = self.handle_expr(exec_node.children[5])
+            if child_count == 6:
+                locs = self.handle_expr(exec_node.children[5])
         return ast.Exec(to_execute, globs, locs, exec_node.lineno,
                         exec_node.column)
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1619,13 +1619,6 @@
     def prepare_exec(f, prog, globals, locals, compile_flags, builtin, codetype):
         """Manipulate parameters to exec statement to (codeobject, dict, dict).
         """
-        if (globals is None and locals is None and
-            isinstance(prog, tuple) and
-            (len(prog) == 2 or len(prog) == 3)):
-            globals = prog[1]
-            if len(prog) == 3:
-                locals = prog[2]
-            prog = prog[0]
         if globals is None:
             globals = f.f_globals
             if locals is None:
diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py
--- a/pypy/interpreter/test/test_exec.py
+++ b/pypy/interpreter/test/test_exec.py
@@ -18,8 +18,7 @@
     ''')
 
 
-class AppTestExecStmt: 
-
+class AppTestExecStmt:
     def test_string(self):
         g = {}
         l = {}
@@ -30,7 +29,7 @@
         g = {}
         exec "a = 3" in g
         assert g['a'] == 3
-        
+
     def test_builtinsupply(self):
         g = {}
         exec "pass" in g
@@ -63,7 +62,7 @@
         l = {}
         exec ("a = 3", g, l)
         assert l['a'] == 3
-        
+
     def test_tupleglobals(self):
         g = {}
         exec ("a = 3", g)
@@ -207,9 +206,8 @@
     def test_exec_and_name_lookups(self):
         ns = {}
         exec """def f():
-        exec 'x=1' in locals()
-        return x
-""" in ns
+            exec 'x=1' in locals()
+            return x""" in ns
 
         f = ns['f']
 
@@ -246,3 +244,18 @@
         exec c
         assert len(x) == 6
         assert ord(x[0]) == 0x0439
+
+    def test_nested_qualified_exec(self):
+        code = ["""
+def g():
+    def f():
+        if True:
+            exec "" in {}, {}
+        """, """
+def g():
+    def f():
+        if True:
+            exec("", {}, {})
+        """]
+        for c in code:
+            compile(c, "<code>", "exec")

From noreply at buildbot.pypy.org  Thu Dec 18 03:00:24 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 18 Dec 2014 03:00:24 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: add failing test for coding spec
Message-ID: <20141218020024.278C61C04B4@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75007:3ad403fdd1ae
Date: 2014-12-17 20:46 -0500
http://bitbucket.org/pypy/pypy/changeset/3ad403fdd1ae/

Log:	add failing test for coding spec

diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -990,8 +990,6 @@
         else:
             raise Exception("DID NOT RAISE")
 
-
-
     def test_bad_oudent(self):
         source = """if 1:
           x
@@ -1005,7 +1003,6 @@
         else:
             raise Exception("DID NOT RAISE")
 
-
     def test_repr_vs_str(self):
         source1 = "x = (\n"
         source2 = "x = (\n\n"
@@ -1026,3 +1023,21 @@
         err3 = eval(repr(err1))
         assert str(err3) == str(err1)
         assert repr(err3) == repr(err1)
+
+    def test_encoding(self):
+        code = b'# -*- coding: badencoding -*-\npass\n'
+        raises(SyntaxError, compile, code, 'tmp', 'exec')
+        code = u"# -*- coding: utf-8 -*-\npass\n"
+        raises(SyntaxError, compile, code, 'tmp', 'exec')
+        code = 'u"\xc2\xa4"\n'
+        assert eval(code) == u'\xc2\xa4'
+        code = u'u"\xc2\xa4"\n'
+        assert eval(code) == u'\xc2\xa4'
+        code = '# -*- coding: latin1 -*-\nu"\xc2\xa4"\n'
+        assert eval(code) == u'\xc2\xa4'
+        code = '# -*- coding: utf-8 -*-\nu"\xc2\xa4"\n'
+        assert eval(code) == u'\xa4'
+        code = '# -*- coding: iso8859-15 -*-\nu"\xc2\xa4"\n'
+        assert eval(code) == u'\xc2\u20ac'
+        code = 'u"""\\\n# -*- coding: utf-8 -*-\n\xc2\xa4"""\n'
+        assert eval(code) == u'# -*- coding: utf-8 -*-\n\xc2\xa4'

From noreply at buildbot.pypy.org  Thu Dec 18 03:00:25 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Thu, 18 Dec 2014 03:00:25 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: merge heads
Message-ID: <20141218020025.662861C04B4@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75008:b5dea7545f7c
Date: 2014-12-17 21:00 -0500
http://bitbucket.org/pypy/pypy/changeset/b5dea7545f7c/

Log:	merge heads

diff --git a/lib-python/2.7/test/_mock_backport.py b/lib-python/2.7/test/_mock_backport.py
--- a/lib-python/2.7/test/_mock_backport.py
+++ b/lib-python/2.7/test/_mock_backport.py
@@ -23,7 +23,7 @@
 
 __version__ = '1.0'
 
-
+import __builtin__
 import inspect
 import pprint
 import sys
@@ -32,7 +32,7 @@
 from functools import wraps, partial
 
 
-_builtins = {name for name in __builtins__ if not name.startswith('_')}
+_builtins = {name for name in dir(__builtin__) if not name.startswith('_')}
 
 BaseExceptions = (BaseException,)
 if 'java' in sys.platform:

From noreply at buildbot.pypy.org  Thu Dec 18 09:16:24 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 18 Dec 2014 09:16:24 +0100 (CET)
Subject: [pypy-commit] pypy default: this broke stuff, revert
Message-ID: <20141218081624.AF4ED1C03A4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75009:1568f745143e
Date: 2014-12-18 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/1568f745143e/

Log:	this broke stuff, revert

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -97,6 +97,7 @@
             jumpop = None
 
         self.import_state(start_label, starting_state)
+
         self.optimizer.propagate_all_forward(clear=False)
 
         if not jumpop:
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -638,7 +638,6 @@
             for i in range(op.numargs()):
                 op.setarg(i, self.renamed(op.getarg(i)))
         if box in self.short_boxes:
-            return
             if op is None:
                 oldop = self.short_boxes[box].clone()
                 oldres = oldop.result

From noreply at buildbot.pypy.org  Thu Dec 18 09:16:26 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 18 Dec 2014 09:16:26 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20141218081626.1F0031C03A4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75010:c3be2cabf20d
Date: 2014-12-18 10:15 +0200
http://bitbucket.org/pypy/pypy/changeset/c3be2cabf20d/

Log:	merge

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -483,11 +483,14 @@
         self.emit_operation(op)
 
     def _optimize_CALL_ARRAYCOPY(self, op):
+        length = self.get_constant_box(op.getarg(5))
+        if length and length.getint() == 0:
+            return True # 0-length arraycopy
+
         source_value = self.getvalue(op.getarg(1))
         dest_value = self.getvalue(op.getarg(2))
         source_start_box = self.get_constant_box(op.getarg(3))
         dest_start_box = self.get_constant_box(op.getarg(4))
-        length = self.get_constant_box(op.getarg(5))
         extrainfo = op.getdescr().get_extra_info()
         if (source_start_box and dest_start_box
             and length and (dest_value.is_virtual() or length.getint() <= 8) and
@@ -498,6 +501,8 @@
             dest_start = dest_start_box.getint()
             # XXX fish fish fish
             arraydescr = extrainfo.write_descrs_arrays[0]
+            if arraydescr.is_array_of_structs():
+                return False       # not supported right now
             for index in range(length.getint()):
                 if source_value.is_virtual():
                     assert isinstance(source_value, VArrayValue)
@@ -527,8 +532,6 @@
                                          descr=arraydescr)
                     self.emit_operation(newop)
             return True
-        if length and length.getint() == 0:
-            return True # 0-length arraycopy
         return False
 
     def optimize_CALL_PURE(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -999,6 +999,40 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct_arraycopy(self):
+        ops = """
+        [f0, f1]
+        p0 = new_array_clear(3, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=compleximagdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=complexrealdescr)
+        call(0, p0, p0, 0, 2, 1, descr=complexarraycopydescr)
+        f2 = getinteriorfield_gc(p0, 2, descr=complexrealdescr)
+        f3 = getinteriorfield_gc(p0, 2, descr=compleximagdescr)
+        escape(f2)
+        escape(f3)
+        finish(1)
+        """
+        expected = """
+        [f0, f1]
+        escape(f1)
+        escape(f0)
+        finish(1)
+        """
+        self.optimize_loop(ops, ops)
+        py.test.skip("XXX missing optimization: ll_arraycopy(array-of-structs)")
+
+    def test_nonvirtual_array_of_struct_arraycopy(self):
+        ops = """
+        [p0]
+        call(0, p0, p0, 0, 2, 1, descr=complexarraycopydescr)
+        f2 = getinteriorfield_gc(p0, 2, descr=compleximagdescr)
+        f3 = getinteriorfield_gc(p0, 2, descr=complexrealdescr)
+        escape(f2)
+        escape(f3)
+        finish(1)
+        """
+        self.optimize_loop(ops, ops)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -223,6 +223,10 @@
     complexarraydescr = cpu.arraydescrof(complexarray)
     complexrealdescr = cpu.interiorfielddescrof(complexarray, "real")
     compleximagdescr = cpu.interiorfielddescrof(complexarray, "imag")
+    complexarraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+            EffectInfo([], [complexarraydescr], [], [], [complexarraydescr], [],
+                       EffectInfo.EF_CANNOT_RAISE,
+                       oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     rawarraydescr = cpu.arraydescrof(lltype.Array(lltype.Signed,
                                                   hints={'nolength': True}))

From noreply at buildbot.pypy.org  Thu Dec 18 09:46:24 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 18 Dec 2014 09:46:24 +0100 (CET)
Subject: [pypy-commit] pypy default: someone has to explain to me why this
	is not reasonable
Message-ID: <20141218084624.0890E1C04B4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75011:57d177436b84
Date: 2014-12-18 10:45 +0200
http://bitbucket.org/pypy/pypy/changeset/57d177436b84/

Log:	someone has to explain to me why this is not reasonable

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -107,7 +107,9 @@
             assert other.level == LEVEL_CONSTANT
             assert other.box.same_constant(self.box)
             return
-        assert self.level <= LEVEL_NONNULL
+        if self.level == LEVEL_KNOWNCLASS:
+            return
+        #assert self.level <= LEVEL_NONNULL
         if other.level == LEVEL_CONSTANT:
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -638,6 +638,7 @@
             for i in range(op.numargs()):
                 op.setarg(i, self.renamed(op.getarg(i)))
         if box in self.short_boxes:
+            return
             if op is None:
                 oldop = self.short_boxes[box].clone()
                 oldres = oldop.result

From noreply at buildbot.pypy.org  Thu Dec 18 10:48:53 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Dec 2014 10:48:53 +0100 (CET)
Subject: [pypy-commit] pypy default: Copy better error message from CPython
 2.7 for list.index().
Message-ID: <20141218094853.7A53E1C3288@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75012:c1e5d1cca6aa
Date: 2014-12-18 09:48 +0000
http://bitbucket.org/pypy/pypy/changeset/c1e5d1cca6aa/

Log:	Copy better error message from CPython 2.7 for list.index(). As
	there is no reason not to, also use it for list.remove().

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -618,8 +618,8 @@
         try:
             i = self.find(w_value, 0, sys.maxint)
         except ValueError:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("list.remove(x): x not in list"))
+            raise oefmt(space.w_ValueError,
+                        "list.remove(): %R is not in list", w_value)
         if i < self.length():  # otherwise list was mutated
             self.pop(i)
 
@@ -633,8 +633,7 @@
         try:
             i = self.find(w_value, i, stop)
         except ValueError:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("list.index(x): x not in list"))
+            raise oefmt(space.w_ValueError, "%R is not in list", w_value)
         return space.wrap(i)
 
     @unwrap_spec(reverse=bool)
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -976,7 +976,10 @@
 
         c = [0.0, 2.2, 4.4]
         assert c.index(0) == 0.0
-        raises(ValueError, c.index, 3)
+        e = raises(ValueError, c.index, 3)
+        import sys
+        if sys.version_info[:2] == (2, 7):     # CPython 2.7, PyPy
+            assert str(e.value) == '3 is not in list'
 
     def test_index_cpython_bug(self):
         if self.on_cpython:
@@ -1228,7 +1231,9 @@
         assert l == [0.0, 1.1, 3.3, 4.4]
         l = [0.0, 3.3, 5.5]
         raises(ValueError, c.remove, 2)
-        raises(ValueError, c.remove, 2.2)
+        e = raises(ValueError, c.remove, 2.2)
+        if not self.on_cpython:
+            assert str(e.value) == 'list.remove(): 2.2 is not in list'
 
     def test_reverse(self):
         c = list('hello world')

From noreply at buildbot.pypy.org  Thu Dec 18 13:49:27 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 18 Dec 2014 13:49:27 +0100 (CET)
Subject: [pypy-commit] pypy default: Backed out changeset 57d177436b84
Message-ID: <20141218124927.3B8441D2891@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75013:01fc6eb5e20c
Date: 2014-12-18 14:47 +0200
http://bitbucket.org/pypy/pypy/changeset/01fc6eb5e20c/

Log:	Backed out changeset 57d177436b84

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -107,9 +107,7 @@
             assert other.level == LEVEL_CONSTANT
             assert other.box.same_constant(self.box)
             return
-        if self.level == LEVEL_KNOWNCLASS:
-            return
-        #assert self.level <= LEVEL_NONNULL
+        assert self.level <= LEVEL_NONNULL
         if other.level == LEVEL_CONSTANT:
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -638,7 +638,6 @@
             for i in range(op.numargs()):
                 op.setarg(i, self.renamed(op.getarg(i)))
         if box in self.short_boxes:
-            return
             if op is None:
                 oldop = self.short_boxes[box].clone()
                 oldres = oldop.result

From noreply at buildbot.pypy.org  Thu Dec 18 13:49:28 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 18 Dec 2014 13:49:28 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20141218124928.889A81D2891@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75014:d19489819ae6
Date: 2014-12-18 14:49 +0200
http://bitbucket.org/pypy/pypy/changeset/d19489819ae6/

Log:	merge

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -618,8 +618,8 @@
         try:
             i = self.find(w_value, 0, sys.maxint)
         except ValueError:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("list.remove(x): x not in list"))
+            raise oefmt(space.w_ValueError,
+                        "list.remove(): %R is not in list", w_value)
         if i < self.length():  # otherwise list was mutated
             self.pop(i)
 
@@ -633,8 +633,7 @@
         try:
             i = self.find(w_value, i, stop)
         except ValueError:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("list.index(x): x not in list"))
+            raise oefmt(space.w_ValueError, "%R is not in list", w_value)
         return space.wrap(i)
 
     @unwrap_spec(reverse=bool)
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -976,7 +976,10 @@
 
         c = [0.0, 2.2, 4.4]
         assert c.index(0) == 0.0
-        raises(ValueError, c.index, 3)
+        e = raises(ValueError, c.index, 3)
+        import sys
+        if sys.version_info[:2] == (2, 7):     # CPython 2.7, PyPy
+            assert str(e.value) == '3 is not in list'
 
     def test_index_cpython_bug(self):
         if self.on_cpython:
@@ -1228,7 +1231,9 @@
         assert l == [0.0, 1.1, 3.3, 4.4]
         l = [0.0, 3.3, 5.5]
         raises(ValueError, c.remove, 2)
-        raises(ValueError, c.remove, 2.2)
+        e = raises(ValueError, c.remove, 2.2)
+        if not self.on_cpython:
+            assert str(e.value) == 'list.remove(): 2.2 is not in list'
 
     def test_reverse(self):
         c = list('hello world')

From noreply at buildbot.pypy.org  Thu Dec 18 17:37:51 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Dec 2014 17:37:51 +0100 (CET)
Subject: [pypy-commit] pypy default: (idea by antocuni) Complain clearly
 when trying to import 'readline' on
Message-ID: <20141218163751.DC41B1C04B4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75015:e8b468bcd294
Date: 2014-12-18 16:37 +0000
http://bitbucket.org/pypy/pypy/changeset/e8b468bcd294/

Log:	(idea by antocuni) Complain clearly when trying to import 'readline'
	on Windows.

diff --git a/lib_pypy/readline.py b/lib_pypy/readline.py
--- a/lib_pypy/readline.py
+++ b/lib_pypy/readline.py
@@ -6,4 +6,11 @@
 are only stubs at the moment.
 """
 
-from pyrepl.readline import *
+try:
+    from pyrepl.readline import *
+except ImportError:
+    import sys
+    if sys.platform == 'win32':
+        raise ImportError("the 'readline' module is not available on Windows"
+                          " (on either PyPy or CPython)")
+    raise

From noreply at buildbot.pypy.org  Thu Dec 18 19:01:26 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Thu, 18 Dec 2014 19:01:26 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Implement uint_floordiv and
 uint_rshift; reduce debug spew
Message-ID: <20141218180126.2F4F41D287A@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75016:3e8180a52286
Date: 2014-12-17 12:47 -0500
http://bitbucket.org/pypy/pypy/changeset/3e8180a52286/

Log:	Implement uint_floordiv and uint_rshift; reduce debug spew

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -44,19 +44,22 @@
         if 0:
             self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_TREE,
                                       r_int(1))
-        if 1:
+        if 0:
             self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
                                       r_int(1))
-        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DEBUGINFO,
-                                  r_int(1))
+        if 1:
+            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DEBUGINFO,
+                                      r_int(1))
         if 1:
             self.ctxt.set_int_option(self.lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
                                      r_int(2))
-        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
-                                  r_int(1))
-        self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
-                                  r_int(1))
-        if 1:
+        if 0:
+            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
+                                      r_int(1))
+        if 0:
+            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
+                                      r_int(1))
+        if 0:
             self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
                                       r_int(1))
         
@@ -75,7 +78,7 @@
                       loopname, logger):
         print('assemble_loop')
         clt = CompiledLoopToken(self.cpu, looptoken.number)
-        print(clt)
+        #print(clt)
         looptoken.compiled_loop_token = clt
         clt._debug_nbargs = len(inputargs)
 
@@ -90,6 +93,8 @@
         self.make_context()
         self.t_Signed = self.ctxt.get_int_type(r_int(self.sizeof_signed),
                                                r_int(1))
+        self.t_UINT = self.ctxt.get_int_type(r_int(self.sizeof_signed),
+                                             r_int(0))
         self.t_float = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_DOUBLE) # FIXME                                          
         self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
         self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
@@ -101,9 +106,9 @@
                                                 self.u_float])
         self.lvalue_for_box = {}
 
-        print(jitframe.JITFRAME)
-        print(dir(jitframe.JITFRAME))
-        print('jitframe.JITFRAME._flds: %r' % jitframe.JITFRAME._flds)
+        #print(jitframe.JITFRAME)
+        #print(dir(jitframe.JITFRAME))
+        #print('jitframe.JITFRAME._flds: %r' % jitframe.JITFRAME._flds)
 
         # For now, build a "struct JITFRAME".
         # This will have the fields of jitframe,
@@ -169,7 +174,7 @@
         struct_jit_frame.set_fields (fields)
 
         # Make function:
-        print('  inputargs: %r' % (inputargs, ))
+        #print('  inputargs: %r' % (inputargs, ))
         #jitframe.JITFRAMEINFOPTR
         params = []
 
@@ -179,10 +184,10 @@
         self.param_addr = self.ctxt.new_param(self.t_void_ptr, "addr")
         params.append(self.param_addr)
 
-        print("loopname: %r" % loopname)
         if not loopname:
             loopname = 'anonloop_%i' % self.num_anon_loops
             self.num_anon_loops += 1
+        print("  loopname: %r" % loopname)
         self.fn = self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_EXPORTED,
                                          t_jit_frame_ptr,
                                          loopname,
@@ -196,10 +201,10 @@
         # Add an initial comment summarizing the loop
         text = '\n\tinputargs: %s\n\n' % inputargs
         for op in operations:
-            print(op)
-            print(type(op))
-            print(dir(op))
-            print(repr(op.getopname()))
+            #print(op)
+            #print(type(op))
+            #print(dir(op))
+            #print(repr(op.getopname()))
             text += '\t%s\n' % op
         self.b_current.add_comment(str(text))
     
@@ -223,10 +228,10 @@
                 src_rvalue)
 
         for op in operations:
-            print(op)
-            print(type(op))
-            print(dir(op))
-            print(repr(op.getopname()))
+            #print(op)
+            #print(type(op))
+            #print(dir(op))
+            #print(repr(op.getopname()))
             # Add a comment describing this ResOperation
             self.b_current.add_comment(str(op))
 
@@ -239,7 +244,7 @@
         clt.frame_info.update_frame_depth(baseofs,
                                           max_args)
         
-        self.ctxt.dump_to_file("/tmp/foo.c", r_int(1))
+        self.ctxt.dump_to_file("/tmp/%s.c" % loopname, r_int(1))
 
         #raise foo
 
@@ -448,6 +453,19 @@
                                              rval0, rval1)
         self.b_current.add_assignment(lvalres, binop_expr)
 
+    def _impl_uint_binop(self, resop, gcc_jit_binary_op):
+        rval0 = self.expr_to_rvalue(resop._arg0)
+        rval0 = self.ctxt.new_cast(rval0, self.t_UINT)
+        rval1 = self.expr_to_rvalue(resop._arg1)
+        rval1 = self.ctxt.new_cast(rval1, self.t_UINT)
+        lvalres = self.expr_to_lvalue(resop.result)
+        binop_expr = self.ctxt.new_binary_op(gcc_jit_binary_op,
+                                             self.t_UINT,
+                                             rval0, rval1)
+        self.b_current.add_assignment(lvalres,
+                                      self.ctxt.new_cast(binop_expr,
+                                                         self.t_Signed))
+
     def emit_int_add(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_PLUS)
     def emit_int_sub(self, resop):
@@ -456,6 +474,8 @@
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_MULT)
     def emit_int_floordiv(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_DIVIDE)
+    def emit_uint_floordiv(self, resop):
+        self._impl_uint_binop(resop, self.lib.GCC_JIT_BINARY_OP_DIVIDE)
     def emit_int_mod(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_MODULO)
     def emit_int_and(self, resop):
@@ -468,6 +488,8 @@
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_RSHIFT)
     def emit_int_lshift(self, resop):
         self.impl_int_binop(resop, self.lib.GCC_JIT_BINARY_OP_LSHIFT)
+    def emit_uint_rshift(self, resop):
+        self._impl_uint_binop(resop, self.lib.GCC_JIT_BINARY_OP_RSHIFT)
 
     # "FLOAT" binary ops:
     def impl_float_binop(self, resop, gcc_jit_binary_op):

From noreply at buildbot.pypy.org  Thu Dec 18 19:01:27 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Thu, 18 Dec 2014 19:01:27 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Clarify _impl_guard;
 start adding boilerplate for bridges
Message-ID: <20141218180127.8E2B41D287A@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75017:a1db64631c41
Date: 2014-12-17 13:20 -0500
http://bitbucket.org/pypy/pypy/changeset/a1db64631c41/

Log:	Clarify _impl_guard; start adding boilerplate for bridges

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -259,6 +259,15 @@
 
         # FIXME: this leaks the gcc_jit_result
 
+    def assemble_bridge(self, logger, faildescr, inputargs, operations,
+                        original_loop_token, log):
+        print('assemble_bridge(%r)' % locals())
+        if not we_are_translated():
+            # Arguments should be unique
+            assert len(set(inputargs)) == len(inputargs)
+
+        raise foo
+
     def expr_to_rvalue(self, expr):
         """
         print('expr_to_rvalue')
@@ -391,9 +400,14 @@
                                             b_true, b_false)
 
         if istrue:
-            self.b_current = b_false
+            b_guard_failure = b_false
+            b_guard_success = b_true
         else:
-            self.b_current = b_true
+            b_guard_failure = b_true
+            b_guard_success = b_false
+
+        # Write out guard failure impl:
+        self.b_current = b_guard_failure
         self._impl_write_output_args(resop._fail_args)
         self._impl_write_jf_descr(resop)
         self.b_current.end_with_return(self.param_frame.as_rvalue ())
@@ -402,10 +416,8 @@
             rd_locs.append(idx * self.sizeof_signed)
         resop.getdescr().rd_locs = rd_locs
 
-        if istrue:
-            self.b_current = b_true
-        else:
-            self.b_current = b_false
+        # Further operations go into the guard success block:
+        self.b_current = b_guard_success
 
     def emit_guard_true(self, resop):
         self._impl_guard(resop, r_int(1))
diff --git a/rpython/jit/backend/libgccjit/runner.py b/rpython/jit/backend/libgccjit/runner.py
--- a/rpython/jit/backend/libgccjit/runner.py
+++ b/rpython/jit/backend/libgccjit/runner.py
@@ -17,20 +17,13 @@
 
     def compile_loop(self, inputargs, operations, looptoken,
                      log=True, name='', logger=None):
-        """
-        import sys
-        sys.stderr.write('compile_loop:\n')
-        for i, arg in enumerate(inputargs):
-            sys.stderr.write('  arg[%i] = %r\n' % (i, arg))
-            sys.stderr.write('    type(arg[%i]) = %r\n' % (i, type(arg)))
-        for i, op in enumerate(operations):
-            sys.stderr.write('  op[%i] = %r\n' % (i, op))
-            sys.stderr.write('    type(op[%i]) = %r\n' % (i, type(op)))
-        sys.stderr.write('  looptoken: %r\n' % (looptoken, ))
-        sys.stderr.write('  log: %r\n' % (log, ))
-        sys.stderr.write('  name: %r\n' % (name, ))
-        sys.stderr.write('  logger: %r\n' % (logger, ))
-        sys.stderr.write('compile_loop: %r\n' % locals ())
-        """
         return self.assembler.assemble_loop(inputargs, operations, looptoken, log,
                                             name, logger)
+
+    def compile_bridge(self, faildescr, inputargs, operations,
+                       original_loop_token, log=True, logger=None):
+        clt = original_loop_token.compiled_loop_token
+        clt.compiling_a_bridge()
+        return self.assembler.assemble_bridge(logger, faildescr, inputargs,
+                                              operations,
+                                              original_loop_token, log=log)

From noreply at buildbot.pypy.org  Thu Dec 18 19:01:28 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Thu, 18 Dec 2014 19:01:28 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Split out guard failure into
 a tail call to another function
Message-ID: <20141218180128.C76D11D287A@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75018:eb3ba1b9910e
Date: 2014-12-17 15:43 -0500
http://bitbucket.org/pypy/pypy/changeset/eb3ba1b9910e/

Log:	Split out guard failure into a tail call to another function

	Eventually this could be a jump through a function pointer.

	Right now it's simply a call to a internal function. The call gets
	inlined away, back into the code we had before.

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -10,6 +10,17 @@
 from rpython.rtyper.lltypesystem.rffi import *
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
 
+class Params:
+    def __init__(self, assembler):
+        self.paramlist = []
+        self.param_frame = assembler.ctxt.new_param(assembler.t_jit_frame_ptr,
+                                                    "jitframe")
+        self.paramlist.append(self.param_frame)
+
+        self.param_addr = assembler.ctxt.new_param(assembler.t_void_ptr,
+                                                   "addr")
+        self.paramlist.append(self.param_addr)
+
 class AssemblerLibgccjit(BaseAssembler):
     _regalloc = None
     #_output_loop_log = None
@@ -34,6 +45,7 @@
         #self.teardown()
 
         self.num_anon_loops = 0
+        self.num_guard_failure_fns = 0
 
         self.sizeof_signed = rffi.sizeof(lltype.Signed)
 
@@ -98,6 +110,7 @@
         self.t_float = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_DOUBLE) # FIXME                                          
         self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
         self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
+        self.t_void = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID)
 
         self.u_signed = self.ctxt.new_field(self.t_Signed, "u_signed")
         self.u_float = self.ctxt.new_field(self.t_float, "u_float")
@@ -134,7 +147,7 @@
 
         struct_jit_frame = self.ctxt.new_opaque_struct ("JITFRAME")
 
-        t_jit_frame_ptr = struct_jit_frame.as_type().get_pointer()
+        self.t_jit_frame_ptr = struct_jit_frame.as_type().get_pointer()
 
         fields = []
         # FIXME: Does the GCStruct implicitly add any fields?
@@ -149,7 +162,7 @@
         make_field('jf_extra_stack_depth', self.t_Signed)
         make_field('jf_savedata', self.t_void_ptr)
         make_field('jf_guard_exc', self.t_void_ptr)
-        make_field('jf_forward', t_jit_frame_ptr)
+        make_field('jf_forward', self.t_jit_frame_ptr)
         # FIXME: for some reason there's an implicit word here;
         # create it
         make_field('jf_frame', self.t_Signed)
@@ -176,22 +189,16 @@
         # Make function:
         #print('  inputargs: %r' % (inputargs, ))
         #jitframe.JITFRAMEINFOPTR
-        params = []
-
-        self.param_frame = self.ctxt.new_param(t_jit_frame_ptr, "jitframe")
-        params.append(self.param_frame)
-
-        self.param_addr = self.ctxt.new_param(self.t_void_ptr, "addr")
-        params.append(self.param_addr)
+        self.loop_params = Params(self)
 
         if not loopname:
             loopname = 'anonloop_%i' % self.num_anon_loops
             self.num_anon_loops += 1
         print("  loopname: %r" % loopname)
         self.fn = self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_EXPORTED,
-                                         t_jit_frame_ptr,
+                                         self.t_jit_frame_ptr,
                                          loopname,
-                                         params,
+                                         self.loop_params.paramlist,
                                          r_int(0))
 
         self.b_current = self.fn.new_block("initial")
@@ -300,7 +307,7 @@
         return self.lvalue_for_box[box]
 
     def get_arg_as_lvalue(self, idx):
-        return self.param_frame.as_rvalue ().dereference_field (
+        return self.loop_params.param_frame.as_rvalue ().dereference_field (
             self.field_for_arg_idx[idx])
 
     def expr_to_lvalue(self, expr):
@@ -371,9 +378,9 @@
         self.b_current.end_with_jump(self.block_for_label_descr[jumpop.getdescr()])
 
     def emit_finish(self, resop):
-        self._impl_write_output_args(resop._args)
-        self._impl_write_jf_descr(resop)
-        self.b_current.end_with_return(self.param_frame.as_rvalue ())
+        self._impl_write_output_args(self.loop_params, resop._args)
+        self._impl_write_jf_descr(self.loop_params, resop)
+        self.b_current.end_with_return(self.loop_params.param_frame.as_rvalue ())
 
     def emit_label(self, resop):
         print(resop)
@@ -408,15 +415,34 @@
 
         # Write out guard failure impl:
         self.b_current = b_guard_failure
-        self._impl_write_output_args(resop._fail_args)
-        self._impl_write_jf_descr(resop)
-        self.b_current.end_with_return(self.param_frame.as_rvalue ())
+
+        # Implement it as a tail-call to a handler function
+        # This will eventually become a function ptr, allowing
+        # patchability
+        failure_params = Params(self)
+        failure_fn = (
+            self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_INTERNAL,
+                                   self.t_jit_frame_ptr,
+                                   "on_guard_failure_%i" % self.num_guard_failure_fns,
+                                   failure_params.paramlist,
+                                   r_int(0)))
+        self.num_guard_failure_fns += 1
+        call = self.ctxt.new_call (failure_fn,
+                                   [param.as_rvalue()
+                                    for param in self.loop_params.paramlist])
+        self._impl_write_output_args(self.loop_params, resop._fail_args)
+        self.b_current.end_with_return(call)
+
+        b_within_failure_fn = failure_fn.new_block("initial")
+        self.b_current = b_within_failure_fn
+        self._impl_write_jf_descr(failure_params, resop)
+        self.b_current.end_with_return(failure_params.param_frame.as_rvalue ())
         rd_locs = []
         for idx, arg in enumerate(resop._fail_args):
             rd_locs.append(idx * self.sizeof_signed)
         resop.getdescr().rd_locs = rd_locs
 
-        # Further operations go into the guard success block:
+        # Further operations go into the guard success block in the original fn:
         self.b_current = b_guard_success
 
     def emit_guard_true(self, resop):
@@ -425,7 +451,7 @@
     def emit_guard_false(self, resop):
         self._impl_guard(resop, r_int(0))
 
-    def _impl_write_output_args(self, args):
+    def _impl_write_output_args(self, params, args):
         # Write outputs back:
         for idx, arg in enumerate(args):
             if arg is not None:
@@ -443,14 +469,14 @@
                                                    r_int(0)))
                 """
 
-    def _impl_write_jf_descr(self, resop):
+    def _impl_write_jf_descr(self, params, resop):
         # Write back to the jf_descr:
         #  "jitframe->jf_descr = resop.getdescr();"
         descr = rffi.cast(lltype.Signed,
                           cast_instance_to_gcref(resop.getdescr()))
 
         self.b_current.add_assignment(
-            self.param_frame.as_rvalue ().dereference_field (
+            params.param_frame.as_rvalue ().dereference_field (
                 self.field_jf_descr),
             self.ctxt.new_rvalue_from_ptr (self.t_void_ptr,
                                            rffi.cast(VOIDP, descr)))
@@ -603,4 +629,3 @@
         self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_GT)
     def emit_float_ge(self, resop):
         self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
-
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -87,6 +87,8 @@
                                                  hints={'nolength': True}))
         self.PARAM_P_P = lltype.Ptr(lltype.Array(self.GCC_JIT_PARAM_P,
                                                  hints={'nolength': True}))
+        self.RVALUE_P_P = lltype.Ptr(lltype.Array(self.GCC_JIT_RVALUE_P,
+                                                  hints={'nolength': True}))
 
         # Entrypoints:
         for returntype, name, paramtypes in [
@@ -253,6 +255,13 @@
                                                     self.GCC_JIT_RVALUE_P]),
 
                 (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_new_call', [self.GCC_JIT_CONTEXT_P,
+                                              self.GCC_JIT_LOCATION_P,
+                                              self.GCC_JIT_FUNCTION_P,
+                                              INT,
+                                              self.RVALUE_P_P]),
+
+                (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_context_new_cast', [self.GCC_JIT_CONTEXT_P,
                                               self.GCC_JIT_LOCATION_P,
                                               self.GCC_JIT_RVALUE_P,
@@ -540,6 +549,20 @@
                                                               op,
                                                               a.inner_rvalue, b.inner_rvalue))
 
+    def new_call(self, fn, args):
+        raw_arg_array = lltype.malloc(self.lib.RVALUE_P_P.TO,
+                                      len(args),
+                                      flavor='raw') # of maybe gc?
+        for i in range(len(args)):
+            raw_arg_array[i] = args[i].inner_rvalue
+        rvalue = self.lib.gcc_jit_context_new_call(self.inner_ctxt,
+                                                   self.lib.null_location_ptr,
+                                                   fn.inner_function,
+                                                   r_int(len(args)),
+                                                   raw_arg_array)
+        lltype.free(raw_arg_array, flavor='raw')
+        return RValue(self.lib, rvalue)
+
     def new_param(self, type_, name):
         name_charp = str2charp(name)
         param = self.lib.gcc_jit_context_new_param(self.inner_ctxt,

From noreply at buildbot.pypy.org  Thu Dec 18 19:01:30 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Thu, 18 Dec 2014 19:01:30 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Begin implementing code
	patching
Message-ID: <20141218180130.02B101D287A@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75019:ee6c48dc603d
Date: 2014-12-18 12:10 -0500
http://bitbucket.org/pypy/pypy/changeset/ee6c48dc603d/

Log:	Begin implementing code patching

	When generating guard failure handling, replace direct call to
	default handler with a jump through a function pointer, so that (in
	theory) we can write to this function pointer to use a different
	handler.

	This requires a couple of extra changes to libgccjit that are
	currently only in my local repo:

	 * a new API entrypoint "gcc_jit_result_get_global"
	   * an extra param to gcc_jit_context_new_global (enum
	gcc_jit_global_kind)

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -20,6 +20,93 @@
         self.param_addr = assembler.ctxt.new_param(assembler.t_void_ptr,
                                                    "addr")
         self.paramlist.append(self.param_addr)
+        self.paramtypes = [assembler.t_jit_frame_ptr,
+                           assembler.t_void_ptr]
+
+class Patchpoint:
+    """
+    We need to be able to patch out the generated code that runs when a
+    guard fails; this class handles that.
+
+    We handle guard failure with a tail-call through a function pointer,
+    equivalent to this C code:
+
+      struct JITFRAME * (*guard_failure_fn_ptr_0) (struct JITFRAME *, void *);
+
+      extern struct JITFRAME *
+      anonloop_0 (struct JITFRAME * jitframe, void * addr)
+      {
+        ...various operations...
+
+        if (!guard)
+         goto on_guard_failure;
+
+        ...various operations...
+
+       on_guard_failure:
+          return guard_failure_fn_ptr_0 (frame, );
+      }
+
+    This can hopefully be optimized to a jump through a ptr, since it's
+    a tail call:
+
+      0x00007fffeb7086d0 <+16>:  jle    0x7fffeb7086c8 <anonloop_0+8>
+      0x00007fffeb7086d2 <+18>:  mov    %rax,0x48(%rdi)
+      0x00007fffeb7086d6 <+22>:  mov    0x2008fb(%rip),%rax        # 0x7fffeb908fd8
+      0x00007fffeb7086dd <+29>:  mov    (%rax),%rax
+      0x00007fffeb7086e0 <+32>:  jmpq   *%rax
+    """
+    def __init__(self, assembler):
+        self.failure_params = Params(assembler)
+        self.serial = assembler.num_guard_failure_fns
+        assembler.num_guard_failure_fns += 1
+        self.t_fn_ptr_type = (
+            assembler.ctxt.new_function_ptr_type (assembler.t_jit_frame_ptr,
+                                                  self.failure_params.paramtypes,
+                                                  r_int(0)))
+        # Create the function ptr
+        # Globals are zero-initialized, so we'll need to
+        # write in the ptr to the initial handler before the loop is called,
+        # or we'll have a jump-through-NULL segfault.
+        self.fn_ptr_name = "guard_failure_fn_ptr_%i" % self.serial
+        self.failure_fn_ptr = (
+            assembler.ctxt.new_global(assembler.lib.GCC_JIT_GLOBAL_EXPORTED,
+                                      self.t_fn_ptr_type,
+                                      self.fn_ptr_name))
+        self.handler_name = "on_guard_failure_%i" % self.serial
+        self.failure_fn = (
+            assembler.ctxt.new_function(assembler.lib.GCC_JIT_FUNCTION_EXPORTED,
+                                        assembler.t_jit_frame_ptr,
+                                        self.handler_name,
+                                        self.failure_params.paramlist,
+                                        r_int(0)))
+
+    def write_initial_handler(self, result):
+        # Get the address of the machine code for the handler;
+        # this is a:
+        #   struct JITFRAME * (*guard_failure_fn) (struct JITFRAME *, void *)
+        handler = result.get_code(self.handler_name)
+
+        # Get the address of the function ptr to be written to;
+        # this is a:
+        #   struct JITFRAME * (**guard_failure_fn) (struct JITFRAME *, void *)
+        # i.e. one extra level of indirection.
+        fn_ptr_ptr = result.get_global(self.fn_ptr_name)
+
+        # We want to write the equivalent of:
+        #    (*fn_ptr_ptr) = handler;
+        #
+        # "fn_ptr_ptr" and "handler" are both currently (void *).
+        # so we need to cast them to a form where we can express
+        # the above.
+
+        # We can't directly express the function ptr ptr in lltype,
+        # so instead pretend we have a (const char **) and a (const char *):
+        fn_ptr_ptr = rffi.cast(rffi.CCHARPP, fn_ptr_ptr)
+        handler = rffi.cast(rffi.CCHARP, handler)
+
+        # ...and write through the ptr:
+        fn_ptr_ptr[0] = handler
 
 class AssemblerLibgccjit(BaseAssembler):
     _regalloc = None
@@ -48,6 +135,7 @@
         self.num_guard_failure_fns = 0
 
         self.sizeof_signed = rffi.sizeof(lltype.Signed)
+        self.patchpoints = []
 
     def make_context(self):
         eci = make_eci()
@@ -65,10 +153,10 @@
         if 1:
             self.ctxt.set_int_option(self.lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
                                      r_int(2))
-        if 0:
+        if 1:
             self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
                                       r_int(1))
-        if 0:
+        if 1:
             self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
                                       r_int(1))
         if 0:
@@ -258,6 +346,10 @@
         jit_result = self.ctxt.compile()
         self.ctxt.release()
 
+        # Patch all patchpoints to their initial handlers:
+        for pp in self.patchpoints:
+            pp.write_initial_handler(jit_result)
+
         fn_ptr = jit_result.get_code(loopname)
 
         looptoken._ll_function_addr = fn_ptr
@@ -416,27 +508,21 @@
         # Write out guard failure impl:
         self.b_current = b_guard_failure
 
-        # Implement it as a tail-call to a handler function
-        # This will eventually become a function ptr, allowing
-        # patchability
-        failure_params = Params(self)
-        failure_fn = (
-            self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_INTERNAL,
-                                   self.t_jit_frame_ptr,
-                                   "on_guard_failure_%i" % self.num_guard_failure_fns,
-                                   failure_params.paramlist,
-                                   r_int(0)))
-        self.num_guard_failure_fns += 1
-        call = self.ctxt.new_call (failure_fn,
-                                   [param.as_rvalue()
-                                    for param in self.loop_params.paramlist])
+        # Implement it as a tail-call through a function ptr to
+        # a handler function, allowing patchability
+        pp = Patchpoint(self)
+        self.patchpoints.append(pp)
+        args = [param.as_rvalue()
+                for param in self.loop_params.paramlist]
+        call = self.ctxt.new_call_through_ptr (pp.failure_fn_ptr.as_rvalue(),
+                                               args)
         self._impl_write_output_args(self.loop_params, resop._fail_args)
         self.b_current.end_with_return(call)
 
-        b_within_failure_fn = failure_fn.new_block("initial")
+        b_within_failure_fn = pp.failure_fn.new_block("initial")
         self.b_current = b_within_failure_fn
-        self._impl_write_jf_descr(failure_params, resop)
-        self.b_current.end_with_return(failure_params.param_frame.as_rvalue ())
+        self._impl_write_jf_descr(pp.failure_params, resop)
+        self.b_current.end_with_return(pp.failure_params.param_frame.as_rvalue ())
         rd_locs = []
         for idx, arg in enumerate(resop._fail_args):
             rd_locs.append(idx * self.sizeof_signed)
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -89,6 +89,8 @@
                                                  hints={'nolength': True}))
         self.RVALUE_P_P = lltype.Ptr(lltype.Array(self.GCC_JIT_RVALUE_P,
                                                   hints={'nolength': True}))
+        self.TYPE_P_P = lltype.Ptr(lltype.Array(self.GCC_JIT_TYPE_P,
+                                                hints={'nolength': True}))
 
         # Entrypoints:
         for returntype, name, paramtypes in [
@@ -120,6 +122,10 @@
                  'gcc_jit_result_get_code', [self.GCC_JIT_RESULT_P,
                                              CCHARP]),
 
+                (VOIDP,
+                 'gcc_jit_result_get_global', [self.GCC_JIT_RESULT_P,
+                                               CCHARP]),
+
                 (lltype.Void,
                  'gcc_jit_result_release', [self.GCC_JIT_RESULT_P]),
 
@@ -170,6 +176,14 @@
                                                     INT,
                                                     self.FIELD_P_P]),
 
+                (self.GCC_JIT_TYPE_P,
+                 'gcc_jit_context_new_function_ptr_type', [self.GCC_JIT_CONTEXT_P,
+                                                           self.GCC_JIT_LOCATION_P,
+                                                           self.GCC_JIT_TYPE_P,
+                                                           INT,
+                                                           self.TYPE_P_P,
+                                                           INT]),
+
                 ############################################################
                 # Constructing functions.
                 ############################################################
@@ -205,6 +219,13 @@
                 ############################################################
                 # lvalues, rvalues and expressions.
                 ############################################################
+                (self.GCC_JIT_LVALUE_P,
+                 'gcc_jit_context_new_global', [self.GCC_JIT_CONTEXT_P,
+                                                self.GCC_JIT_LOCATION_P,
+                                                INT, # enum gcc_jit_global_kind
+                                                self.GCC_JIT_TYPE_P,
+                                                CCHARP]),
+
                 (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_lvalue_as_rvalue', [self.GCC_JIT_LVALUE_P]),
 
@@ -262,6 +283,13 @@
                                               self.RVALUE_P_P]),
 
                 (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_new_call_through_ptr',[self.GCC_JIT_CONTEXT_P,
+                                                         self.GCC_JIT_LOCATION_P,
+                                                         self.GCC_JIT_RVALUE_P,
+                                                         INT,
+                                                         self.RVALUE_P_P]),
+
+                (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_context_new_cast', [self.GCC_JIT_CONTEXT_P,
                                               self.GCC_JIT_LOCATION_P,
                                               self.GCC_JIT_RVALUE_P,
@@ -352,6 +380,13 @@
 
         self.make_enum_values(
             """
+            GCC_JIT_GLOBAL_EXPORTED,
+            GCC_JIT_GLOBAL_INTERNAL,
+            GCC_JIT_GLOBAL_IMPORTED
+            """)
+
+        self.make_enum_values(
+            """
             GCC_JIT_UNARY_OP_MINUS,
             GCC_JIT_UNARY_OP_BITWISE_NEGATE,
             GCC_JIT_UNARY_OP_LOGICAL_NEGATE,
@@ -502,6 +537,23 @@
         free_charp(name_charp)
         return Type(self.lib, inner_type)
 
+    def new_function_ptr_type(self, returntype, param_types, is_variadic):
+        raw_type_array = lltype.malloc(self.lib.TYPE_P_P.TO,
+                                       len(param_types),
+                                       flavor='raw') # of maybe gc?
+        for i in range(len(param_types)):
+            raw_type_array[i] = param_types[i].inner_type
+
+        type_ = self.lib.gcc_jit_context_new_function_ptr_type(self.inner_ctxt,
+                                                               self.lib.null_location_ptr,
+                                                               returntype.inner_type,
+                                                               r_int(len(param_types)),
+                                                               raw_type_array,
+                                                               is_variadic)
+        lltype.free(raw_type_array, flavor='raw')
+
+        return Type(self.lib, type_)
+
     def new_rvalue_from_int(self, type_, llvalue):
         return RValue(self.lib,
                       self.lib.gcc_jit_context_new_rvalue_from_int(self.inner_ctxt,
@@ -563,6 +615,20 @@
         lltype.free(raw_arg_array, flavor='raw')
         return RValue(self.lib, rvalue)
 
+    def new_call_through_ptr(self, fn_ptr, args):
+        raw_arg_array = lltype.malloc(self.lib.RVALUE_P_P.TO,
+                                      len(args),
+                                      flavor='raw') # of maybe gc?
+        for i in range(len(args)):
+            raw_arg_array[i] = args[i].inner_rvalue
+        rvalue = self.lib.gcc_jit_context_new_call_through_ptr(self.inner_ctxt,
+                                                               self.lib.null_location_ptr,
+                                                               fn_ptr.inner_rvalue,
+                                                               r_int(len(args)),
+                                                               raw_arg_array)
+        lltype.free(raw_arg_array, flavor='raw')
+        return RValue(self.lib, rvalue)
+
     def new_param(self, type_, name):
         name_charp = str2charp(name)
         param = self.lib.gcc_jit_context_new_param(self.inner_ctxt,
@@ -593,6 +659,16 @@
 
         return Function(self.lib, fn)
 
+    def new_global(self, kind, type_, name):
+        name_charp = str2charp(name)
+        lvalue = self.lib.gcc_jit_context_new_global(self.inner_ctxt,
+                                                     self.lib.null_location_ptr,
+                                                     kind,
+                                                     type_.inner_type,
+                                                     name_charp)
+        free_charp(name_charp)
+        return LValue(self.lib, lvalue)
+
     def new_cast(self, rvalue, type_):
         return RValue(self.lib,
                       self.lib.gcc_jit_context_new_cast(self.inner_ctxt,
@@ -743,5 +819,12 @@
         free_charp(name_charp)
         return fn_ptr
 
+    def get_global(self, name):
+        name_charp = str2charp(name)
+        sym_ptr = self.lib.gcc_jit_result_get_global(self.inner_result,
+                                                     name_charp)
+        free_charp(name_charp)
+        return sym_ptr
+
     def release(self):
         self.lib.gcc_jit_result_release(self.inner_result)

From noreply at buildbot.pypy.org  Thu Dec 18 19:01:31 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Thu, 18 Dec 2014 19:01:31 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Document the changes to
 libgccjit that I need to merge
Message-ID: <20141218180131.308BB1D287A@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75020:5d890f2ddecd
Date: 2014-12-18 12:17 -0500
http://bitbucket.org/pypy/pypy/changeset/5d890f2ddecd/

Log:	Document the changes to libgccjit that I need to merge

diff --git a/rpython/jit/backend/libgccjit/notes.rst b/rpython/jit/backend/libgccjit/notes.rst
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/libgccjit/notes.rst
@@ -0,0 +1,16 @@
+This is an experiment, and merely a work-in-progress.
+
+In particular, this needs some changes to libgccjit that are currently
+only in my local repo:
+
+   * new API entrypoints:
+
+     * :c:func:`gcc_jit_result_get_global`
+
+     * :c:func:`gcc_jit_context_new_rvalue_from_long`
+
+   * a new value :c:macro:`GCC_JIT_UNARY_OP_ABS` within
+     :c:type:`enum gcc_jit_unary_op`.
+
+   * an extra param to :c:func:`gcc_jit_context_new_global`
+     (enum gcc_jit_global_kind).

From noreply at buildbot.pypy.org  Thu Dec 18 22:38:41 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Dec 2014 22:38:41 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Add Function.__qualname__, basic form.
Message-ID: <20141218213841.631041C03A4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75021:a81d1ddf3dfe
Date: 2014-12-18 22:36 +0100
http://bitbucket.org/pypy/pypy/changeset/a81d1ddf3dfe/

Log:	Add Function.__qualname__, basic form.

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -39,6 +39,7 @@
                  closure=None, w_ann=None, forcename=None):
         self.space = space
         self.name = forcename or code.co_name
+        self.qualname = self.name.decode('utf-8')  # So far
         self.w_doc = None   # lazily read from code.getdocstring()
         self.code = code       # Code instance
         self.w_func_globals = w_globals  # the globals dictionary
@@ -400,7 +401,20 @@
         except OperationError, e:
             if e.match(space, space.w_TypeError):
                 raise OperationError(space.w_TypeError,
-                                     space.wrap("func_name must be set "
+                                     space.wrap("__name__ must be set "
+                                                "to a string object"))
+            raise
+
+    def fget_func_qualname(self, space):
+        return space.wrap(self.qualname)
+
+    def fset_func_qualname(self, space, w_name):
+        try:
+            self.qualname = space.unicode_w(w_name)
+        except OperationError, e:
+            if e.match(space, space.w_TypeError):
+                raise OperationError(space.w_TypeError,
+                                     space.wrap("__qualname__ must be set "
                                                 "to a string object"))
             raise
 
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -21,6 +21,13 @@
         assert f.__name__ == 'f'
         assert f.__module__ == 'mymodulename'
 
+    def test_qualname(self):
+        def f(): pass
+        assert f.__qualname__ == 'f'
+        f.__qualname__ = 'qualname'
+        assert f.__qualname__ == 'qualname'
+        raises(TypeError, "f.__qualname__ = b'name'")
+
     def test_annotations(self):
         def f(): pass
         ann = f.__annotations__
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -814,6 +814,8 @@
                                   Function.fset_func_code)
 getset_func_name = GetSetProperty(Function.fget_func_name,
                                   Function.fset_func_name)
+getset_func_qualname = GetSetProperty(Function.fget_func_qualname,
+                                      Function.fset_func_qualname)
 getset_func_annotations = GetSetProperty(Function.fget_func_annotations,
                                         Function.fset_func_annotations,
                                         Function.fdel_func_annotations)
@@ -831,6 +833,7 @@
     __code__ = getset_func_code,
     __doc__ = getset_func_doc,
     __name__ = getset_func_name,
+    __qualname__ = getset_func_qualname,
     __dict__ = getset_func_dict,
     __defaults__ = getset_func_defaults,
     __kwdefaults__ = getset_func_kwdefaults,

From noreply at buildbot.pypy.org  Thu Dec 18 22:38:43 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Dec 2014 22:38:43 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Really implement __qualname__ for nested
	blocks.
Message-ID: <20141218213843.368AF1C03A4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75022:9ebba4eb7768
Date: 2014-12-18 22:36 +0100
http://bitbucket.org/pypy/pypy/changeset/9ebba4eb7768/

Log:	Really implement __qualname__ for nested blocks.

diff --git a/lib-python/3/importlib/_bootstrap.py b/lib-python/3/importlib/_bootstrap.py
--- a/lib-python/3/importlib/_bootstrap.py
+++ b/lib-python/3/importlib/_bootstrap.py
@@ -405,8 +405,10 @@
 due to the addition of new opcodes).
 
 """
-_RAW_MAGIC_NUMBER = 3230 | ord('\r') << 16 | ord('\n') << 24
-_MAGIC_BYTES = bytes(_RAW_MAGIC_NUMBER >> n & 0xff for n in range(0, 25, 8))
+
+# PyPy change
+import _imp
+_MAGIC_BYTES = _imp.get_magic()
 
 _PYCACHE = '__pycache__'
 
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -157,6 +157,7 @@
     def __init__(self, space, name, first_lineno, scope, compile_info):
         self.space = space
         self.name = name
+        self.qualname = name
         self.first_lineno = first_lineno
         self.compile_info = compile_info
         self.first_block = self.new_block()
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -205,7 +205,8 @@
         """Convenience function for compiling a sub scope."""
         generator = kind(self.space, name, node, lineno, self.symbols,
                          self.compile_info)
-        return generator.assemble()
+        generator.qualname = '%s.%s' % (self.qualname, name)
+        return generator.assemble(), generator.qualname
 
     def push_frame_block(self, kind, block):
         self.frame_blocks.append((kind, block))
@@ -282,9 +283,11 @@
         self.add_none_to_final_return = False
         mod.body.walkabout(self)
 
-    def _make_function(self, code, num_defaults=0):
+    def _make_function(self, code, num_defaults=0, qualname=None):
         """Emit the opcodes to turn a code object into a function."""
         code_index = self.add_const(code)
+        w_qualname = self.space.wrap(qualname or code.co_name)
+        qualname_index = self.add_const(w_qualname)
         if code.co_freevars:
             # Load cell and free vars to pass on.
             for free in code.co_freevars:
@@ -296,9 +299,11 @@
                 self.emit_op_arg(ops.LOAD_CLOSURE, index)
             self.emit_op_arg(ops.BUILD_TUPLE, len(code.co_freevars))
             self.emit_op_arg(ops.LOAD_CONST, code_index)
+            self.emit_op_arg(ops.LOAD_CONST, qualname_index)
             self.emit_op_arg(ops.MAKE_CLOSURE, num_defaults)
         else:
             self.emit_op_arg(ops.LOAD_CONST, code_index)
+            self.emit_op_arg(ops.LOAD_CONST, qualname_index)
             self.emit_op_arg(ops.MAKE_FUNCTION, num_defaults)
 
     def _visit_kwonlydefaults(self, args):
@@ -359,9 +364,9 @@
         oparg = num_defaults
         oparg |= kw_default_count << 8
         oparg |= num_annotations << 16
-        code = self.sub_scope(FunctionCodeGenerator, func.name, func,
-                              func.lineno)
-        self._make_function(code, oparg)
+        code, qualname = self.sub_scope(FunctionCodeGenerator, func.name, func,
+                                        func.lineno)
+        self._make_function(code, oparg, qualname=qualname)
         # Apply decorators.
         if func.decorator_list:
             for i in range(len(func.decorator_list)):
@@ -377,20 +382,22 @@
             kw_default_count = self._visit_kwonlydefaults(args)
         self.visit_sequence(args.defaults)
         default_count = len(args.defaults) if args.defaults is not None else 0
-        code = self.sub_scope(LambdaCodeGenerator, "<lambda>", lam, lam.lineno)
+        code, qualname = self.sub_scope(
+            LambdaCodeGenerator, "<lambda>", lam, lam.lineno)
         oparg = default_count
         oparg |= kw_default_count << 8
-        self._make_function(code, oparg)
+        self._make_function(code, oparg, qualname=qualname)
 
     def visit_ClassDef(self, cls):
         self.update_position(cls.lineno, True)
         self.visit_sequence(cls.decorator_list)
         # 1. compile the class body into a code object
-        code = self.sub_scope(ClassCodeGenerator, cls.name, cls, cls.lineno)
+        code, qualname = self.sub_scope(
+            ClassCodeGenerator, cls.name, cls, cls.lineno)
         # 2. load the 'build_class' function
         self.emit_op(ops.LOAD_BUILD_CLASS)
         # 3. load a function (or closure) made from the code object
-        self._make_function(code, 0)
+        self._make_function(code, qualname=qualname)
         # 4. load class name
         self.load_const(self.space.wrap(cls.name.decode('utf-8')))
         # 5. generate the rest of the code for the call
@@ -1140,9 +1147,9 @@
         self.use_next_block(anchor)
 
     def _compile_comprehension(self, node, name, sub_scope):
-        code = self.sub_scope(sub_scope, name, node, node.lineno)
+        code, qualname = self.sub_scope(sub_scope, name, node, node.lineno)
         self.update_position(node.lineno)
-        self._make_function(code)
+        self._make_function(code, qualname=qualname)
         first_comp = node.get_generators()[0]
         assert isinstance(first_comp, ast.comprehension)
         first_comp.iter.walkabout(self)
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -36,10 +36,10 @@
                           'w_kw_defs?']
 
     def __init__(self, space, code, w_globals=None, defs_w=[], w_kw_defs=None,
-                 closure=None, w_ann=None, forcename=None):
+                 closure=None, w_ann=None, forcename=None, qualname=None):
         self.space = space
         self.name = forcename or code.co_name
-        self.qualname = self.name.decode('utf-8')  # So far
+        self.qualname = qualname or self.name.decode('utf-8')
         self.w_doc = None   # lazily read from code.getdocstring()
         self.code = code       # Code instance
         self.w_func_globals = w_globals  # the globals dictionary
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -36,7 +36,7 @@
 # different value for the highest 16 bits. Bump pypy_incremental_magic every
 # time you make pyc files incompatible
 
-pypy_incremental_magic = 48 # bump it by 16
+pypy_incremental_magic = 64 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1215,6 +1215,8 @@
     @jit.unroll_safe
     def _make_function(self, oparg, freevars=None):
         space = self.space
+        w_qualname = self.popvalue()
+        qualname = self.space.unicode_w(w_qualname)
         w_codeobj = self.popvalue()
         codeobj = self.space.interp_w(PyCode, w_codeobj)
         if freevars is not None:
@@ -1238,7 +1240,7 @@
                 w_def = self.popvalue()
                 space.setitem(w_kw_defs, w_def, w_name)
         fn = function.Function(space, codeobj, self.w_globals, defaultarguments,
-                               w_kw_defs, freevars, w_ann)
+                               w_kw_defs, freevars, w_ann, qualname=qualname)
         self.pushvalue(space.wrap(fn))
 
     def MAKE_FUNCTION(self, oparg, next_instr):
@@ -1246,7 +1248,7 @@
 
     @jit.unroll_safe
     def MAKE_CLOSURE(self, oparg, next_instr):
-        w_freevarstuple = self.peekvalue(1)
+        w_freevarstuple = self.peekvalue(2)
         freevars = [self.space.interp_w(Cell, cell)
                     for cell in self.space.fixedview(w_freevarstuple)]
         self._make_function(oparg, freevars)
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -23,7 +23,7 @@
 
     def test_qualname(self):
         def f(): pass
-        assert f.__qualname__ == 'f'
+        assert f.__qualname__ == 'test_qualname.f'
         f.__qualname__ = 'qualname'
         assert f.__qualname__ == 'qualname'
         raises(TypeError, "f.__qualname__ = b'name'")

From noreply at buildbot.pypy.org  Thu Dec 18 22:38:44 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Dec 2014 22:38:44 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Add type.__qualname__
Message-ID: <20141218213844.5E3191C03A4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75023:9db6340d5ce9
Date: 2014-12-18 22:35 +0100
http://bitbucket.org/pypy/pypy/changeset/9db6340d5ce9/

Log:	Add type.__qualname__

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -157,7 +157,6 @@
     def __init__(self, space, name, first_lineno, scope, compile_info):
         self.space = space
         self.name = name
-        self.qualname = name
         self.first_lineno = first_lineno
         self.compile_info = compile_info
         self.first_block = self.new_block()
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -177,7 +177,8 @@
     across the AST tree generating bytecode as needed.
     """
 
-    def __init__(self, space, name, tree, lineno, symbols, compile_info):
+    def __init__(self, space, name, tree, lineno, symbols, compile_info,
+                 qualname):
         self.scope = symbols.find_scope(tree)
         assemble.PythonCodeMaker.__init__(self, space, name, lineno,
                                           self.scope, compile_info)
@@ -185,6 +186,7 @@
         self.frame_blocks = []
         self.interactive = False
         self.temporary_name_counter = 1
+        self.qualname = qualname
         self._compile(tree)
 
     def _compile(self, tree):
@@ -203,10 +205,13 @@
 
     def sub_scope(self, kind, name, node, lineno):
         """Convenience function for compiling a sub scope."""
+        if self.qualname:
+            qualname = '%s.%s' % (self.qualname, name)
+        else:
+            qualname = name
         generator = kind(self.space, name, node, lineno, self.symbols,
-                         self.compile_info)
-        generator.qualname = '%s.%s' % (self.qualname, name)
-        return generator.assemble(), generator.qualname
+                         self.compile_info, qualname)
+        return generator.assemble(), qualname
 
     def push_frame_block(self, kind, block):
         self.frame_blocks.append((kind, block))
@@ -285,9 +290,7 @@
 
     def _make_function(self, code, num_defaults=0, qualname=None):
         """Emit the opcodes to turn a code object into a function."""
-        code_index = self.add_const(code)
         w_qualname = self.space.wrap(qualname or code.co_name)
-        qualname_index = self.add_const(w_qualname)
         if code.co_freevars:
             # Load cell and free vars to pass on.
             for free in code.co_freevars:
@@ -298,12 +301,12 @@
                     index = self.free_vars[free]
                 self.emit_op_arg(ops.LOAD_CLOSURE, index)
             self.emit_op_arg(ops.BUILD_TUPLE, len(code.co_freevars))
-            self.emit_op_arg(ops.LOAD_CONST, code_index)
-            self.emit_op_arg(ops.LOAD_CONST, qualname_index)
+            self.load_const(code)
+            self.load_const(w_qualname)
             self.emit_op_arg(ops.MAKE_CLOSURE, num_defaults)
         else:
-            self.emit_op_arg(ops.LOAD_CONST, code_index)
-            self.emit_op_arg(ops.LOAD_CONST, qualname_index)
+            self.load_const(code)
+            self.load_const(w_qualname)
             self.emit_op_arg(ops.MAKE_FUNCTION, num_defaults)
 
     def _visit_kwonlydefaults(self, args):
@@ -1242,7 +1245,7 @@
 
     def __init__(self, space, tree, symbols, compile_info):
         PythonCodeGenerator.__init__(self, space, "<module>", tree, -1,
-                                     symbols, compile_info)
+                                     symbols, compile_info, qualname=None)
 
     def _compile(self, tree):
         tree.walkabout(self)
@@ -1361,6 +1364,10 @@
         self.name_op("__name__", ast.Load)
         # ... and store it as __module__
         self.name_op("__module__", ast.Store)
+        # store the qualname
+        w_qualname = self.space.wrap(self.qualname)
+        self.load_const(w_qualname)
+        self.name_op("__qualname__", ast.Store)
         # compile the body proper
         self._handle_body(cls.body)
         # return the (empty) __class__ cell
diff --git a/pypy/interpreter/test/test_class.py b/pypy/interpreter/test/test_class.py
--- a/pypy/interpreter/test/test_class.py
+++ b/pypy/interpreter/test/test_class.py
@@ -109,3 +109,10 @@
         c = C()
         assert c.one == "two"
         raises(AttributeError, getattr, c, "two")
+
+    def test_qualname(self):
+        class C:
+            class D:
+                pass
+        assert C.__qualname__ == 'test_qualname.C'
+        assert C.D.__qualname__ == 'test_qualname.C.D'

From noreply at buildbot.pypy.org  Thu Dec 18 22:38:45 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Dec 2014 22:38:45 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Add "<locals>" in the qualname when the
 object comes from an executed function block.
Message-ID: <20141218213845.90F521C03A4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75024:daf8e9141435
Date: 2014-12-18 22:35 +0100
http://bitbucket.org/pypy/pypy/changeset/daf8e9141435/

Log:	Add "<locals>" in the qualname when the object comes from an
	executed function block.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -186,7 +186,10 @@
         self.frame_blocks = []
         self.interactive = False
         self.temporary_name_counter = 1
-        self.qualname = qualname
+        if isinstance(self.scope, symtable.FunctionScope):
+            self.qualname = qualname + '.<locals>'
+        else:
+            self.qualname = qualname
         self._compile(tree)
 
     def _compile(self, tree):
diff --git a/pypy/interpreter/test/test_class.py b/pypy/interpreter/test/test_class.py
--- a/pypy/interpreter/test/test_class.py
+++ b/pypy/interpreter/test/test_class.py
@@ -114,5 +114,5 @@
         class C:
             class D:
                 pass
-        assert C.__qualname__ == 'test_qualname.C'
-        assert C.D.__qualname__ == 'test_qualname.C.D'
+        assert C.__qualname__ == 'test_qualname.<locals>.C'
+        assert C.D.__qualname__ == 'test_qualname.<locals>.C.D'
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -22,8 +22,12 @@
         assert f.__module__ == 'mymodulename'
 
     def test_qualname(self):
-        def f(): pass
-        assert f.__qualname__ == 'test_qualname.f'
+        def f():
+            def g():
+                pass
+            return g
+        assert f.__qualname__ == 'test_qualname.<locals>.f'
+        assert f().__qualname__ == 'test_qualname.<locals>.f.<locals>.g'
         f.__qualname__ = 'qualname'
         assert f.__qualname__ == 'qualname'
         raises(TypeError, "f.__qualname__ = b'name'")

From noreply at buildbot.pypy.org  Thu Dec 18 22:38:46 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Dec 2014 22:38:46 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: unicode_internal codec accept both bytes
 and unicode for both functions
Message-ID: <20141218213846.CF9431C03A4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75025:0b478e4a57e3
Date: 2014-12-18 22:35 +0100
http://bitbucket.org/pypy/pypy/changeset/0b478e4a57e3/

Log:	unicode_internal codec accept both bytes and unicode for both
	functions

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -512,7 +512,7 @@
     wrap_decoder.func_name = rname
     globals()[name] = wrap_decoder
 
-for encoders in [
+for encoder in [
          "ascii_encode",
          "latin_1_encode",
          "utf_7_encode",
@@ -524,11 +524,10 @@
          "utf_32_le_encode",
          "unicode_escape_encode",
          "raw_unicode_escape_encode",
-         "unicode_internal_encode",
         ]:
-    make_encoder_wrapper(encoders)
+    make_encoder_wrapper(encoder)
 
-for decoders in [
+for decoder in [
          "ascii_decode",
          "latin_1_decode",
          "utf_7_decode",
@@ -539,7 +538,7 @@
          "utf_32_be_decode",
          "utf_32_le_decode",
          ]:
-    make_decoder_wrapper(decoders)
+    make_decoder_wrapper(decoder)
 
 if hasattr(runicode, 'str_decode_mbcs'):
     # mbcs functions are not regular, because we have to pass
@@ -810,6 +809,8 @@
 
 @unwrap_spec(errors='str_or_None')
 def unicode_internal_decode(space, w_string, errors="strict"):
+    space.warn(space.wrap("unicode_internal codec has been deprecated"),
+               space.w_DeprecationWarning)
     if errors is None:
         errors = 'strict'
     # special case for this codec: unicodes are returned as is
@@ -828,6 +829,23 @@
         final, state.decode_error_handler)
     return space.newtuple([space.wrap(result), space.wrap(consumed)])
 
+ at unwrap_spec(errors='str_or_None')
+def unicode_internal_encode(space, w_uni, errors="strict"):
+    space.warn(space.wrap("unicode_internal codec has been deprecated"),
+               space.w_DeprecationWarning)
+    if errors is None:
+        errors = 'strict'
+    if space.isinstance_w(w_uni, space.w_unicode):
+        uni = space.unicode_w(w_uni)
+        state = space.fromcache(CodecState)
+        result = runicode.unicode_encode_unicode_internal(
+            uni, len(uni), errors, state.encode_error_handler)
+        return space.newtuple([space.wrapbytes(result), space.wrap(len(uni))])
+    else:
+        # special case for this codec: bytes are returned as is
+        string = space.readbuf_w(w_uni).as_str()
+        return space.newtuple([space.wrapbytes(string), space.wrap(len(string))])
+
 # ____________________________________________________________
 # support for the "string escape" translation
 # This is a bytes-to bytes transformation
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -295,6 +295,12 @@
         assert _codecs.unicode_internal_decode(array.array('b', bytes))[0] == u"a"
         assert _codecs.unicode_internal_decode(memoryview(bytes))[0] == u"a"
 
+        # This codec accepts bytes and unicode on both sides
+        _codecs.unicode_internal_decode(u'\0\0\0\0')
+        _codecs.unicode_internal_decode(b'\0\0\0\0')
+        _codecs.unicode_internal_encode(u'\0\0\0\0')
+        _codecs.unicode_internal_encode(b'\0\0\0\0')
+
     def test_raw_unicode_escape(self):
         import _codecs
         assert str(b"\u0663", "raw-unicode-escape") == "\u0663"

From noreply at buildbot.pypy.org  Thu Dec 18 22:38:48 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 18 Dec 2014 22:38:48 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Add rfloat.log2(),
 which tries to be exact for powers of two.
Message-ID: <20141218213848.4020C1C03A4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75026:d3c7a156cc70
Date: 2014-12-18 22:35 +0100
http://bitbucket.org/pypy/pypy/changeset/d3c7a156cc70/

Log:	Add rfloat.log2(), which tries to be exact for powers of two. Use it
	in math.log2.

diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py
--- a/pypy/module/math/interp_math.py
+++ b/pypy/module/math/interp_math.py
@@ -202,6 +202,8 @@
             x = _get_double(space, w_x)
             if base == 10.0:
                 result = math.log10(x)
+            elif base == 2.0:
+                result = rfloat.log2(x)
             else:
                 result = math.log(x)
                 if base != 0.0:
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -281,6 +281,35 @@
             return (u - 1.) * x / math.log(u)
         return math.exp(x) - 1.
 
+def log2(x):
+    # Uses an algorithm that should:
+    #   (a) produce exact results for powers of 2, and
+    #   (b) be monotonic, assuming that the system log is monotonic.
+    if not isfinite(x):
+        if isnan(x):
+            return x  # log2(nan) = nan
+        elif x > 0.0:
+            return x  # log2(+inf) = +inf
+        else:
+            # log2(-inf) = nan, invalid-operation
+            raise ValueError("math domain error")
+
+    if x > 0.0:
+        if 0:  # HAVE_LOG2
+            return math.log2(x)
+        m, e = math.frexp(x)
+        # We want log2(m * 2**e) == log(m) / log(2) + e.  Care is needed when
+        # x is just greater than 1.0: in that case e is 1, log(m) is negative,
+        # and we get significant cancellation error from the addition of
+        # log(m) / log(2) to e.  The slight rewrite of the expression below
+        # avoids this problem.
+        if x >= 1.0:
+            return math.log(2.0 * m) / math.log(2.0) + (e - 1)
+        else:
+            return math.log(m) / math.log(2.0) + e
+    else:
+        raise ValueError("math domain error")
+
 def round_away(x):
     # round() from libm, which is not available on all platforms!
     absx = abs(x)
diff --git a/rpython/rlib/test/test_rfloat.py b/rpython/rlib/test/test_rfloat.py
--- a/rpython/rlib/test/test_rfloat.py
+++ b/rpython/rlib/test/test_rfloat.py
@@ -265,3 +265,12 @@
                 if s.strip(): # empty s raises OperationError directly
                     py.test.raises(ParseStringError, string_to_float, s)
     py.test.raises(ParseStringError, string_to_float, "")
+
+def test_log2():
+    from rpython.rlib import rfloat
+    assert rfloat.log2(1.0) == 0.0
+    assert rfloat.log2(2.0) == 1.0
+    assert rfloat.log2(2.0**1023) == 1023.0
+    assert 1.584 < rfloat.log2(3.0) < 1.585
+    py.test.raises(ValueError, rfloat.log2, 0)
+    py.test.raises(ValueError, rfloat.log2, -1)

From noreply at buildbot.pypy.org  Thu Dec 18 23:38:27 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Dec 2014 23:38:27 +0100 (CET)
Subject: [pypy-commit] creflect default: Stop trying to be clever and reject
 overloaded functions. Change
Message-ID: <20141218223827.5FAD01D2891@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r201:83606604d6cd
Date: 2014-12-18 17:34 +0000
http://bitbucket.org/cffi/creflect/changeset/83606604d6cd/

Log:	Stop trying to be clever and reject overloaded functions. Change
	the tests to use one possible workaround instead.

diff --git a/creflect/codegen.py b/creflect/codegen.py
--- a/creflect/codegen.py
+++ b/creflect/codegen.py
@@ -65,7 +65,6 @@
     outerblock = CodeBlock(parent=None)
     outerblock.writeline('#include "creflect.h"')
     outerblock.crx_func_name = crx_func_name
-    outerblock.func_names_seen = set()
 
     funcblock = CodeBlock(outerblock)
 
diff --git a/creflect/model.py b/creflect/model.py
--- a/creflect/model.py
+++ b/creflect/model.py
@@ -348,7 +348,7 @@
     def shadow_global_var(self, top_level_block, fnname):
         if self.ellipsis:
             return fnname
-        shadowname = _unique_name(top_level_block, '%s__d%s_%s', fnname)
+        shadowname = _unique_name(top_level_block, '%s__d_%s', fnname)
         wrline = top_level_block.writeline
         args = [arg.get_c_name('a%d' % i) for i, arg in enumerate(self.args)]
         decl = '%s(%s)' % (shadowname, ', '.join(args) or 'void')
@@ -365,7 +365,7 @@
         # a special-case for global function declarations
         assert not self.ellipsis
         toplevel = block.crx_top_level
-        crx_func_name = _unique_name(toplevel, '%s__c%s_%s', fnname)
+        crx_func_name = _unique_name(toplevel, '%s__c_%s', fnname)
         wrline = toplevel.writeline
         wrline('static void %s(void *args[], void *result) {' % (
             crx_func_name,))
@@ -379,13 +379,7 @@
 
 
 def _unique_name(toplevel, pattern, fnname):
-    result = pattern % (toplevel.crx_func_name, '', fnname)
-    n = 0
-    while result in toplevel.func_names_seen:
-        n += 1
-        result = pattern % (toplevel.crx_func_name, n, fnname)
-    toplevel.func_names_seen.add(result)
-    return result
+    return pattern % (toplevel.crx_func_name, fnname)
 
 
 class PointerType(BaseType):
diff --git a/creflect/test/codegen/func-006.c b/creflect/test/codegen/func-006.c
--- a/creflect/test/codegen/func-006.c
+++ b/creflect/test/codegen/func-006.c
@@ -1,30 +1,32 @@
-/* Multiple declarations of a function with the same name:
-   a way to work around "..." in signatures */
+/* A manual way to work around "..." in signatures */
 
-int f(int);
-int f(int, long);
+int f_1(int);
+int f_2(int, long);
 
 # ____________________________________________________________
 # drop
 
+#define f_1  f
+#define f_2  f
+
 int f(int a, ...) { return 42 * a; }
 
 # ____________________________________________________________
 
-static void testfunc_006__c_f(void *args[], void *result) {
-    *(int *)result = f(*(int *)args[0]);
+static void testfunc_006__c_f_1(void *args[], void *result) {
+    *(int *)result = f_1(*(int *)args[0]);
 }
 
-static int testfunc_006__d_f(int a0) {
-    return f(a0);
+static int testfunc_006__d_f_1(int a0) {
+    return f_1(a0);
 }
 
-static void testfunc_006__c1_f(void *args[], void *result) {
-    *(int *)result = f(*(int *)args[0], *(long *)args[1]);
+static void testfunc_006__c_f_2(void *args[], void *result) {
+    *(int *)result = f_2(*(int *)args[0], *(long *)args[1]);
 }
 
-static int testfunc_006__d1_f(int a0, long a1) {
-    return f(a0, a1);
+static int testfunc_006__d_f_2(int a0, long a1) {
+    return f_2(a0, a1);
 }
 
 void testfunc_006(_crx_builder_t *cb)
@@ -36,8 +38,8 @@
         t1 = cb->get_signed_type(cb, sizeof(int), "int");
         a1[0].type = t1;
         a1[0].qualifiers = 0;
-        cb->define_func(cb, "f", t1, a1, 1, &testfunc_006__c_f, &testfunc_006__d_f);
-#expect FUNC f: int -> int
+        cb->define_func(cb, "f_1", t1, a1, 1, &testfunc_006__c_f_1, &testfunc_006__d_f_1);
+#expect FUNC f_1: int -> int
     }
     {
         t2 = cb->get_signed_type(cb, sizeof(long), "long");
@@ -45,7 +47,7 @@
         a2[0].qualifiers = 0;
         a2[1].type = t2;
         a2[1].qualifiers = 0;
-        cb->define_func(cb, "f", t1, a2, 2, &testfunc_006__c1_f, &testfunc_006__d1_f);
-#expect FUNC f: int -> long -> int
+        cb->define_func(cb, "f_2", t1, a2, 2, &testfunc_006__c_f_2, &testfunc_006__d_f_2);
+#expect FUNC f_2: int -> long -> int
     }
 }
diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -828,10 +828,10 @@
 
     PyObject *l_dict = ((zeffir_builder_t *)cb)->l_dict;
     PyObject *l_libname_obj = ((zeffir_builder_t *)cb)->lib->l_libname_obj;
-    PyObject *x = PyDict_GetItemString(l_dict, name);
+    PyObject *x;
 
     assert(trampl != NULL);
-    x = make_builtin_func(l_libname_obj, name, ret, args, nargs, trampl, x);
+    x = make_builtin_func(l_libname_obj, name, ret, args, nargs, trampl);
     if (x == NULL)
         return;
 
diff --git a/zeffir/cfunc.c b/zeffir/cfunc.c
--- a/zeffir/cfunc.c
+++ b/zeffir/cfunc.c
@@ -1,5 +1,5 @@
 
-typedef struct ZefFuncSupportObject_s {
+typedef struct {
     PyObject_HEAD
 
     /* note that 'zfs_trampl' is either a '_crx_trampoline0_fn' or a
@@ -9,7 +9,6 @@
     int                 zfs_nargs;
     int                 zfs_dotdotdot;
     void               *zfs_trampl;
-    struct ZefFuncSupportObject_s *zfs_overloaded_next;
     PyMethodDef         zfs_md;
     size_t              zfs_size_args;
     CTypeDescrObject   *zfs_ret;
@@ -25,15 +24,6 @@
     for (i = zfs->zfs_nargs - 1; i >= 0; i--)
         Py_DECREF(zfs->zfs_args[i]);
     Py_DECREF(zfs->zfs_ret);
-
-    /* avoid long chain of recursive calls to zef_func_support_dealloc */
-    while (zfs->zfs_overloaded_next != NULL) {
-        ZefFuncSupportObject *zfs2 = zfs->zfs_overloaded_next;
-        zfs->zfs_overloaded_next = zfs2->zfs_overloaded_next;
-        zfs2->zfs_overloaded_next = NULL;
-        Py_DECREF(zfs2);
-    }
-
     PyObject_Del(zfs);
 }
 
@@ -134,27 +124,10 @@
         if (PyErr_Occurred())
             return NULL;
 
-        if (zfs->zfs_overloaded_next == NULL) {
-            PyErr_Format(PyExc_TypeError,
-                         "'%s' expected %d arguments, but got %zd",
-                         funcname, nargs, actualnargs);
-            return NULL;
-        }
-
-        /* XXX right now, pick the first overloaded function with the
-           correct number of arguments.  Later we'll need to pick the
-           right one based on the provided argument types. */
-        zfs = zfs->zfs_overloaded_next;
-        while (zfs->zfs_nargs != actualnargs) {
-            zfs = zfs->zfs_overloaded_next;
-            if (zfs == NULL) {
-                PyErr_Format(PyExc_TypeError,
-                             "'%s': no version with %zd arguments found",
-                             funcname, actualnargs);
-                return NULL;
-            }
-        }
-        nargs = zfs->zfs_nargs;
+        PyErr_Format(PyExc_TypeError,
+                     "'%s' expected %d arguments, but got %zd",
+                     funcname, nargs, actualnargs);
+        return NULL;
     }
 
     void *llargs[nargs];
@@ -283,7 +256,6 @@
 
     memset(&zfs->zfs_md, 0, sizeof(PyMethodDef));
     zfs->zfs_trampl = NULL;
-    zfs->zfs_overloaded_next = NULL;
     zfs->zfs_nargs = nargs;
     zfs->zfs_dotdotdot = dotdotdot;
     zfs->zfs_ret = ret;
@@ -316,8 +288,7 @@
 static PyObject *make_builtin_func(PyObject *libname_obj,
                                    const char *funcname, _crx_type_t *ret,
                                    _crx_qual_type args[], int nargs,
-                                   _crx_trampoline0_fn trampl,
-                                   PyObject *overloaded)
+                                   _crx_trampoline0_fn trampl)
 {
     char *p;
     ZefFuncSupportObject *zfs;
@@ -334,27 +305,11 @@
     zfs->zfs_md.ml_flags = METH_VARARGS;
     /*zfs->zfs_md.ml_doc = ... */
 
-    if (overloaded != NULL && PyCFunction_Check(overloaded) &&
-            PyCFunction_GET_FUNCTION(overloaded) == &zfs_call) {
-
-        /* attach 'zfs' to the end of the chain of zfs_overloaded_next */
-        PyObject *oself = PyCFunction_GET_SELF(overloaded);
-        ZefFuncSupportObject *zfs1 = (ZefFuncSupportObject *)oself;
-
-        while (zfs1->zfs_overloaded_next != NULL) {
-            zfs1 = zfs1->zfs_overloaded_next;
-        }
-        zfs1->zfs_overloaded_next = zfs;
-        Py_INCREF(overloaded);
-        return overloaded;
-    }
-    else {
-        /* make a new PyCFunction */
-        PyObject *res = PyCFunction_NewEx(&zfs->zfs_md, (PyObject *)zfs,
-                                          libname_obj);
-        Py_DECREF(zfs);
-        return res;
-    }
+    /* make a new PyCFunction */
+    PyObject *res = PyCFunction_NewEx(&zfs->zfs_md, (PyObject *)zfs,
+                                      libname_obj);
+    Py_DECREF(zfs);
+    return res;
 }
 
 static PyObject *cfunctype_getargs(CTypeDescrObject *ct)
diff --git a/zeffir/test/function.crx b/zeffir/test/function.crx
--- a/zeffir/test/function.crx
+++ b/zeffir/test/function.crx
@@ -36,6 +36,12 @@
     return result;
 }
 
+#define sum_ints_0  sum_ints
+#define sum_ints_1  sum_ints
+#define sum_ints_2  sum_ints
+#define sum_ints_3  sum_ints
+#define sum_ints_4  sum_ints
+
 
 // CREFLECT: start
 
@@ -43,10 +49,10 @@
 int add_from_array(int array[], int count);
 void returning_nothing(char *);
 void cant_call_dotdotdot(int x, ...);
-int sum_ints(int);
-int sum_ints(int, int);
-int sum_ints(int, int, int);
-int sum_ints(int, int, int, int);
-int sum_ints(int, int, int, int, int);
+int sum_ints_0(int);
+int sum_ints_1(int, int);
+int sum_ints_2(int, int, int);
+int sum_ints_3(int, int, int, int);
+int sum_ints_4(int, int, int, int, int);
 
 // CREFLECT: end
diff --git a/zeffir/test/test_function.py b/zeffir/test/test_function.py
--- a/zeffir/test/test_function.py
+++ b/zeffir/test/test_function.py
@@ -52,8 +52,19 @@
 
 def test_overloaded_function():
     ffi, lib = support.compile_and_open('function')
-    assert lib.sum_ints(0) == 0
-    assert lib.sum_ints(1, 10) == 10
-    assert lib.sum_ints(2, 20, 30) == 50
-    assert lib.sum_ints(3, 40, 50, 60) == 150
-    assert lib.sum_ints(4, 70, 80, 90, 100) == 340
+    assert lib.sum_ints_0(0) == 0
+    assert lib.sum_ints_1(1, 10) == 10
+    assert lib.sum_ints_2(2, 20, 30) == 50
+    assert lib.sum_ints_3(3, 40, 50, 60) == 150
+    assert lib.sum_ints_4(4, 70, 80, 90, 100) == 340
+    #
+    # or, this can be done in pure Python with some code like that:
+    _sum_ints = [lib.sum_ints_0, lib.sum_ints_1, lib.sum_ints_2,
+                lib.sum_ints_3, lib.sum_ints_4]
+    def sum_ints(*args):
+        return _sum_ints[len(args)](len(args), *args)
+    assert sum_ints() == 0
+    assert sum_ints(10) == 10
+    assert sum_ints(20, 30) == 50
+    assert sum_ints(40, 50, 60) == 150
+    assert sum_ints(70, 80, 90, 100) == 340

From noreply at buildbot.pypy.org  Thu Dec 18 23:38:28 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 18 Dec 2014 23:38:28 +0100 (CET)
Subject: [pypy-commit] creflect default: more tests
Message-ID: <20141218223828.89DF31D2891@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r202:80584114b8eb
Date: 2014-12-18 17:38 +0000
http://bitbucket.org/cffi/creflect/changeset/80584114b8eb/

Log:	more tests

diff --git a/zeffir/test/test_funcptr.py b/zeffir/test/test_funcptr.py
--- a/zeffir/test/test_funcptr.py
+++ b/zeffir/test/test_funcptr.py
@@ -15,6 +15,8 @@
 def test_get_fnptr():
     ffi, lib = support.compile_and_open('funcptr')
     fn = lib.get_fnptr(42)
+    assert fn == lib.get_fnptr(42)
+    assert fn != lib.get_fnptr(1)
     assert fn(100) == 142
     assert ffi.typeof(fn) == ffi.typeof("int(*)(int)")
     #

From noreply at buildbot.pypy.org  Fri Dec 19 00:30:13 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 00:30:13 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: update version to 2.7.9
Message-ID: <20141218233013.1B2941D2903@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75027:2d54007751f0
Date: 2014-12-18 18:23 -0500
http://bitbucket.org/pypy/pypy/changeset/2d54007751f0/

Log:	update version to 2.7.9

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -21,12 +21,12 @@
 /* Version parsed out into numeric values */
 #define PY_MAJOR_VERSION	2
 #define PY_MINOR_VERSION	7
-#define PY_MICRO_VERSION	8
+#define PY_MICRO_VERSION	9
 #define PY_RELEASE_LEVEL	PY_RELEASE_LEVEL_FINAL
 #define PY_RELEASE_SERIAL	0
 
 /* Version as a string */
-#define PY_VERSION		"2.7.8"
+#define PY_VERSION		"2.7.9"
 
 /* PyPy version as a string */
 #define PYPY_VERSION "2.5.0-alpha0"
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -6,7 +6,7 @@
 from pypy.interpreter import gateway
 
 #XXX # the release serial 42 is not in range(16)
-CPYTHON_VERSION            = (2, 7, 8, "final", 42)
+CPYTHON_VERSION            = (2, 7, 9, "final", 42)
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 

From noreply at buildbot.pypy.org  Fri Dec 19 00:30:14 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 00:30:14 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: skip some added tests when run
 against older cpython
Message-ID: <20141218233014.5086B1D2903@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75028:b85da8ff6c09
Date: 2014-12-18 18:29 -0500
http://bitbucket.org/pypy/pypy/changeset/b85da8ff6c09/

Log:	skip some added tests when run against older cpython

diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -5,6 +5,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.argument import Arguments
 
+
 class BaseTestCompiler:
     def setup_method(self, method):
         self.compiler = self.space.createcompiler()
@@ -709,13 +710,13 @@
         else:
             py.test.fail("Did not raise")
 
+
 class TestECCompiler(BaseTestCompiler):
     def setup_method(self, method):
         self.compiler = self.space.getexecutioncontext().compiler
 
 
 class AppTestCompiler:
-
     def test_bom_with_future(self):
         s = '\xef\xbb\xbffrom __future__ import division\nx = 1/2'
         ns = {}
@@ -766,8 +767,8 @@
 ##    def test_try_except_finally(self):
 ##        py.test.skip("unsupported")
 
+
 class AppTestOptimizer:
-
     def setup_class(cls):
         cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
 
@@ -920,14 +921,14 @@
             sys.stdout = save_stdout
         output = s.getvalue()
         assert "STOP_CODE" not in output
-    
+
     def test_optimize_list_comp(self):
         source = """def _f(a):
             return [x for x in a if None]
         """
         exec source
         code = _f.func_code
-        
+
         import StringIO, sys, dis
         s = StringIO.StringIO()
         out = sys.stdout
@@ -959,7 +960,7 @@
         """
         exec source
         code = _f.func_code
-        
+
         import StringIO, sys, dis
         s = StringIO.StringIO()
         out = sys.stdout
@@ -1039,5 +1040,8 @@
         assert eval(code) == u'\xa4'
         code = '# -*- coding: iso8859-15 -*-\nu"\xc2\xa4"\n'
         assert eval(code) == u'\xc2\u20ac'
+        import sys
+        if sys.version_info < (2, 7, 9):
+            skip()
         code = 'u"""\\\n# -*- coding: utf-8 -*-\n\xc2\xa4"""\n'
         assert eval(code) == u'# -*- coding: utf-8 -*-\n\xc2\xa4'
diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py
--- a/pypy/interpreter/test/test_exec.py
+++ b/pypy/interpreter/test/test_exec.py
@@ -246,6 +246,9 @@
         assert ord(x[0]) == 0x0439
 
     def test_nested_qualified_exec(self):
+        import sys
+        if sys.version_info < (2, 7, 9):
+            skip()
         code = ["""
 def g():
     def f():
diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -194,6 +194,8 @@
     def test_group_takes_long(self):
         import re
         import sys
+        if sys.version_info < (2, 7, 9):
+            skip()
         assert re.match("(foo)", "foo").group(1L) == "foo"
         exc = raises(IndexError, re.match("", "").group, sys.maxint + 1)
         assert str(exc.value) == "no such group"

From noreply at buildbot.pypy.org  Fri Dec 19 00:50:58 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 00:50:58 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: update _ctypes_test.c from cpython
Message-ID: <20141218235058.79CDB1C03A4@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75029:3f0415c7d6b1
Date: 2014-12-18 18:39 -0500
http://bitbucket.org/pypy/pypy/changeset/3f0415c7d6b1/

Log:	update _ctypes_test.c from cpython

diff --git a/lib_pypy/_ctypes_test.c b/lib_pypy/_ctypes_test.c
--- a/lib_pypy/_ctypes_test.c
+++ b/lib_pypy/_ctypes_test.c
@@ -541,6 +541,49 @@
     return 1;
 }
 
+EXPORT(long left = 10);
+EXPORT(long top = 20);
+EXPORT(long right = 30);
+EXPORT(long bottom = 40);
+
+EXPORT(RECT) ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr,
+                        RECT *er, POINT fp, RECT gr)
+{
+    /*Check input */
+    if (ar.left + br->left + dr.left + er->left + gr.left != left * 5)
+    {
+        ar.left = 100;
+        return ar;
+    }
+    if (ar.right + br->right + dr.right + er->right + gr.right != right * 5)
+    {
+        ar.right = 100;
+        return ar;
+    }
+    if (cp.x != fp.x)
+    {
+        ar.left = -100;
+    }
+    if (cp.y != fp.y)
+    {
+        ar.left = -200;
+    }
+    switch(i)
+    {
+    case 0:
+        return ar;
+        break;
+    case 1:
+        return dr;
+        break;
+    case 2:
+        return gr;
+        break;
+
+    }
+    return ar;
+}
+
 typedef struct {
     short x;
     short y;

From noreply at buildbot.pypy.org  Fri Dec 19 00:50:59 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 00:50:59 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: skip a test_unicode when no
	ctypes.pythonapi exists
Message-ID: <20141218235059.BB1711C03A4@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75030:773018d6ae59
Date: 2014-12-18 18:50 -0500
http://bitbucket.org/pypy/pypy/changeset/773018d6ae59/

Log:	skip a test_unicode when no ctypes.pythonapi exists

diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py
--- a/lib-python/2.7/test/test_unicode.py
+++ b/lib-python/2.7/test/test_unicode.py
@@ -1664,6 +1664,10 @@
     # Test PyUnicode_FromFormat()
     def test_from_format(self):
         test_support.import_module('ctypes')
+        try:
+            from ctypes import pythonapi
+        except ImportError:
+            self.skipTest( "no pythonapi in ctypes")
         from ctypes import (
             pythonapi, py_object, sizeof,
             c_int, c_long, c_longlong, c_ssize_t,

From noreply at buildbot.pypy.org  Fri Dec 19 01:19:21 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 01:19:21 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: add a failing binascii.a2b_uu case
Message-ID: <20141219001921.8857C1D2891@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75031:c297e68f8ff0
Date: 2014-12-18 19:19 -0500
http://bitbucket.org/pypy/pypy/changeset/c297e68f8ff0/

Log:	add a failing binascii.a2b_uu case

diff --git a/pypy/module/binascii/test/test_binascii.py b/pypy/module/binascii/test/test_binascii.py
--- a/pypy/module/binascii/test/test_binascii.py
+++ b/pypy/module/binascii/test/test_binascii.py
@@ -11,6 +11,7 @@
         assert self.binascii.a2b_uu("") == "\x00" * 0x20
         #
         for input, expected in [
+            ("!,_", "3"),
             (" ", ""),
             ("!", "\x00"),
             ("!6", "X"),

From noreply at buildbot.pypy.org  Fri Dec 19 05:06:27 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 05:06:27 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: fix test_readline version check
Message-ID: <20141219040627.13C641C04B4@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75032:96a806f14899
Date: 2014-12-18 23:06 -0500
http://bitbucket.org/pypy/pypy/changeset/96a806f14899/

Log:	fix test_readline version check

diff --git a/lib-python/2.7/test/test_readline.py b/lib-python/2.7/test/test_readline.py
--- a/lib-python/2.7/test/test_readline.py
+++ b/lib-python/2.7/test/test_readline.py
@@ -44,7 +44,8 @@
 
 class TestReadline(unittest.TestCase):
 
-    @unittest.skipIf(readline._READLINE_VERSION < 0x0600
+    @unittest.skipIf(hasattr(readline, '_READLINE_VERSION')
+                     and readline._READLINE_VERSION < 0x0600
                      and "libedit" not in readline.__doc__,
                      "not supported in this library version")
     def test_init(self):

From noreply at buildbot.pypy.org  Fri Dec 19 05:27:03 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 05:27:03 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: fix test_xdrlib by correcting
 struct.pack exception types
Message-ID: <20141219042703.B5C161C03A4@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75033:8ed6988ac4dd
Date: 2014-12-18 23:27 -0500
http://bitbucket.org/pypy/pypy/changeset/8ed6988ac4dd/

Log:	fix test_xdrlib by correcting struct.pack exception types

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -84,7 +84,12 @@
         else:
             msg = "integer argument expected, got non-integer"
         space.warn(space.wrap(msg), space.w_DeprecationWarning)
-        return space.int(w_obj)   # wrapped float -> wrapped int or long
+        try:
+            return space.int(w_obj)   # wrapped float -> wrapped int or long
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                raise StructError("cannot convert argument to integer")
+            raise
 
     def accept_bool_arg(self):
         w_obj = self.accept_obj_arg()
@@ -100,7 +105,12 @@
 
     def accept_float_arg(self):
         w_obj = self.accept_obj_arg()
-        return self.space.float_w(w_obj)
+        try:
+            return self.space.float_w(w_obj)
+        except OperationError as e:
+            if e.match(self.space, self.space.w_TypeError):
+                raise StructError("required argument is not a float")
+            raise
 
 
 class UnpackFormatIterator(FormatIterator):
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -328,6 +328,12 @@
             raises(error, unpack, "0p", "")   # segfaults on CPython 2.5.2!
         raises(error, pack, "b", 150)   # argument out of range
         # XXX the accepted ranges still differs between PyPy and CPython
+        exc = raises(error, pack, ">d", 'abc')
+        assert str(exc.value) == "required argument is not a float"
+        exc = raises(error, pack, ">l", 'abc')
+        assert str(exc.value) == "cannot convert argument to integer"
+        exc = raises(error, pack, ">H", 'abc')
+        assert str(exc.value) == "cannot convert argument to integer"
 
     def test_overflow_error(self):
         """

From noreply at buildbot.pypy.org  Fri Dec 19 08:17:37 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 08:17:37 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: provide complex.__int__ method
Message-ID: <20141219071737.7D4821C0483@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75034:04003981e306
Date: 2014-12-19 02:15 -0500
http://bitbucket.org/pypy/pypy/changeset/04003981e306/

Log:	provide complex.__int__ method

diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -274,8 +274,7 @@
         return space.newlong_from_rbigint(val)
 
     def int(self, space):
-        raise oefmt(space.w_TypeError,
-                    "can't convert complex to int; use int(abs(z))")
+        raise oefmt(space.w_TypeError, "can't convert complex to int")
 
     def _to_complex(self, space, w_obj):
         if isinstance(w_obj, W_ComplexObject):
@@ -381,8 +380,7 @@
         return space.newbool((self.realval != 0.0) or (self.imagval != 0.0))
 
     def descr_float(self, space):
-        raise oefmt(space.w_TypeError,
-                    "can't convert complex to float; use abs(z)")
+        raise oefmt(space.w_TypeError, "can't convert complex to float")
 
     def descr_neg(self, space):
         return W_ComplexObject(-self.realval, -self.imagval)
@@ -603,6 +601,7 @@
     __coerce__ = interp2app(W_ComplexObject.descr_coerce),
     __format__ = interp2app(W_ComplexObject.descr_format),
     __nonzero__ = interp2app(W_ComplexObject.descr_nonzero),
+    __int__ = interp2app(W_ComplexObject.int),
     __float__ = interp2app(W_ComplexObject.descr_float),
     __neg__ = interp2app(W_ComplexObject.descr_neg),
     __pos__ = interp2app(W_ComplexObject.descr_pos),
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -161,6 +161,12 @@
     def test_coerce(self):
         raises(OverflowError, complex.__coerce__, 1+1j, 1L<<10000)
 
+    def test_convert(self):
+        exc = raises(TypeError, complex.__int__, 3j)
+        assert str(exc.value) == "can't convert complex to int"
+        exc = raises(TypeError, complex.__float__, 3j)
+        assert str(exc.value) == "can't convert complex to float"
+
     def test_richcompare(self):
         assert complex.__lt__(1+1j, None) is NotImplemented
         assert complex.__eq__(1+1j, 2+2j) is False

From noreply at buildbot.pypy.org  Fri Dec 19 08:17:38 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 08:17:38 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: further fix struct.pack conversions
Message-ID: <20141219071738.B20A71C0483@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75035:09d67d2e9a36
Date: 2014-12-19 02:17 -0500
http://bitbucket.org/pypy/pypy/changeset/09d67d2e9a36/

Log:	further fix struct.pack conversions

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -65,32 +65,25 @@
             w_index = w_obj
         else:
             w_index = None
-            w_index_method = space.lookup(w_obj, "__index__")
-            if w_index_method is not None:
+            if space.lookup(w_obj, '__index__'):
                 try:
                     w_index = space.index(w_obj)
                 except OperationError, e:
                     if not e.match(space, space.w_TypeError):
                         raise
                     pass
+            if w_index is None and space.lookup(w_obj, '__int__'):
+                if space.isinstance_w(w_obj, space.w_float):
+                    msg = "integer argument expected, got float"
+                else:
+                    msg = "integer argument expected, got non-integer" \
+                          " (implicit conversion using __int__ is deprecated)"
+                space.warn(space.wrap(msg), space.w_DeprecationWarning)
+                w_index = space.int(w_obj)   # wrapped float -> wrapped int or long
             if w_index is None:
-                w_index = self._maybe_float(w_obj)
+                raise StructError("cannot convert argument to integer")
         return getattr(space, meth)(w_index)
 
-    def _maybe_float(self, w_obj):
-        space = self.space
-        if space.isinstance_w(w_obj, space.w_float):
-            msg = "struct: integer argument expected, got float"
-        else:
-            msg = "integer argument expected, got non-integer"
-        space.warn(space.wrap(msg), space.w_DeprecationWarning)
-        try:
-            return space.int(w_obj)   # wrapped float -> wrapped int or long
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                raise StructError("cannot convert argument to integer")
-            raise
-
     def accept_bool_arg(self):
         w_obj = self.accept_obj_arg()
         return self.space.is_true(w_obj)
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -63,12 +63,22 @@
     def test_deprecation_warning(self):
         import warnings
         for code in 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q':
-            with warnings.catch_warnings(record=True) as w:
-                warnings.simplefilter("always")
-                raises(TypeError, self.struct.pack, code, 3j)
-            assert len(w) == 1
-            assert str(w[0].message) == "integer argument expected, got non-integer"
-            assert w[0].category is DeprecationWarning
+            for val in [3., 3j]:
+                with warnings.catch_warnings(record=True) as w:
+                    warnings.simplefilter("always")
+                    if type(val) is float:
+                        self.struct.pack(code, val)
+                    else:
+                        raises(TypeError, self.struct.pack, code, val)
+                assert len(w) == 1
+                if type(val) is float:
+                    assert str(w[0].message) == (
+                        "integer argument expected, got float")
+                else:
+                    assert str(w[0].message) == (
+                        "integer argument expected, got non-integer"
+                        " (implicit conversion using __int__ is deprecated)")
+                assert w[0].category is DeprecationWarning
 
     def test_pack_standard_little(self):
         """

From noreply at buildbot.pypy.org  Fri Dec 19 13:42:45 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 19 Dec 2014 13:42:45 +0100 (CET)
Subject: [pypy-commit] pypy default: backout step 1
Message-ID: <20141219124245.A9A9D1C11BF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75036:1f3bc0c62780
Date: 2014-12-19 14:41 +0200
http://bitbucket.org/pypy/pypy/changeset/1f3bc0c62780/

Log:	backout step 1

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -97,7 +97,6 @@
             jumpop = None
 
         self.import_state(start_label, starting_state)
-
         self.optimizer.propagate_all_forward(clear=False)
 
         if not jumpop:
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -638,6 +638,7 @@
             for i in range(op.numargs()):
                 op.setarg(i, self.renamed(op.getarg(i)))
         if box in self.short_boxes:
+            return
             if op is None:
                 oldop = self.short_boxes[box].clone()
                 oldres = oldop.result

From noreply at buildbot.pypy.org  Fri Dec 19 13:42:47 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 19 Dec 2014 13:42:47 +0100 (CET)
Subject: [pypy-commit] pypy default: backout, fixing tests
Message-ID: <20141219124247.0701D1C11BF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75037:c3ff0dd6252b
Date: 2014-12-19 14:42 +0200
http://bitbucket.org/pypy/pypy/changeset/c3ff0dd6252b/

Log:	backout, fixing tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1181,7 +1181,7 @@
         i1 = getfield_gc(p0, descr=valuedescr)
         i2 = int_sub(i1, 1)
         i3 = int_add(i0, i1)
-        #i4 = same_as(i2) # This same_as should be killed by backend
+        i4 = same_as(i2) # This same_as should be killed by backend
         jump(i3, i1, i2)
         """
         expected = """
@@ -1317,7 +1317,7 @@
         p30 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p30, i28, descr=nextdescr)
         setfield_gc(p3, p30, descr=valuedescr)
-        #p46 = same_as(p30) # This same_as should be killed by backend
+        p46 = same_as(p30) # This same_as should be killed by backend
         jump(i29, p30, p3)
         """
         expected = """
@@ -2414,16 +2414,16 @@
         guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(p1, i2, descr=valuedescr)
-        #i7 = same_as(i2) # This same_as should be killed by backend
-        #i6 = same_as(i4)
-        jump(p1, i1, i2, i4)
-        """
-        expected = """
-        [p1, i1, i2, i4]
+        i7 = same_as(i2) # This same_as should be killed by backend
+        i6 = same_as(i4)
+        jump(p1, i1, i2, i4, i6)
+        """
+        expected = """
+        [p1, i1, i2, i4, i5]
         setfield_gc(p1, i1, descr=valuedescr)
         guard_true(i4) []
         setfield_gc(p1, i2, descr=valuedescr)
-        jump(p1, i1, i2, 1)
+        jump(p1, i1, i2, i5, i5)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -2446,15 +2446,15 @@
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         escape()
-        #i5 = same_as(i4)
-        jump(p1, i2, i4)
-        """
-        expected = """
-        [p1, i2, i4]
+        i5 = same_as(i4)
+        jump(p1, i2, i4, i5)
+        """
+        expected = """
+        [p1, i2, i4, i5]
         guard_true(i4) [p1]
         setfield_gc(p1, NULL, descr=nextdescr)
         escape()
-        jump(p1, i2, 1)
+        jump(p1, i2, i5, i5)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -2476,15 +2476,15 @@
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         escape()
-        #i5 = same_as(i4)
-        jump(p1, i2, i4)
-        """
-        expected = """
-        [p1, i2, i4]
+        i5 = same_as(i4)
+        jump(p1, i2, i4, i5)
+        """
+        expected = """
+        [p1, i2, i4, i5]
         guard_true(i4) [i2, p1]
         setfield_gc(p1, NULL, descr=nextdescr)
         escape()
-        jump(p1, i2, 1)
+        jump(p1, i2, i5, i5)
         """
         self.optimize_loop(ops, expected)
 
@@ -2507,17 +2507,17 @@
         guard_true(i5) []
         i4 = int_neg(i2)
         setfield_gc(p1, i2, descr=valuedescr)
-        #i8 = same_as(i2) # This same_as should be killed by backend
-        #i7 = same_as(i4)
-        jump(p1, i1, i2, i4)
-        """
-        expected = """
-        [p1, i1, i2, i4]
+        i8 = same_as(i2) # This same_as should be killed by backend
+        i7 = same_as(i4)
+        jump(p1, i1, i2, i4, i7)
+        """
+        expected = """
+        [p1, i1, i2, i4, i7]
         setfield_gc(p1, i1, descr=valuedescr)
         i5 = int_eq(i4, 5)
         guard_true(i5) []
         setfield_gc(p1, i2, descr=valuedescr)
-        jump(p1, i1, i2, 5)
+        jump(p1, i1, i2, i7, i7)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -2735,16 +2735,16 @@
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        #i101 = same_as(i4)
-        jump(p1, i2, i4, p4)
-        """
-        expected = """
-        [p1, i2, i4, p4]
+        i101 = same_as(i4)
+        jump(p1, i2, i4, p4, i101)
+        """
+        expected = """
+        [p1, i2, i4, p4, i5]
         guard_true(i4) [p1, p4]
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        jump(p1, i2, 1, p4)
+        jump(p1, i2, i5, p4, i5)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -3620,7 +3620,7 @@
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call_assembler(i1, descr=asmdescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        #i143 = same_as(i3) # Should be killed by backend
+        i143 = same_as(i3) # Should be killed by backend
         jump(p1, i4, i3)
         '''
         self.optimize_loop(ops, ops, preamble)
@@ -3645,18 +3645,18 @@
         jump(p1, i4, i3)
         '''
         expected = '''
-        [p1, i4, i3]
-        setfield_gc(p1, i3, descr=valuedescr)
-        jump(p1, i3, i3)
+        [p1, i4, i3, i5]
+        setfield_gc(p1, i5, descr=valuedescr)
+        jump(p1, i3, i5, i5)
         '''
         preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        #i148 = same_as(i3)
-        #i147 = same_as(i3)
-        jump(p1, i4, i3)
+        i148 = same_as(i3)
+        i147 = same_as(i3)
+        jump(p1, i4, i3, i148)
         '''
         self.optimize_loop(ops, expected, preamble)
 
@@ -3670,17 +3670,17 @@
         jump(p1, i4, i3)
         '''
         expected = '''
-        [p1, i4, i3]
+        [p1, i4, i3, i5]
         setfield_gc(p1, i4, descr=valuedescr)
-        jump(p1, i3, i3)
+        jump(p1, i3, i5, i5)
         '''
         preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i1, descr=valuedescr)
-        #i151 = same_as(i3)
-        jump(p1, i4, i3)
+        i151 = same_as(i3)
+        jump(p1, i4, i3, i151)
         '''
         self.optimize_loop(ops, expected, preamble)
 
@@ -3700,14 +3700,14 @@
         escape(i1)
         escape(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        #i153 = same_as(i4)
-        jump(i0, i4)
+        i153 = same_as(i4)
+        jump(i0, i4, i153)
         '''
         expected = '''
-        [i0, i4]
+        [i0, i4, i5]
         escape(42)
         escape(i4)
-        jump(i0, i4)
+        jump(i0, i5, i5)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
@@ -3731,14 +3731,14 @@
         escape(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
         guard_no_exception() []
-        #i155 = same_as(i4)
-        jump(i0, i4)
+        i155 = same_as(i4)
+        jump(i0, i4, i155)
         '''
         expected = '''
-        [i0, i2]
+        [i0, i2, i3]
         escape(42)
         escape(i2)
-        jump(i0, i2)
+        jump(i0, i3, i3)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
@@ -5864,12 +5864,11 @@
         [p0]
         p1 = getfield_gc(p0, descr=valuedescr)
         setfield_gc(p0, p0, descr=valuedescr)
-        #p4450 = same_as(p0) # Should be killed by backend
+        p4450 = same_as(p0) # Should be killed by backend
         jump(p0)
         """
         expected = """
         [p0]
-        p1 = getfield_gc(p0, descr=valuedescr)
         setfield_gc(p0, p0, descr=valuedescr)
         jump(p0)
         """
@@ -6265,13 +6264,13 @@
         [p1, i1, i2, i3]
         escape(i3)
         i4 = int_sub(i2, i1)
-        #i5 = same_as(i4)
-        jump(p1, i1, i2, i4)
-        """
-        expected = """
-        [p1, i1, i2, i3]
+        i5 = same_as(i4)
+        jump(p1, i1, i2, i4, i5)
+        """
+        expected = """
+        [p1, i1, i2, i3, i4]
         escape(i3)
-        jump(p1, i1, i2, i3)
+        jump(p1, i1, i2, i4, i4)
         """
         self.optimize_strunicode_loop(ops, expected, preamble)
 
@@ -6291,15 +6290,14 @@
         escape(i5)
         i4 = int_sub(i2, i1)
         setfield_gc(p2, i4, descr=valuedescr)
-        #i8 = same_as(i4)
-        jump(p1, i1, i2, p2, i4)
-        """
-        expected = """
-        [p1, i1, i2, p2, i5]
-        i311 = getfield_gc(p2, descr=valuedescr)
-        escape(i311)
-        setfield_gc(p2, i5, descr=valuedescr)
-        jump(p1, i1, i2, p2, i5)
+        i8 = same_as(i4)
+        jump(p1, i1, i2, p2, i8, i4)
+        """
+        expected = """
+        [p1, i1, i2, p2, i5, i6]
+        escape(i5)
+        setfield_gc(p2, i6, descr=valuedescr)
+        jump(p1, i1, i2, p2, i6, i6)
         """
         self.optimize_strunicode_loop(ops, expected, preamble)
 
@@ -7106,18 +7104,19 @@
         p188 = getarrayitem_gc(p187, 42, descr=<GcPtrArrayDescr>)
         guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
-        jump(p25, p187, i184)
+        p26 = same_as(p25)
+        jump(p25, p187, i184, p26)
         """
         short = """
         [p1, p187, i184]
         p188 = getarrayitem_gc(p187, 42, descr=<GcPtrArrayDescr>)
         guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc(ConstPtr(myptr), descr=otherdescr)
-        jump(p25, p187, i184)
-        """
-        expected = """
-        [p25, p187, i184]
-        jump(p25, p187, i184)
+        jump(p1, p187, i184, p25)
+        """
+        expected = """
+        [p25, p187, i184, p189]
+        jump(p189, p187, i184, p189)
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
@@ -7226,16 +7225,14 @@
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
         setfield_gc(p1, ii, descr=otherdescr)
-        jump(p0, p1, ii2, ii)
-        """
-        expected = """
-        [p0, p1, ii, ii2]
-        i1 = getfield_gc(p0, descr=valuedescr)
-        i2 = getfield_gc(p1, descr=otherdescr)
+        jump(p0, p1, ii2, ii, ii, ii)
+        """
+        expected = """
+        [p0, p1, ii, ii2, i1, i2]
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
         setfield_gc(p1, ii, descr=otherdescr)
-        jump(p0, p1, ii2, ii)
+        jump(p0, p1, ii2, ii, ii, ii)
         """
         self.optimize_loop(ops, expected)
 
@@ -7726,10 +7723,10 @@
         jump(p1)
         """
         expected = """
-        [p0]
+        [p0, p1]
         call(p0, descr=nonwritedescr)
-        call(p0, descr=writeadescr)
-        jump(p0)
+        call(p1, descr=writeadescr)
+        jump(p1, p1)
         """
         self.optimize_loop(ops, expected)
 
@@ -7840,11 +7837,10 @@
         jump(p5, p6)
         """
         expected = """
-        [p5, p6, i10]
-        i11 = getfield_gc(p6, descr=nextdescr)
+        [p5, p6, i10, i11]
         call(i10, i11, descr=nonwritedescr)
         setfield_gc(p6, i10, descr=nextdescr)
-        jump(p5, p6, i10)
+        jump(p5, p6, i10, i10)
         """
         self.optimize_loop(ops, expected)
 
@@ -7860,12 +7856,11 @@
         jump(p5, p6)
         """
         expected = """
-        [p5, p6, i12, i10]
-        i11 = getfield_gc(p6, descr=nextdescr)
-        i13 = int_add(i11, 7)
+        [p5, p6, i14, i12, i10]
+        i13 = int_add(i14, 7)
         call(i12, i13, descr=nonwritedescr)
         setfield_gc(p6, i10, descr=nextdescr)
-        jump(p5, p6, i12, i10)
+        jump(p5, p6, i10, i12, i10)
         """
         self.optimize_loop(ops, expected)
 
@@ -7985,9 +7980,9 @@
         jump(i1, i3)
         """
         expected = """
-        [i1, i2, i6]
+        [i1, i2, i6, i3]
         call(i6, descr=nonwritedescr)
-        jump(i1, i2, i6)
+        jump(i1, i3, i6, i3)
         """
         short = """
         [i1, i2]
@@ -7995,7 +7990,7 @@
         i4 = int_add(i3, i3)
         i5 = int_add(i4, i4)
         i6 = int_add(i5, i5)
-        jump(i1, i3, i6)
+        jump(i1, i2, i6, i3)
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
@@ -8150,18 +8145,19 @@
         call(i2, descr=nonwritedescr)
         setfield_gc(p22, i1, descr=valuedescr)
         guard_nonnull_class(p18, ConstClass(node_vtable)) []
-        jump(p22, p18, i1)
+        i10 = same_as(i1)
+        jump(p22, p18, i1, i10)
         """
         short = """
         [p22, p18, i1]
-        jump(p22, p18, i1)
-        """
-        expected = """
-        [p22, p18, i2]
-        i1 = getfield_gc(p22, descr=valuedescr)
-        call(i1, descr=nonwritedescr)
-        setfield_gc(p22, i2, descr=valuedescr)
-        jump(p22, p18, i2)
+        i2 = getfield_gc(p22, descr=valuedescr)
+        jump(p22, p18, i1, i2)
+        """
+        expected = """
+        [p22, p18, i1, i2]
+        call(i2, descr=nonwritedescr)
+        setfield_gc(p22, i1, descr=valuedescr)
+        jump(p22, p18, i1, i1)
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -638,7 +638,6 @@
             for i in range(op.numargs()):
                 op.setarg(i, self.renamed(op.getarg(i)))
         if box in self.short_boxes:
-            return
             if op is None:
                 oldop = self.short_boxes[box].clone()
                 oldres = oldop.result

From noreply at buildbot.pypy.org  Fri Dec 19 15:59:06 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 15:59:06 +0100 (CET)
Subject: [pypy-commit] cffi default: fix an obscure-case leak
Message-ID: <20141219145906.73C391C0399@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1593:c6be8df144eb
Date: 2014-12-19 15:59 +0100
http://bitbucket.org/cffi/cffi/changeset/c6be8df144eb/

Log:	fix an obscure-case leak

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3070,6 +3070,7 @@
             char *data = ((CDataObject *)io)->c_data;
             /*READ(data, sizeof(long double)*/
             lvalue = read_raw_longdouble_data(data);
+            Py_DECREF(io);
             cd = _new_casted_primitive(ct);
             if (cd != NULL)
                 write_raw_longdouble_data(cd->c_data, lvalue);

From noreply at buildbot.pypy.org  Fri Dec 19 16:42:48 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 16:42:48 +0100 (CET)
Subject: [pypy-commit] cffi default: typo
Message-ID: <20141219154248.A3B4A1D27F5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1594:992bdcd2ed2c
Date: 2014-12-19 16:43 +0100
http://bitbucket.org/cffi/cffi/changeset/992bdcd2ed2c/

Log:	typo

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -408,7 +408,7 @@
 def test_invalid_indexing():
     p = new_primitive_type("int")
     x = cast(p, 42)
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises(TypeError, "x[0]")
 
 def test_default_str():
     BChar = new_primitive_type("char")

From noreply at buildbot.pypy.org  Fri Dec 19 17:10:08 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:10:08 +0100 (CET)
Subject: [pypy-commit] creflect default: next test
Message-ID: <20141219161008.9EB781C0483@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r203:2cd634f7b816
Date: 2014-12-19 15:11 +0100
http://bitbucket.org/cffi/creflect/changeset/2cd634f7b816/

Log:	next test

diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -947,10 +947,9 @@
     PyObject *result, *s;
 
     if (cd->c_type->ct_flags & CT_PRIMITIVE_ANY) {
-        abort();//XXX
-#if 0
         if (cd->c_type->ct_flags & CT_IS_ENUM) {
-            s = convert_cdata_to_enum_string(cd, 1);
+            abort();
+            //XXX s = convert_cdata_to_enum_string(cd, 1);
         }
         else if (cd->c_type->ct_flags & CT_IS_LONGDOUBLE) {
             long double lvalue;
@@ -967,7 +966,6 @@
             s = PyObject_Repr(o);
             Py_DECREF(o);
         }
-#endif
     }
     else if ((cd->c_type->ct_flags & CT_ARRAY) && cd->c_type->ct_length < 0) {
         s = PyString_FromFormat("sliced length %zd", get_array_length(cd));
diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
--- a/zeffir/test/test_c.py
+++ b/zeffir/test/test_c.py
@@ -55,3 +55,18 @@
     assert p.kind == "primitive"
     assert p.cname == "signed char"
     check_dir(p, ['cname', 'kind'])
+
+def test_cast_to_signed_char():
+    ffi = support.new_ffi()
+    t = ffi.typeof("signed char")
+    x = ffi.cast(t, -65 + 17*256)
+    assert repr(x) == "<cdata 'signed char' -65>"
+    assert repr(type(x)) == "<%s 'zeffir.CData'>" % type_or_class
+    assert int(x) == -65
+    x = ffi.cast(t, -66 + (1<<199)*256)
+    assert repr(x) == "<cdata 'signed char' -66>"
+    assert int(x) == -66
+    assert (x == ffi.cast(t, -66)) is False
+    assert (x != ffi.cast(t, -66)) is True
+    assert (x == ffi.cast("short", -66)) is False
+    assert (x != ffi.cast("short", -66)) is True

From noreply at buildbot.pypy.org  Fri Dec 19 17:10:09 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:10:09 +0100 (CET)
Subject: [pypy-commit] creflect default: more tests
Message-ID: <20141219161009.B9D2C1C0483@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r204:b1521d66a224
Date: 2014-12-19 15:23 +0100
http://bitbucket.org/cffi/creflect/changeset/b1521d66a224/

Log:	more tests

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -116,8 +116,7 @@
 static _crx_type_t *zef_get_char_type(_crx_builder_t *cb)
 {
     return _zef_primitive(cb, sizeof(char), "char",
-                          CT_PRIMITIVE_SIGNED | CT_PRIMITIVE_CHAR |
-                          CT_PRIMITIVE_FITS_LONG);
+                          CT_PRIMITIVE_CHAR | CT_PRIMITIVE_FITS_LONG);
 }
 
 static _crx_type_t *zef_get_bool_type(_crx_builder_t *cb)
@@ -289,9 +288,7 @@
     ct->ct_flags = CT_POINTER;
     if (totype->ct_flags & CT_VOID)
         ct->ct_flags |= CT_IS_VOID_PTR;
-    if ((totype->ct_flags & CT_VOID) ||
-        ((totype->ct_flags & CT_PRIMITIVE_CHAR) &&
-         totype->ct_size == sizeof(char)))
+    if (totype->ct_flags & (CT_VOID | CT_PRIMITIVE_CHAR))
         ct->ct_flags |= CT_CAST_ANYTHING;   /* 'void *' or 'char *' only */
 
     put_cached_type(types_dict, name_obj, ct);
diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1087,12 +1087,8 @@
     }
     else if (cd->c_type->ct_flags & CT_PRIMITIVE_CHAR) {
         /*READ(cd->c_data, cd->c_type->ct_size)*/
-        if (cd->c_type->ct_size == sizeof(char))
-            return PyInt_FromLong((unsigned char)cd->c_data[0]);
-#ifdef HAVE_WCHAR_H
-        else
-            return PyInt_FromLong((long)*(wchar_t *)cd->c_data);
-#endif
+        assert(cd->c_type->ct_size == sizeof(char));
+        return PyInt_FromLong((unsigned char)cd->c_data[0]);
     }
     else if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
         PyObject *o = cdata_float(cd);
@@ -1826,8 +1822,9 @@
             return NULL;
         return new_simple_cdata((char *)(Py_intptr_t)value, ct);
     }
-    else if (ct->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED
-                             |CT_PRIMITIVE_CHAR)) {
+    else if (ct->ct_flags & (CT_PRIMITIVE_SIGNED |
+                             CT_PRIMITIVE_UNSIGNED |
+                             CT_PRIMITIVE_CHAR)) {
         /* cast to an integer type or a char */
         return (PyObject *)cast_to_integer_or_char(ct, ob);
     }
diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
--- a/zeffir/test/test_c.py
+++ b/zeffir/test/test_c.py
@@ -70,3 +70,86 @@
     assert (x != ffi.cast(t, -66)) is True
     assert (x == ffi.cast("short", -66)) is False
     assert (x != ffi.cast("short", -66)) is True
+
+def test_sizeof_type():
+    ffi = support.new_ffi()
+    py.test.raises(TypeError, ffi.sizeof, 42.5)
+    assert ffi.sizeof("short") == 2
+    assert ffi.sizeof(ffi.typeof("short")) == 2
+
+def test_integer_types():
+    ffi = support.new_ffi()
+    for name in ['signed char', 'short', 'int', 'long', 'long long']:
+        p = ffi.typeof(name)
+        size = ffi.sizeof(p)
+        min = -(1 << (8*size-1))
+        max = (1 << (8*size-1)) - 1
+        assert int(ffi.cast(p, min)) == min
+        assert int(ffi.cast(p, max)) == max
+        assert int(ffi.cast(p, min - 1)) == max
+        assert int(ffi.cast(p, max + 1)) == min
+        py.test.raises(TypeError, ffi.cast, p, None)
+        assert long(ffi.cast(p, min - 1)) == max
+        assert int(ffi.cast(p, b'\x08')) == 8
+    for name in ['char', 'short', 'int', 'long', 'long long']:
+        p = ffi.typeof('unsigned ' + name)
+        size = ffi.sizeof(p)
+        max = (1 << (8*size)) - 1
+        assert int(ffi.cast(p, 0)) == 0
+        assert int(ffi.cast(p, max)) == max
+        assert int(ffi.cast(p, -1)) == max
+        assert int(ffi.cast(p, max + 1)) == 0
+        assert long(ffi.cast(p, -1)) == max
+        assert int(ffi.cast(p, b'\xFE')) == 254
+
+def test_no_float_on_int_types():
+    ffi = support.new_ffi()
+    py.test.raises(TypeError, float, ffi.cast('long', 42))
+    py.test.raises(TypeError, complex, ffi.cast('long', 42))
+
+def test_float_types():
+    ffi = support.new_ffi()
+    INF = 1E200 * 1E200
+    for name in ["float", "double"]:
+        p = ffi.typeof(name)
+        assert bool(ffi.cast(p, 0))
+        assert bool(ffi.cast(p, INF))
+        assert bool(ffi.cast(p, -INF))
+        assert int(ffi.cast(p, -150)) == -150
+        assert int(ffi.cast(p, 61.91)) == 61
+        assert long(ffi.cast(p, 61.91)) == 61
+        assert type(int(ffi.cast(p, 61.91))) is int
+        assert type(int(ffi.cast(p, 1E22))) is long
+        assert type(long(ffi.cast(p, 61.91))) is long
+        assert type(long(ffi.cast(p, 1E22))) is long
+        py.test.raises(OverflowError, int, ffi.cast(p, INF))
+        py.test.raises(OverflowError, int, ffi.cast(p, -INF))
+        assert float(ffi.cast(p, 1.25)) == 1.25
+        assert float(ffi.cast(p, INF)) == INF
+        assert float(ffi.cast(p, -INF)) == -INF
+        if name == "float":
+            assert float(ffi.cast(p, 1.1)) != 1.1     # rounding error
+            assert float(ffi.cast(p, 1E200)) == INF   # limited range
+
+        assert ffi.cast(p, -1.1) != ffi.cast(p, -1.1)
+        assert repr(float(ffi.cast(p, -0.0))) == '-0.0'
+        assert float(ffi.cast(p, b'\x09')) == 9.0
+        assert float(ffi.cast(p, True)) == 1.0
+        py.test.raises(TypeError, ffi.cast, p, None)
+
+def test_complex_types():
+    py.test.skip("later")
+
+def test_character_type():
+    ffi = support.new_ffi()
+    p = ffi.typeof("char")
+    assert bool(ffi.cast(p, '\x00'))
+    assert ffi.cast(p, '\x00') != ffi.cast(p, -17*256)
+    assert int(ffi.cast(p, 'A')) == 65
+    assert long(ffi.cast(p, 'A')) == 65
+    assert type(int(ffi.cast(p, 'A'))) is int
+    assert type(long(ffi.cast(p, 'A'))) is long
+    assert str(ffi.cast(p, 'A')) == repr(ffi.cast(p, 'A'))
+    assert repr(ffi.cast(p, 'A')) == "<cdata 'char' %s'A'>" % mandatory_b_prefix
+    assert repr(ffi.cast(p, 255)) == r"<cdata 'char' %s'\xff'>" % mandatory_b_prefix
+    assert repr(ffi.cast(p, 0)) == r"<cdata 'char' %s'\x00'>" % mandatory_b_prefix

From noreply at buildbot.pypy.org  Fri Dec 19 17:10:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:10:10 +0100 (CET)
Subject: [pypy-commit] creflect default: more tests
Message-ID: <20141219161010.BC5D41C0483@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r205:43aafe465a8b
Date: 2014-12-19 15:26 +0100
http://bitbucket.org/cffi/creflect/changeset/43aafe465a8b/

Log:	more tests

diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
--- a/zeffir/test/test_c.py
+++ b/zeffir/test/test_c.py
@@ -153,3 +153,41 @@
     assert repr(ffi.cast(p, 'A')) == "<cdata 'char' %s'A'>" % mandatory_b_prefix
     assert repr(ffi.cast(p, 255)) == r"<cdata 'char' %s'\xff'>" % mandatory_b_prefix
     assert repr(ffi.cast(p, 0)) == r"<cdata 'char' %s'\x00'>" % mandatory_b_prefix
+
+def test_pointer_type():
+    ffi = support.new_ffi()
+    p = ffi.typeof("int")
+    assert repr(p) == "<ctype 'int'>"
+    p = ffi.typeof("int*")
+    assert repr(p) == "<ctype 'int *'>"
+    p = ffi.typeof("int**")
+    assert repr(p) == "<ctype 'int * *'>"
+    p = ffi.typeof("int***")
+    assert repr(p) == "<ctype 'int * * *'>"
+
+def test_inspect_pointer_type():
+    ffi = support.new_ffi()
+    p1 = ffi.typeof("int")
+    p2 = ffi.typeof("int*")
+    assert p2.kind == "pointer"
+    assert p2.cname == "int *"
+    assert p2.item is p1
+    check_dir(p2, ['cname', 'kind', 'item'])
+    p3 = ffi.typeof("int**")
+    assert p3.item is p2
+
+def test_pointer_to_int():
+    ffi = support.new_ffi()
+    py.test.raises(TypeError, ffi.new, "int")
+    py.test.raises(TypeError, ffi.new, "int", None)
+    py.test.raises(TypeError, ffi.new, "int", 42)
+    p = ffi.new("int*")
+    assert repr(p) == "<cdata 'int *' owning %d bytes>" % size_of_int()
+    p = ffi.new("int*", None)
+    assert repr(p) == "<cdata 'int *' owning %d bytes>" % size_of_int()
+    p = ffi.new("int*", 5000)
+    assert repr(p) == "<cdata 'int *' owning %d bytes>" % size_of_int()
+    q = ffi.cast("int *", p)
+    assert repr(q).startswith("<cdata 'int *' 0x")
+    assert p == q
+    assert hash(p) == hash(q)

From noreply at buildbot.pypy.org  Fri Dec 19 17:10:11 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:10:11 +0100 (CET)
Subject: [pypy-commit] creflect default: more tests
Message-ID: <20141219161011.BCEC01C0483@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r206:98742ee6ef39
Date: 2014-12-19 15:31 +0100
http://bitbucket.org/cffi/creflect/changeset/98742ee6ef39/

Log:	more tests

diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
--- a/zeffir/test/test_c.py
+++ b/zeffir/test/test_c.py
@@ -191,3 +191,34 @@
     assert repr(q).startswith("<cdata 'int *' 0x")
     assert p == q
     assert hash(p) == hash(q)
+
+def test_pointer_bool():
+    ffi = support.new_ffi()
+    p = ffi.cast("int*", 0)
+    assert bool(p) is False
+    p = ffi.cast("int*", 42)
+    assert bool(p) is True
+
+def test_pointer_to_pointer():
+    ffi = support.new_ffi()
+    p = ffi.new("int**", None)
+    assert repr(p) == "<cdata 'int * *' owning %d bytes>" % size_of_ptr()
+
+def test_reading_pointer_to_int():
+    ffi = support.new_ffi()
+    p = ffi.new("int *")
+    assert p[0] == 0
+    p = ffi.new("int *", 5000)
+    assert p[0] == 5000
+    py.test.raises(IndexError, "p[1]")
+    py.test.raises(IndexError, "p[-1]")
+
+def test_reading_pointer_to_float():
+    ffi = support.new_ffi()
+    py.test.raises(TypeError, ffi.new, "float")
+    p = ffi.new("float *", None)
+    assert p[0] == 0.0 and type(p[0]) is float
+    p = ffi.new("float *", 1.25)
+    assert p[0] == 1.25 and type(p[0]) is float
+    p = ffi.new("float *", 1.1)
+    assert p[0] != 1.1 and abs(p[0] - 1.1) < 1E-5   # rounding errors

From noreply at buildbot.pypy.org  Fri Dec 19 17:10:12 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:10:12 +0100 (CET)
Subject: [pypy-commit] creflect default: Change ffi.new/ffi.cast on
 primitive types. Now ffi.cast() generally returns
Message-ID: <20141219161012.C61101C0483@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r207:4783e7b29a9b
Date: 2014-12-19 16:29 +0100
http://bitbucket.org/cffi/creflect/changeset/4783e7b29a9b/

Log:	Change ffi.new/ffi.cast on primitive types. Now ffi.cast()
	generally returns the Python object; to get a cdata object, use
	ffi.new().

diff --git a/zeffir/cdata.c b/zeffir/cdata.c
--- a/zeffir/cdata.c
+++ b/zeffir/cdata.c
@@ -1762,10 +1762,9 @@
     return res;
 }
 
-static CDataObject *cast_to_integer_or_char(CTypeDescrObject *ct, PyObject *ob)
+static PyObject *cast_to_integer_or_char(CTypeDescrObject *ct, PyObject *ob)
 {
     unsigned PY_LONG_LONG value;
-    CDataObject *cd;
 
     if (CData_Check(ob) &&
             ((CDataObject *)ob)->c_type->ct_flags & (CT_POINTER | CT_ARRAY)) {
@@ -1791,12 +1790,42 @@
         if (value == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())
             return NULL;
     }
-    if (ct->ct_flags & CT_IS_BOOL)
-        value = !!value;
-    cd = _new_casted_primitive(ct);
-    if (cd != NULL)
-        write_raw_integer_data(cd->c_data, value, ct->ct_size);
-    return cd;
+    if (ct->ct_flags & CT_PRIMITIVE_SIGNED) {
+        switch (ct->ct_size) {
+        case sizeof(char):
+            if (ct->ct_flags & CT_IS_BOOL)
+                return PyBool_FromLong(value != 0);
+            return PyInt_FromLong((signed char)value);
+        case sizeof(short):
+            return PyInt_FromLong((short)value);
+        case sizeof(int):
+            return PyInt_FromLong((int)value);
+        case sizeof(long):
+            return PyInt_FromLong((long)value);
+        default:
+            return PyLong_FromLongLong((PY_LONG_LONG)value);
+        }
+    }
+    else if (ct->ct_flags & CT_PRIMITIVE_CHAR) {
+        char c = (char)value;
+        return PyString_FromStringAndSize(&c, 1);
+    }
+    else {
+        switch (ct->ct_size) {
+        case sizeof(char):
+            return PyInt_FromLong((unsigned char)value);
+        case sizeof(short):
+            return PyInt_FromLong((unsigned short)value);
+        case sizeof(int):
+            if (sizeof(int) < sizeof(long))
+                return PyInt_FromLong((unsigned int)value);
+            /* else, fall-through */
+        case sizeof(long):
+            return PyLong_FromUnsignedLong((unsigned long)value);
+        default:
+            return PyLong_FromUnsignedLongLong(value);
+        }
+    }
 }
 
 static PyObject *do_cast(CTypeDescrObject *ct, PyObject *ob)
@@ -1826,7 +1855,7 @@
                              CT_PRIMITIVE_UNSIGNED |
                              CT_PRIMITIVE_CHAR)) {
         /* cast to an integer type or a char */
-        return (PyObject *)cast_to_integer_or_char(ct, ob);
+        return cast_to_integer_or_char(ct, ob);
     }
     else if (ct->ct_flags & CT_PRIMITIVE_FLOAT) {
         /* cast to a float */
@@ -1847,40 +1876,38 @@
             Py_INCREF(io);
         }
 
+        if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
+            (CData_Check(io) &&
+             (((CDataObject *)io)->c_type->ct_flags & CT_IS_LONGDOUBLE))) {
+            return io;   /* no more convertion needed; don't loose precision */
+        }
+
         if (PyBytes_Check(io)) {
             if (PyBytes_GET_SIZE(io) != 1) {
                 Py_DECREF(io);
                 goto cannot_cast;
             }
             value = (unsigned char)PyBytes_AS_STRING(io)[0];
+            Py_DECREF(io);
         }
-        else if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
-                 CData_Check(io) &&
-                 (((CDataObject *)io)->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
-            long double lvalue;
-            char *data = ((CDataObject *)io)->c_data;
-            /*READ(data, sizeof(long double)*/
-            lvalue = read_raw_longdouble_data(data);
+        else {
+            value = PyFloat_AsDouble(io);
+            Py_DECREF(io);
+            if (value == -1.0 && PyErr_Occurred())
+                return NULL;
+        }
+
+        if (ct->ct_flags & CT_IS_LONGDOUBLE) {
             cd = _new_casted_primitive(ct);
             if (cd != NULL)
-                write_raw_longdouble_data(cd->c_data, lvalue);
+                write_raw_longdouble_data(cd->c_data, value);
             return (PyObject *)cd;
         }
         else {
-            value = PyFloat_AsDouble(io);
+            if (ct->ct_size < sizeof(double))
+                value = (float)value;   /* explicitly loose precision */
+            return PyFloat_FromDouble(value);
         }
-        Py_DECREF(io);
-        if (value == -1.0 && PyErr_Occurred())
-            return NULL;
-
-        cd = _new_casted_primitive(ct);
-        if (cd != NULL) {
-            if (!(ct->ct_flags & CT_IS_LONGDOUBLE))
-                write_raw_float_data(cd->c_data, value, ct->ct_size);
-            else
-                write_raw_longdouble_data(cd->c_data, (long double)value);
-        }
-        return (PyObject *)cd;
     }
     else {
         PyErr_Format(PyExc_TypeError, "cannot cast to ctype '%s'",
diff --git a/zeffir/ffi_obj.c b/zeffir/ffi_obj.c
--- a/zeffir/ffi_obj.c
+++ b/zeffir/ffi_obj.c
@@ -240,10 +240,14 @@
             }
         }
     }
+    else if (ct->ct_flags & CT_PRIMITIVE_ANY) {
+        cd = _new_casted_primitive(ct);
+        datasize = ct->ct_size;
+        goto initialize_casted_primitive;
+    }
     else {
         PyErr_Format(PyExc_TypeError,
-                     "expected a pointer, struct, union, or array ctype, "
-                     "got '%s'", ct->ct_name);
+                     "cannot create cdata '%s' objects", ct->ct_name);
         return NULL;
     }
 
@@ -255,6 +259,7 @@
     if (explicitlength >= 0)
         ((CDataObject_own_length*)cd)->length = explicitlength;
 
+ initialize_casted_primitive:
     memset(cd->c_data, 0, datasize);
     if (init != Py_None) {
         if (convert_from_object(cd->c_data,
diff --git a/zeffir/test/test_basic.py b/zeffir/test/test_basic.py
--- a/zeffir/test/test_basic.py
+++ b/zeffir/test/test_basic.py
@@ -55,6 +55,20 @@
     q = ffi.cast("int *", p)
     assert q[1] in [0x61626364, 0x64636261]
 
+    q = ffi.cast("int", 42)
+    assert type(q) is int and q == 42
+    q = ffi.cast("float", 42)
+    assert type(q) is float and q == 42.0
+    q = ffi.cast("long double", 42)
+    assert repr(q).startswith("<cdata 'long double' ")
+
+    q = ffi.new("int", 42)
+    assert repr(q) == "<cdata 'int' 42>"
+    q = ffi.new("float", 42)
+    assert repr(q) == "<cdata 'float' 42.0>"
+    q = ffi.new("long double", 42)
+    assert repr(q).startswith("<cdata 'long double' ")
+
 def test_getctype():
     ffi = support.new_ffi()
     assert ffi.getctype("int") == "int"
diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
--- a/zeffir/test/test_c.py
+++ b/zeffir/test/test_c.py
@@ -60,16 +60,17 @@
     ffi = support.new_ffi()
     t = ffi.typeof("signed char")
     x = ffi.cast(t, -65 + 17*256)
-    assert repr(x) == "<cdata 'signed char' -65>"
+    assert type(x) is int and x == -65
+    x = ffi.cast(t, -66 + (1<<199)*256)
+    assert type(x) is int and x == -66
+    x = ffi.new(t, -66)
+    assert repr(x) == "<cdata 'signed char' -66>"
     assert repr(type(x)) == "<%s 'zeffir.CData'>" % type_or_class
-    assert int(x) == -65
-    x = ffi.cast(t, -66 + (1<<199)*256)
-    assert repr(x) == "<cdata 'signed char' -66>"
     assert int(x) == -66
-    assert (x == ffi.cast(t, -66)) is False
-    assert (x != ffi.cast(t, -66)) is True
-    assert (x == ffi.cast("short", -66)) is False
-    assert (x != ffi.cast("short", -66)) is True
+    assert (x == ffi.new(t, -66)) is False
+    assert (x != ffi.new(t, -66)) is True
+    assert (x == ffi.new("short", -66)) is False
+    assert (x != ffi.new("short", -66)) is True
 
 def test_sizeof_type():
     ffi = support.new_ffi()
@@ -84,75 +85,120 @@
         size = ffi.sizeof(p)
         min = -(1 << (8*size-1))
         max = (1 << (8*size-1)) - 1
-        assert int(ffi.cast(p, min)) == min
-        assert int(ffi.cast(p, max)) == max
-        assert int(ffi.cast(p, min - 1)) == max
-        assert int(ffi.cast(p, max + 1)) == min
+        assert type(ffi.cast(p, min)) in (int, long)
+        assert ffi.cast(p, min) == min
+        assert ffi.cast(p, max) == max
+        assert ffi.cast(p, min - 1) == max
+        assert ffi.cast(p, max + 1) == min
         py.test.raises(TypeError, ffi.cast, p, None)
-        assert long(ffi.cast(p, min - 1)) == max
-        assert int(ffi.cast(p, b'\x08')) == 8
+        assert ffi.cast(p, b'\x08') == 8
     for name in ['char', 'short', 'int', 'long', 'long long']:
         p = ffi.typeof('unsigned ' + name)
         size = ffi.sizeof(p)
         max = (1 << (8*size)) - 1
-        assert int(ffi.cast(p, 0)) == 0
-        assert int(ffi.cast(p, max)) == max
-        assert int(ffi.cast(p, -1)) == max
-        assert int(ffi.cast(p, max + 1)) == 0
-        assert long(ffi.cast(p, -1)) == max
-        assert int(ffi.cast(p, b'\xFE')) == 254
+        assert type(ffi.cast(p, min)) in (int, long)
+        assert ffi.cast(p, 0) == 0
+        assert ffi.cast(p, max) == max
+        assert ffi.cast(p, -1) == max
+        assert ffi.cast(p, max + 1) == 0
+        assert ffi.cast(p, b'\xFE') == 254
+
+def test_integer_types_new():
+    ffi = support.new_ffi()
+    for name in ['signed char', 'short', 'int', 'long', 'long long']:
+        p = ffi.typeof(name)
+        size = ffi.sizeof(p)
+        min = -(1 << (8*size-1))
+        max = (1 << (8*size-1)) - 1
+        assert bool(ffi.new(p, 0))
+        assert int(ffi.new(p, min)) == min
+        assert int(ffi.new(p, max)) == max
+        py.test.raises(OverflowError, ffi.new, p, min - 1)
+        py.test.raises(OverflowError, ffi.new, p, max + 1)
+        py.test.raises(TypeError, ffi.new, p, b'\x08')
+    for name in ['char', 'short', 'int', 'long', 'long long']:
+        p = ffi.typeof('unsigned ' + name)
+        size = ffi.sizeof(p)
+        max = (1 << (8*size)) - 1
+        assert bool(ffi.new(p, 0))
+        assert int(ffi.new(p, 0)) == 0
+        assert int(ffi.new(p, max)) == max
+        py.test.raises(OverflowError, ffi.new, p, -1)
+        py.test.raises(OverflowError, ffi.new, p, max + 1)
+        py.test.raises(TypeError, ffi.new, p, b'\xFE')
 
 def test_no_float_on_int_types():
     ffi = support.new_ffi()
-    py.test.raises(TypeError, float, ffi.cast('long', 42))
-    py.test.raises(TypeError, complex, ffi.cast('long', 42))
+    py.test.raises(TypeError, float, ffi.new('long', 42))
+    py.test.raises(TypeError, complex, ffi.new('long', 42))
 
 def test_float_types():
     ffi = support.new_ffi()
     INF = 1E200 * 1E200
     for name in ["float", "double"]:
         p = ffi.typeof(name)
-        assert bool(ffi.cast(p, 0))
-        assert bool(ffi.cast(p, INF))
-        assert bool(ffi.cast(p, -INF))
-        assert int(ffi.cast(p, -150)) == -150
-        assert int(ffi.cast(p, 61.91)) == 61
-        assert long(ffi.cast(p, 61.91)) == 61
-        assert type(int(ffi.cast(p, 61.91))) is int
-        assert type(int(ffi.cast(p, 1E22))) is long
-        assert type(long(ffi.cast(p, 61.91))) is long
-        assert type(long(ffi.cast(p, 1E22))) is long
-        py.test.raises(OverflowError, int, ffi.cast(p, INF))
-        py.test.raises(OverflowError, int, ffi.cast(p, -INF))
-        assert float(ffi.cast(p, 1.25)) == 1.25
-        assert float(ffi.cast(p, INF)) == INF
-        assert float(ffi.cast(p, -INF)) == -INF
+        assert type(ffi.cast(p, 0)) is float
+        assert ffi.cast(p, 0) == 0.0
+        assert ffi.cast(p, INF) == INF
+        assert ffi.cast(p, -INF) == -INF
+        assert ffi.cast(p, -150) == -150
         if name == "float":
-            assert float(ffi.cast(p, 1.1)) != 1.1     # rounding error
-            assert float(ffi.cast(p, 1E200)) == INF   # limited range
+            assert ffi.cast(p, 1.1) != 1.1     # rounding error
+            assert ffi.cast(p, 1E200) == INF   # limited range
+        else:
+            assert ffi.cast(p, 61.91) == 61.91
 
-        assert ffi.cast(p, -1.1) != ffi.cast(p, -1.1)
-        assert repr(float(ffi.cast(p, -0.0))) == '-0.0'
-        assert float(ffi.cast(p, b'\x09')) == 9.0
-        assert float(ffi.cast(p, True)) == 1.0
+        assert repr(ffi.cast(p, -0.0)) == '-0.0'
+        assert ffi.cast(p, b'\x09') == 9.0
+        assert ffi.cast(p, True) == 1.0
         py.test.raises(TypeError, ffi.cast, p, None)
 
+def test_float_types_new():
+    ffi = support.new_ffi()
+    INF = 1E200 * 1E200
+    for name in ["float", "double"]:
+        p = ffi.typeof(name)
+        assert bool(ffi.new(p, 0))
+        assert bool(ffi.new(p, INF))
+        assert bool(ffi.new(p, -INF))
+        assert int(ffi.new(p, -150)) == -150
+        assert int(ffi.new(p, 61.91)) == 61
+        assert long(ffi.new(p, 61.91)) == 61
+        assert type(int(ffi.new(p, 61.91))) is int
+        assert type(int(ffi.new(p, 1E22))) is long
+        assert type(long(ffi.new(p, 61.91))) is long
+        assert type(long(ffi.new(p, 1E22))) is long
+        py.test.raises(OverflowError, int, ffi.new(p, INF))
+        py.test.raises(OverflowError, int, ffi.new(p, -INF))
+        assert float(ffi.new(p, 1.25)) == 1.25
+        assert float(ffi.new(p, INF)) == INF
+        assert float(ffi.new(p, -INF)) == -INF
+        if name == "float":
+            assert float(ffi.new(p, 1.1)) != 1.1     # rounding error
+            assert float(ffi.new(p, 1E200)) == INF   # limited range
+
+        assert ffi.new(p, -1.1) != ffi.new(p, -1.1)
+        assert repr(float(ffi.new(p, -0.0))) == '-0.0'
+        py.test.raises(TypeError, ffi.new, p, b'\x09')
+        assert float(ffi.new(p, True)) == 1.0
+
 def test_complex_types():
     py.test.skip("later")
 
 def test_character_type():
     ffi = support.new_ffi()
     p = ffi.typeof("char")
-    assert bool(ffi.cast(p, '\x00'))
-    assert ffi.cast(p, '\x00') != ffi.cast(p, -17*256)
-    assert int(ffi.cast(p, 'A')) == 65
-    assert long(ffi.cast(p, 'A')) == 65
-    assert type(int(ffi.cast(p, 'A'))) is int
-    assert type(long(ffi.cast(p, 'A'))) is long
-    assert str(ffi.cast(p, 'A')) == repr(ffi.cast(p, 'A'))
-    assert repr(ffi.cast(p, 'A')) == "<cdata 'char' %s'A'>" % mandatory_b_prefix
-    assert repr(ffi.cast(p, 255)) == r"<cdata 'char' %s'\xff'>" % mandatory_b_prefix
-    assert repr(ffi.cast(p, 0)) == r"<cdata 'char' %s'\x00'>" % mandatory_b_prefix
+    assert bool(ffi.new(p, '\x00'))
+    assert type(ffi.cast(p, 50505)) is bytes
+    assert ffi.new(p, '\x00') != ffi.new(p, '\x00')
+    assert ffi.cast(p, 65 + 1233*256) == 'A'
+    assert int(ffi.new(p, 'A')) == 65
+    assert long(ffi.new(p, 'A')) == 65
+    assert type(int(ffi.new(p, 'A'))) is int
+    assert type(long(ffi.new(p, 'A'))) is long
+    py.test.raises(TypeError, ffi.new, p, 65)
+    assert str(ffi.new(p, 'A')) == repr(ffi.new(p, 'A'))
+    assert repr(ffi.new(p, '\xff')) == r"<cdata 'char' %s'\xff'>" % mandatory_b_prefix
 
 def test_pointer_type():
     ffi = support.new_ffi()
@@ -178,9 +224,6 @@
 
 def test_pointer_to_int():
     ffi = support.new_ffi()
-    py.test.raises(TypeError, ffi.new, "int")
-    py.test.raises(TypeError, ffi.new, "int", None)
-    py.test.raises(TypeError, ffi.new, "int", 42)
     p = ffi.new("int*")
     assert repr(p) == "<cdata 'int *' owning %d bytes>" % size_of_int()
     p = ffi.new("int*", None)
@@ -215,7 +258,6 @@
 
 def test_reading_pointer_to_float():
     ffi = support.new_ffi()
-    py.test.raises(TypeError, ffi.new, "float")
     p = ffi.new("float *", None)
     assert p[0] == 0.0 and type(p[0]) is float
     p = ffi.new("float *", 1.25)

From noreply at buildbot.pypy.org  Fri Dec 19 17:10:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:10:13 +0100 (CET)
Subject: [pypy-commit] creflect default: next tests
Message-ID: <20141219161013.D2A651C0483@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r208:7489cad8641a
Date: 2014-12-19 16:34 +0100
http://bitbucket.org/cffi/creflect/changeset/7489cad8641a/

Log:	next tests

diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
--- a/zeffir/test/test_c.py
+++ b/zeffir/test/test_c.py
@@ -264,3 +264,34 @@
     assert p[0] == 1.25 and type(p[0]) is float
     p = ffi.new("float *", 1.1)
     assert p[0] != 1.1 and abs(p[0] - 1.1) < 1E-5   # rounding errors
+
+def test_cast_float_to_int():
+    ffi = support.new_ffi()
+    for type in ["int", "unsigned int", "long", "unsigned long",
+                 "long long", "unsigned long long"]:
+        p = ffi.typeof(type)
+        assert ffi.cast(p, 4.2) == 4
+        py.test.raises(TypeError, ffi.new, type, 4.2)
+        py.test.raises(TypeError, ffi.new, type + "*", 4.2)
+
+def test_newp_integer_types():
+    ffi = support.new_ffi()
+    for name in ['signed char', 'short', 'int', 'long', 'long long']:
+        p = ffi.typeof(name)
+        pp = ffi.typeof(name + "*")
+        size = ffi.sizeof(p)
+        min = -(1 << (8*size-1))
+        max = (1 << (8*size-1)) - 1
+        assert ffi.new(pp, min)[0] == min
+        assert ffi.new(pp, max)[0] == max
+        py.test.raises(OverflowError, ffi.new, pp, min - 1)
+        py.test.raises(OverflowError, ffi.new, pp, max + 1)
+    for name in ['char', 'short', 'int', 'long', 'long long']:
+        p = ffi.typeof('unsigned ' + name)
+        pp = ffi.typeof('unsigned ' + name + "*")
+        size = ffi.sizeof(p)
+        max = (1 << (8*size)) - 1
+        assert ffi.new(pp, 0)[0] == 0
+        assert ffi.new(pp, max)[0] == max
+        py.test.raises(OverflowError, ffi.new, pp, -1)
+        py.test.raises(OverflowError, ffi.new, pp, max + 1)

From noreply at buildbot.pypy.org  Fri Dec 19 17:10:14 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:10:14 +0100 (CET)
Subject: [pypy-commit] creflect default: more tests
Message-ID: <20141219161014.CB5991C0483@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r209:77b498c94a46
Date: 2014-12-19 16:58 +0100
http://bitbucket.org/cffi/creflect/changeset/77b498c94a46/

Log:	more tests

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -956,7 +956,7 @@
     _crx_qual_type result;
     const char *err = creflect_decl_parser(&builder.cb, str, &result);
     if (PyErr_Occurred()) {
-        if (PyErr_ExceptionMatches(ZefError)) {
+        if (PyErr_ExceptionMatches(ZefError) && err != NULL) {
             PyObject *exc, *val, *tb, *ps;
             PyErr_Fetch(&exc, &val, &tb);
             ps = PyObject_Str(val);
diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
--- a/zeffir/test/test_c.py
+++ b/zeffir/test/test_c.py
@@ -295,3 +295,124 @@
         assert ffi.new(pp, max)[0] == max
         py.test.raises(OverflowError, ffi.new, pp, -1)
         py.test.raises(OverflowError, ffi.new, pp, max + 1)
+
+def test_reading_pointer_to_char():
+    ffi = support.new_ffi()
+    p = ffi.new("char *", None)
+    assert p[0] == b'\x00'
+    p = ffi.new("char *", b'A')
+    assert p[0] == b'A'
+    py.test.raises(TypeError, ffi.new, "char *", 65)
+    py.test.raises(TypeError, ffi.new, "char *", b"foo")
+    py.test.raises(TypeError, ffi.new, "char *", u+"foo")
+    c = ffi.new("char", b'A')
+    assert str(c) == repr(c)
+    assert int(c) == ord(b'A')
+    py.test.raises(TypeError, ffi.cast, "char", b'foo')
+    py.test.raises(TypeError, ffi.cast, "char", u+'foo')
+    py.test.raises(TypeError, ffi.new, "char", b'foo')
+    py.test.raises(TypeError, ffi.new, "char", u+'foo')
+
+def test_reading_pointer_to_pointer():
+    ffi = support.new_ffi()
+    q = ffi.new("int *", 42)
+    assert q[0] == 42
+    p = ffi.new("int * *")
+    assert p[0] is not None
+    assert p[0] == ffi.cast("void *", 0)
+    assert p[0] == ffi.cast("char *", 0)
+    assert p[0] != None
+    assert repr(p[0]) == "<cdata 'int *' NULL>"
+    p[0] = q
+    assert p[0] != ffi.cast("void *", 0)
+    assert p[0] != ffi.cast("char *", 0)
+    assert p[0][0] == 42
+    q[0] += 1
+    assert p[0][0] == 43
+    p = ffi.new("int * *", q)
+    assert p[0][0] == 43
+
+def test_hash_differences():
+    ffi = support.new_ffi()
+    for i in range(1, 50):
+        x1 = ffi.new("char", chr(i))
+        x2 = ffi.new("int", i)
+        if hash(x1) != hash(x2):
+            break
+    else:
+        raise AssertionError("hashes are equal")
+    for i in range(1, 50):
+        if hash(ffi.new("float", i)) != hash(float(i)):
+            break
+    else:
+        raise AssertionError("hashes are equal")
+
+def test_no_len_on_nonarray():
+    ffi = support.new_ffi()
+    py.test.raises(TypeError, len, ffi.new("int", 42))
+
+def test_cmp_none():
+    ffi = support.new_ffi()
+    x = ffi.new("int", 42)
+    assert (x == None) is False
+    assert (x != None) is True
+    assert (x == ["hello"]) is False
+    assert (x != ["hello"]) is True
+    y = ffi.new("int", 0)
+    assert (y == None) is False
+
+def test_invalid_indexing():
+    ffi = support.new_ffi()
+    x = ffi.new("int", 42)
+    py.test.raises(TypeError, "x[0]")
+
+def test_default_str():
+    ffi = support.new_ffi()
+    x = ffi.new("char", 'A')
+    assert str(x) == repr(x)
+    x = ffi.new("int", 42)
+    assert str(x) == repr(x)
+    x = ffi.new("int[10]")
+    assert str(x) == repr(x)
+
+def test_cast_from_cdataint():
+    ffi = support.new_ffi()
+    x = ffi.new("int", 0)
+    y = ffi.cast("int *", x)
+    assert bool(y) is False
+    #
+    x = ffi.new("int", 42)
+    y = ffi.new("int", x)
+    assert int(x) == int(y) == 42 == ffi.cast("int", y)
+    x = ffi.new("char", '\xFE')
+    y = ffi.new("char", x)
+    assert int(x) == int(y) == 254 == ffi.cast("int", y)
+    x = ffi.new("double", 42.42)
+    y = ffi.new("double", x)
+    assert float(x) == float(y) == 42.42 == ffi.cast("double", y)
+
+def test_void_type():
+    ffi = support.new_ffi()
+    p = ffi.typeof("void")
+    assert p.kind == "void"
+    assert p.cname == "void"
+    check_dir(p, ['kind', 'cname'])
+
+def test_array_type():
+    ffi = support.new_ffi()
+    p = ffi.typeof("int")
+    assert repr(p) == "<ctype 'int'>"
+    #
+    py.test.raises(ffi.error, ffi.typeof, "int[-42]")
+    #
+    p1 = ffi.typeof("int[]")
+    assert repr(p1) == "<ctype 'int[]'>"
+    py.test.raises(ffi.error, ffi.typeof, "int[][]")
+    e = py.test.raises(ffi.error, ffi.typeof, "int[42][]")
+    assert str(e.value) == "array type has incomplete element type 'int[]'"
+    #
+    p2 = ffi.typeof(" int [ 25 ] [ 42 ] ")
+    assert repr(p2) == "<ctype 'int[25][42]'>"
+    #
+    py.test.raises(OverflowError, ffi.typeof, "int[%d]" % (sys.maxsize + 1))
+    py.test.raises(OverflowError, ffi.typeof, "int[%d]" % (sys.maxsize // 3))

From noreply at buildbot.pypy.org  Fri Dec 19 17:10:15 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:10:15 +0100 (CET)
Subject: [pypy-commit] creflect default: fix
Message-ID: <20141219161015.D16541C0483@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r210:84540aee69f5
Date: 2014-12-19 17:04 +0100
http://bitbucket.org/cffi/creflect/changeset/84540aee69f5/

Log:	fix

diff --git a/zeffir/builder.c b/zeffir/builder.c
--- a/zeffir/builder.c
+++ b/zeffir/builder.c
@@ -319,7 +319,6 @@
     CTypeDescrObject *ct;
     PyObject *name_obj;
     char extra_text[32];
-    Py_ssize_t length = (Py_ssize_t)len;
     Py_ssize_t arraysize;
 
     if (ctitem->ct_size < 0) {
@@ -328,10 +327,10 @@
         return NULL;
     }
 
-    if (length < 0)
+    if (len == (size_t)-1)
         sprintf(extra_text, "[]");
     else
-        sprintf(extra_text, "[%zd]", length);
+        sprintf(extra_text, "[%zu]", len);
 
     name_obj = combine_type_name(ctitem, extra_text);
     if (name_obj == NULL)
@@ -343,12 +342,13 @@
             goto done;
     }
 
-    if (length < 0) {
+    if (len == (size_t)-1) {
         arraysize = -1;
     }
     else {
-        arraysize = length * ctitem->ct_size;
-        if (length > 0 && (arraysize / length) != ctitem->ct_size) {
+        arraysize = ((Py_ssize_t)len) * ctitem->ct_size;
+        if ((Py_ssize_t)len < 0 ||
+                (len > 0 && (arraysize / (Py_ssize_t)len) != ctitem->ct_size)) {
             PyErr_SetString(PyExc_OverflowError,
                             "array size would overflow a Py_ssize_t");
             goto done;
@@ -366,20 +366,20 @@
         goto done;
 
     ct->ct_size = arraysize;
-    ct->ct_length = length;
+    ct->ct_length = (Py_ssize_t)len;
     ct->ct_flags = CT_ARRAY;
     ct->ct_itemdescr = ctitem;        Py_INCREF(ctitem);
     ct->ct_stuff = (PyObject *)ctptr; Py_INCREF(ctptr);
 
     if (cb != NULL) {
-        if (length < 0 && ctptr->ct_stuff == NULL) {
+        if (len == (size_t)-1 && ctptr->ct_stuff == NULL) {
             Py_INCREF(ct);
             ctptr->ct_stuff = (PyObject *)ct;
         }
         put_cached_type(get_types_dict(cb), name_obj, ct);
     }
     else {
-        assert(length < 0);
+        assert(len == (size_t)-1);
         assert(ctptr->ct_stuff == NULL);
         ctptr->ct_stuff = (PyObject *)ct;
     }
diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
--- a/zeffir/test/test_c.py
+++ b/zeffir/test/test_c.py
@@ -414,5 +414,8 @@
     p2 = ffi.typeof(" int [ 25 ] [ 42 ] ")
     assert repr(p2) == "<ctype 'int[25][42]'>"
     #
+    e = py.test.raises(ffi.error, ffi.typeof, "int[%d]" % (sys.maxsize ** 3))
+    assert str(e.value) == "at pos 4: number too large"
     py.test.raises(OverflowError, ffi.typeof, "int[%d]" % (sys.maxsize + 1))
+    py.test.raises(OverflowError, ffi.typeof, "int[%d]" % sys.maxsize)
     py.test.raises(OverflowError, ffi.typeof, "int[%d]" % (sys.maxsize // 3))

From noreply at buildbot.pypy.org  Fri Dec 19 17:10:16 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:10:16 +0100 (CET)
Subject: [pypy-commit] creflect default: more tests
Message-ID: <20141219161016.C6C2E1C0483@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r211:d45b3ca636ab
Date: 2014-12-19 17:10 +0100
http://bitbucket.org/cffi/creflect/changeset/d45b3ca636ab/

Log:	more tests

diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
--- a/zeffir/test/test_c.py
+++ b/zeffir/test/test_c.py
@@ -419,3 +419,102 @@
     py.test.raises(OverflowError, ffi.typeof, "int[%d]" % (sys.maxsize + 1))
     py.test.raises(OverflowError, ffi.typeof, "int[%d]" % sys.maxsize)
     py.test.raises(OverflowError, ffi.typeof, "int[%d]" % (sys.maxsize // 3))
+
+def test_inspect_array_type():
+    ffi = support.new_ffi()
+    p1 = ffi.typeof("int[]")
+    assert p1.kind == "array"
+    assert p1.cname == "int[]"
+    assert p1.item is ffi.typeof("int")
+    assert p1.length is None
+    check_dir(p1, ['cname', 'kind', 'item', 'length'])
+    p1 = ffi.typeof("int[42]")
+    assert p1.kind == "array"
+    assert p1.cname == "int[42]"
+    assert p1.item is ffi.typeof("int")
+    assert p1.length == 42
+    check_dir(p1, ['cname', 'kind', 'item', 'length'])
+
+def test_array_instance():
+    ffi = support.new_ffi()
+    LENGTH = 1423
+    p1 = ffi.typeof("int[%d]" % LENGTH)
+    a = ffi.new(p1)
+    assert repr(a) == "<cdata 'int[%d]' owning %d bytes>" % (
+        LENGTH, LENGTH * size_of_int())
+    assert len(a) == LENGTH
+    for i in range(LENGTH):
+        assert a[i] == 0
+    py.test.raises(IndexError, "a[LENGTH]")
+    py.test.raises(IndexError, "a[-1]")
+    for i in range(LENGTH):
+        a[i] = i * i + 1
+    for i in range(LENGTH):
+        assert a[i] == i * i + 1
+    e = py.test.raises(IndexError, "a[LENGTH+100] = 500")
+    assert ('(expected %d < %d)' % (LENGTH+100, LENGTH)) in str(e.value)
+    py.test.raises(TypeError, int, a)
+
+def test_array_of_unknown_length_instance():
+    ffi = support.new_ffi()
+    p1 = ffi.typeof("int[]")
+    py.test.raises(TypeError, ffi.new, p1)
+    py.test.raises(ValueError, ffi.new, p1, -42)
+    a = ffi.new(p1, 42)
+    assert len(a) == 42
+    for i in range(42):
+        a[i] -= i
+    for i in range(42):
+        assert a[i] == -i
+    py.test.raises(IndexError, "a[42]")
+    py.test.raises(IndexError, "a[-1]")
+    py.test.raises(IndexError, "a[42] = 123")
+    py.test.raises(IndexError, "a[-1] = 456")
+
+def test_array_of_unknown_length_instance_with_initializer():
+    ffi = support.new_ffi()
+    a = ffi.new("int[]", list(range(42)))
+    assert len(a) == 42
+    a = ffi.new("int[]", tuple(range(142)))
+    assert len(a) == 142
+    assert a[99] == 99
+
+def test_array_initializer():
+    ffi = support.new_ffi()
+    a = ffi.new("int[42]", list(range(100, 142)))
+    for i in range(42):
+        assert a[i] == 100 + i
+    #
+    a = ffi.new("int[44]", tuple(range(100, 142)))
+    for i in range(42):
+        assert a[i] == 100 + i
+    assert a[42] == 0      # extra uninitialized items
+    assert a[43] == 0
+
+def test_array_add():
+    ffi = support.new_ffi()
+    p2 = ffi.typeof("int[3][5]")
+    a = ffi.new(p2, [list(range(n, n+5)) for n in [100, 200, 300]])
+    assert repr(a) == "<cdata 'int[3][5]' owning %d bytes>" % (
+        3*5*size_of_int(),)
+    assert repr(a + 0).startswith("<cdata 'int(*)[5]' 0x")
+    assert 0 + a == a + 0 != 1 + a == a + 1
+    assert repr(a[0]).startswith("<cdata 'int[5]' 0x")
+    assert repr((a + 0)[0]).startswith("<cdata 'int[5]' 0x")
+    assert repr(a[0] + 0).startswith("<cdata 'int *' 0x")
+    assert type(a[0][0]) is int
+    assert type((a[0] + 0)[0]) is int
+
+def test_array_sub():
+    ffi = support.new_ffi()
+    a = ffi.new("int[5]")
+    p = a + 1
+    assert p - a == 1
+    assert p - (a+0) == 1
+    assert a == (p - 1)
+    q = ffi.new("short *")
+    py.test.raises(TypeError, "p - q")
+    py.test.raises(TypeError, "q - p")
+    py.test.raises(TypeError, "a - q")
+    e = py.test.raises(TypeError, "q - a")
+    assert str(e.value) == "cannot subtract cdata 'short *' and cdata 'int *'"

From noreply at buildbot.pypy.org  Fri Dec 19 17:28:16 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:28:16 +0100 (CET)
Subject: [pypy-commit] creflect default: more test
Message-ID: <20141219162816.9C7AD1C11BF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r212:0ef23a1fa399
Date: 2014-12-19 17:13 +0100
http://bitbucket.org/cffi/creflect/changeset/0ef23a1fa399/

Log:	more test

diff --git a/zeffir/test/test_c.py b/zeffir/test/test_c.py
--- a/zeffir/test/test_c.py
+++ b/zeffir/test/test_c.py
@@ -518,3 +518,22 @@
     py.test.raises(TypeError, "a - q")
     e = py.test.raises(TypeError, "q - a")
     assert str(e.value) == "cannot subtract cdata 'short *' and cdata 'int *'"
+
+def test_cast_between_pointers():
+    ffi = support.new_ffi()
+    BIntP = ffi.typeof("int *")
+    a = ffi.new("int[]", [40, 41, 42, 43, 44])
+    b = ffi.cast("short *", a)
+    c = ffi.cast(BIntP, b)
+    assert c[3] == 43
+    d = ffi.cast("long long", c)
+    e = ffi.cast(BIntP, d)
+    assert e[3] == 43
+    f = ffi.cast(BIntP, d)
+    assert f[3] == 43
+    #
+    b = ffi.cast("short *", 0)
+    assert not b
+    c = ffi.cast(BIntP, b)
+    assert not c
+    assert ffi.cast("long long", c) == 0

From noreply at buildbot.pypy.org  Fri Dec 19 17:28:17 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 19 Dec 2014 17:28:17 +0100 (CET)
Subject: [pypy-commit] creflect default: update comment
Message-ID: <20141219162817.B33791C11BF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r213:229436d9afb4
Date: 2014-12-19 17:28 +0100
http://bitbucket.org/cffi/creflect/changeset/229436d9afb4/

Log:	update comment

diff --git a/zeffir/ctype.c b/zeffir/ctype.c
--- a/zeffir/ctype.c
+++ b/zeffir/ctype.c
@@ -4,7 +4,7 @@
 #define CT_PRIMITIVE_UNSIGNED 2    /* unsigned integer, bool */
 #define CT_PRIMITIVE_CHAR     4    /* char */
 #define CT_PRIMITIVE_FLOAT    8    /* float, double, long double */
-#define CT_POINTER           16    /* pointer, excluding ptr-to-func */
+#define CT_POINTER           16    /* pointer */
 #define CT_ARRAY             32    /* array */
 #define CT_STRUCT            64    /* struct */
 #define CT_UNION            128    /* union */

From noreply at buildbot.pypy.org  Fri Dec 19 17:29:04 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 19 Dec 2014 17:29:04 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Write down an idea
Message-ID: <20141219162904.4A1621C11BF@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r5483:733ab4c473b1
Date: 2014-12-19 08:29 -0800
http://bitbucket.org/pypy/extradoc/changeset/733ab4c473b1/

Log:	Write down an idea

diff --git a/planning/jit.txt b/planning/jit.txt
--- a/planning/jit.txt
+++ b/planning/jit.txt
@@ -105,6 +105,9 @@
 
   (This applies to the read direction as well)
 
+- hash(str[x:y]) should have an optimization in the JIT so that no malloc/copy
+  occurs
+
 PYTHON EXAMPLES
 ---------------
 

From noreply at buildbot.pypy.org  Fri Dec 19 18:49:23 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 18:49:23 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: fix test_doctest test_lineendings
	not closing files
Message-ID: <20141219174923.4D6DF1D2848@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75038:11ea34526fbb
Date: 2014-12-19 12:45 -0500
http://bitbucket.org/pypy/pypy/changeset/11ea34526fbb/

Log:	fix test_doctest test_lineendings not closing files

diff --git a/lib-python/2.7/test/test_doctest.py b/lib-python/2.7/test/test_doctest.py
--- a/lib-python/2.7/test/test_doctest.py
+++ b/lib-python/2.7/test/test_doctest.py
@@ -2580,7 +2580,8 @@
 
     >>> import tempfile, os
     >>> fn = tempfile.mktemp()
-    >>> open(fn, 'w').write('Test:\r\n\r\n  >>> x = 1 + 1\r\n\r\nDone.\r\n')
+    >>> with open(fn, 'w') as fobj:
+    ...     fobj.write('Test:\r\n\r\n  >>> x = 1 + 1\r\n\r\nDone.\r\n')
     >>> doctest.testfile(fn, False)
     TestResults(failed=0, attempted=1)
     >>> os.remove(fn)
@@ -2588,7 +2589,8 @@
 And now *nix line endings:
 
     >>> fn = tempfile.mktemp()
-    >>> open(fn, 'w').write('Test:\n\n  >>> x = 1 + 1\n\nDone.\n')
+    >>> with open(fn, 'w') as fobj:
+    ...     fobj.write('Test:\n\n  >>> x = 1 + 1\n\nDone.\n')
     >>> doctest.testfile(fn, False)
     TestResults(failed=0, attempted=1)
     >>> os.remove(fn)

From noreply at buildbot.pypy.org  Fri Dec 19 18:49:24 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 18:49:24 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: fix test_sysconfig by skipping
 cpython-specific test
Message-ID: <20141219174924.98F991D2848@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75039:196b3896b1cd
Date: 2014-12-19 12:49 -0500
http://bitbucket.org/pypy/pypy/changeset/196b3896b1cd/

Log:	fix test_sysconfig by skipping cpython-specific test

diff --git a/lib-python/2.7/test/test_sysconfig.py b/lib-python/2.7/test/test_sysconfig.py
--- a/lib-python/2.7/test/test_sysconfig.py
+++ b/lib-python/2.7/test/test_sysconfig.py
@@ -8,7 +8,7 @@
 from copy import copy, deepcopy
 
 from test.test_support import (run_unittest, TESTFN, unlink, get_attribute,
-                               import_module)
+                               import_module, check_impl_detail)
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -256,6 +256,8 @@
                   'posix_home', 'posix_prefix', 'posix_user', 'pypy')
         self.assertEqual(get_scheme_names(), wanted)
 
+    @unittest.skipIf(check_impl_detail(pypy=True),
+                     'Test is not PyPy compatible')
     @unittest.skipIf(sys.platform.startswith('win'),
                      'Test is not Windows compatible')
     def test_get_makefile_filename(self):

From noreply at buildbot.pypy.org  Fri Dec 19 23:29:01 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Fri, 19 Dec 2014 23:29:01 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: start working on _ssl
Message-ID: <20141219222901.BE8381C11BF@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75040:f92139314948
Date: 2014-12-19 17:26 -0500
http://bitbucket.org/pypy/pypy/changeset/f92139314948/

Log:	start working on _ssl

diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py
--- a/pypy/module/_ssl/__init__.py
+++ b/pypy/module/_ssl/__init__.py
@@ -5,9 +5,19 @@
     See the socket module for documentation."""
 
     interpleveldefs = {
-        'sslwrap': 'interp_ssl.sslwrap',
-        'SSLError': 'interp_ssl.get_error(space)',
         '_test_decode_cert': 'interp_ssl._test_decode_cert',
+        'txt2obj': 'interp_ssl.txt2obj',
+        'nid2obj': 'interp_ssl.nid2obj',
+
+        'SSLError': "interp_ssl.get_exception_class(space, 'w_sslerror')",
+        'SSLZeroReturnError': "interp_ssl.get_exception_class(space, 'w_sslzeroreturnerror')",
+        'SSLWantReadError': "interp_ssl.get_exception_class(space, 'w_sslwantreaderror')",
+        'SSLWantWriteError': "interp_ssl.get_exception_class(space, 'w_sslwantwriteerror')",
+        'SSLSyscallError': "interp_ssl.get_exception_class(space, 'w_sslsyscallerror')",
+        'SSLEOFError': "interp_ssl.get_exception_class(space, 'w_ssleoferror')",
+
+        '_SSLSocket': 'interp_ssl._SSLSocket',
+        '_SSLContext': 'interp_ssl._SSLContext',
     }
 
     appleveldefs = {
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -4,7 +4,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
 from pypy.module._socket import interp_socket
@@ -50,6 +50,16 @@
 constants["CERT_OPTIONAL"] = PY_SSL_CERT_OPTIONAL
 constants["CERT_REQUIRED"] = PY_SSL_CERT_REQUIRED
 
+constants["VERIFY_DEFAULT"] = 0
+constants["VERIFY_CRL_CHECK_LEAF"] = X509_V_FLAG_CRL_CHECK
+constants["VERIFY_CRL_CHECK_CHAIN"] = X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL
+constants["VERIFY_X509_STRICT"] = X509_V_FLAG_X509_STRICT
+
+constants["HAS_SNI"] = HAS_SNI
+constants["HAS_TLS_UNIQUE"] = HAVE_OPENSSL_FINISHED
+constants["HAS_ECDH"] = not OPENSSL_NO_ECDH
+constants["HAS_NPN"] = OPENSSL_NPN_NEGOTIATED
+
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
 if not OPENSSL_NO_SSL3:
@@ -64,11 +74,13 @@
 ver, fix    = divmod(ver, 256)
 ver, minor  = divmod(ver, 256)
 ver, major  = divmod(ver, 256)
-constants["OPENSSL_VERSION_INFO"] = (major, minor, fix, patch, status)
+version_info = (major, minor, fix, patch, status)
+constants["OPENSSL_VERSION_INFO"] = version_info
+constants["_OPENSSL_API_VERSION"] = version_info
 constants["OPENSSL_VERSION"] = SSLEAY_VERSION
 
-def ssl_error(space, msg, errno=0):
-    w_exception_class = get_error(space)
+def ssl_error(space, msg, errno=0, exc='w_sslerror'):
+    w_exception_class = get_exception_class(space, exc)
     w_exception = space.call_function(w_exception_class,
                                       space.wrap(errno), space.wrap(msg))
     return OperationError(w_exception_class, w_exception)
@@ -111,41 +123,54 @@
         return space.wrap(bytes)
 
 
-class SSLObject(W_Root):
-    def __init__(self, space):
+class _SSLSocket(W_Root):
+    @staticmethod
+    def descr_new(space, sslctx, w_sock, socket_type, hostname, w_ssl_sock):
+        self = _SSLSocket()
+
         self.space = space
-        self.w_socket = None
-        self.ctx = lltype.nullptr(SSL_CTX.TO)
-        self.ssl = lltype.nullptr(SSL.TO)
+        self.ctx = sslctx
         self.peer_cert = lltype.nullptr(X509.TO)
-        self._server = lltype.malloc(rffi.CCHARP.TO, X509_NAME_MAXLEN,
-                                     flavor='raw')
-        self._server[0] = '\0'
-        self._issuer = lltype.malloc(rffi.CCHARP.TO, X509_NAME_MAXLEN,
-                                     flavor='raw')
-        self._issuer[0] = '\0'
         self.shutdown_seen_zero = False
+        self.handshake_done = False
 
-    def server(self, space):
-        return space.wrap(rffi.charp2str(self._server))
+        sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
+        self.ssl = libssl_SSL_new(sslctx.ctx) # new ssl struct
+        libssl_SSL_set_fd(self.ssl, sock_fd) # set the socket for SSL
+        # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address
+        # of a str object may be changed by the garbage collector.
+        libssl_SSL_set_mode(
+            self.ssl, SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER)
 
-    def issuer(self, space):
-        return space.wrap(rffi.charp2str(self._issuer))
+        # If the socket is in non-blocking mode or timeout mode, set the BIO
+        # to non-blocking mode (blocking is the default)
+        w_timeout = space.call_method(w_sock, "gettimeout")
+        has_timeout = not space.is_none(w_timeout)
+        if has_timeout:
+            # Set both the read and write BIO's to non-blocking mode
+            libssl_BIO_set_nbio(libssl_SSL_get_rbio(self.ssl), 1)
+            libssl_BIO_set_nbio(libssl_SSL_get_wbio(self.ssl), 1)
+
+        if socket_type == PY_SSL_CLIENT:
+            libssl_SSL_set_connect_state(self.ssl)
+        else:
+            libssl_SSL_set_accept_state(self.ssl)
+
+        self.socket_type = socket_type
+        self.w_socket = w_sock
+        self.w_ssl_sock = None
+        return self
 
     def __del__(self):
-        self.enqueue_for_destruction(self.space, SSLObject.destructor,
+        self.enqueue_for_destruction(self.space, _SSLSocket.destructor,
                                      '__del__() method of ')
 
     def destructor(self):
-        assert isinstance(self, SSLObject)
+        assert isinstance(self, _SSLSocket)
         if self.peer_cert:
             libssl_X509_free(self.peer_cert)
         if self.ssl:
             libssl_SSL_free(self.ssl)
-        if self.ctx:
-            libssl_SSL_CTX_free(self.ctx)
-        lltype.free(self._server, flavor='raw')
-        lltype.free(self._issuer, flavor='raw')
 
     @unwrap_spec(data='bufferstr')
     def write(self, space, data):
@@ -205,7 +230,7 @@
         return space.wrap(count)
 
     @unwrap_spec(num_bytes=int)
-    def read(self, space, num_bytes=1024):
+    def read(self, space, num_bytes):
         """read([len]) -> string
 
         Read up to len bytes from the SSL socket."""
@@ -300,13 +325,7 @@
         if self.peer_cert:
             libssl_X509_free(self.peer_cert)
         self.peer_cert = libssl_SSL_get_peer_certificate(self.ssl)
-        if self.peer_cert:
-            libssl_X509_NAME_oneline(
-                libssl_X509_get_subject_name(self.peer_cert),
-                self._server, X509_NAME_MAXLEN)
-            libssl_X509_NAME_oneline(
-                libssl_X509_get_issuer_name(self.peer_cert),
-                self._issuer, X509_NAME_MAXLEN)
+        self.handshake_done = True
 
     def shutdown(self, space):
         # Guard against closed socket
@@ -625,110 +644,17 @@
 
     return space.newtuple([w_name, w_value])
 
-SSLObject.typedef = TypeDef("SSLObject",
-    server = interp2app(SSLObject.server),
-    issuer = interp2app(SSLObject.issuer),
-    write = interp2app(SSLObject.write),
-    pending = interp2app(SSLObject.pending),
-    read = interp2app(SSLObject.read),
-    do_handshake = interp2app(SSLObject.do_handshake),
-    shutdown = interp2app(SSLObject.shutdown),
-    cipher = interp2app(SSLObject.cipher),
-    peer_certificate = interp2app(SSLObject.peer_certificate),
+_SSLSocket.typedef = TypeDef("_SSLSocket",
+    do_handshake = interp2app(_SSLSocket.do_handshake),
+    write = interp2app(_SSLSocket.write),
+    read = interp2app(_SSLSocket.read),
+    pending = interp2app(_SSLSocket.pending),
+    peer_certificate = interp2app(_SSLSocket.peer_certificate),
+    cipher = interp2app(_SSLSocket.cipher),
+    shutdown = interp2app(_SSLSocket.shutdown),
 )
 
 
-def new_sslobject(space, w_sock, side, w_key_file, w_cert_file,
-                  cert_mode, protocol, w_cacerts_file, w_ciphers):
-    ss = SSLObject(space)
-
-    sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
-    w_timeout = space.call_method(w_sock, "gettimeout")
-    has_timeout = not space.is_none(w_timeout)
-    key_file = space.str_or_None_w(w_key_file)
-    cert_file = space.str_or_None_w(w_cert_file)
-    cacerts_file = space.str_or_None_w(w_cacerts_file)
-    ciphers = space.str_or_None_w(w_ciphers)
-
-    if side == PY_SSL_SERVER and (not key_file or not cert_file):
-        raise ssl_error(space, "Both the key & certificate files "
-                        "must be specified for server-side operation")
-
-    # set up context
-    if protocol == PY_SSL_VERSION_TLS1:
-        method = libssl_TLSv1_method()
-    elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
-        method = libssl_SSLv3_method()
-    elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
-        method = libssl_SSLv2_method()
-    elif protocol == PY_SSL_VERSION_SSL23:
-        method = libssl_SSLv23_method()
-    else:
-        raise ssl_error(space, "Invalid SSL protocol variant specified")
-    ss.ctx = libssl_SSL_CTX_new(method)
-    if not ss.ctx:
-        raise ssl_error(space, "Could not create SSL context")
-
-    if ciphers:
-        ret = libssl_SSL_CTX_set_cipher_list(ss.ctx, ciphers)
-        if ret == 0:
-            raise ssl_error(space, "No cipher can be selected.")
-
-    if cert_mode != PY_SSL_CERT_NONE:
-        if not cacerts_file:
-            raise ssl_error(space,
-                            "No root certificates specified for "
-                            "verification of other-side certificates.")
-        ret = libssl_SSL_CTX_load_verify_locations(ss.ctx, cacerts_file, None)
-        if ret != 1:
-            raise _ssl_seterror(space, None, 0)
-
-    if key_file:
-        ret = libssl_SSL_CTX_use_PrivateKey_file(ss.ctx, key_file,
-                                                 SSL_FILETYPE_PEM)
-        if ret < 1:
-            raise ssl_error(space, "SSL_CTX_use_PrivateKey_file error")
-
-        ret = libssl_SSL_CTX_use_certificate_chain_file(ss.ctx, cert_file)
-        if ret < 1:
-            raise ssl_error(space, "SSL_CTX_use_certificate_chain_file error")
-
-    # ssl compatibility
-    options = SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
-    if protocol != PY_SSL_VERSION_SSL2:
-        # SSLv2 is extremely broken, don't use it unless a user specifically
-        # requests it
-        options |= SSL_OP_NO_SSLv2
-    libssl_SSL_CTX_set_options(ss.ctx, options)
-
-    verification_mode = SSL_VERIFY_NONE
-    if cert_mode == PY_SSL_CERT_OPTIONAL:
-        verification_mode = SSL_VERIFY_PEER
-    elif cert_mode == PY_SSL_CERT_REQUIRED:
-        verification_mode = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT
-    libssl_SSL_CTX_set_verify(ss.ctx, verification_mode, None)
-    ss.ssl = libssl_SSL_new(ss.ctx) # new ssl struct
-    libssl_SSL_set_fd(ss.ssl, sock_fd) # set the socket for SSL
-    # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address
-    # of a str object may be changed by the garbage collector.
-    libssl_SSL_set_mode(
-        ss.ssl, SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER)
-
-    # If the socket is in non-blocking mode or timeout mode, set the BIO
-    # to non-blocking mode (blocking is the default)
-    if has_timeout:
-        # Set both the read and write BIO's to non-blocking mode
-        libssl_BIO_set_nbio(libssl_SSL_get_rbio(ss.ssl), 1)
-        libssl_BIO_set_nbio(libssl_SSL_get_wbio(ss.ssl), 1)
-
-    if side == PY_SSL_CLIENT:
-        libssl_SSL_set_connect_state(ss.ssl)
-    else:
-        libssl_SSL_set_accept_state(ss.ssl)
-
-    ss.w_socket = w_sock
-    return ss
-
 def checkwait(space, w_sock, writing):
     """If the socket has a timeout, do a select()/poll() on the socket.
     The argument writing indicates the direction.
@@ -843,24 +769,25 @@
     return ssl_error(space, errstr, errval)
 
 
- at unwrap_spec(side=int, cert_mode=int, protocol=int)
-def sslwrap(space, w_socket, side, w_key_file=None, w_cert_file=None,
-            cert_mode=PY_SSL_CERT_NONE, protocol=PY_SSL_VERSION_SSL23,
-            w_cacerts_file=None, w_ciphers=None):
-    """sslwrap(socket, side, [keyfile, certfile]) -> sslobject"""
-    return space.wrap(new_sslobject(
-        space, w_socket, side, w_key_file, w_cert_file,
-        cert_mode, protocol,
-        w_cacerts_file, w_ciphers))
-
 class Cache:
     def __init__(self, space):
         w_socketerror = interp_socket.get_error(space, "error")
-        self.w_error = space.new_exception_class(
+        self.w_sslerror = space.new_exception_class(
             "_ssl.SSLError", w_socketerror)
+        self.w_sslzeroreturnerror = space.new_exception_class(
+            "_ssl.SSLZeroReturnError", self.w_sslerror)
+        self.w_sslwantreaderror = space.new_exception_class(
+            "_ssl.SSLWantReadError", self.w_sslerror)
+        self.w_sslwantwriteerror = space.new_exception_class(
+            "_ssl.SSLWantWriteError", self.w_sslerror)
+        self.w_sslsyscallerror = space.new_exception_class(
+            "_ssl.SSLSyscallError", self.w_sslerror)
+        self.w_ssleoferror = space.new_exception_class(
+            "_ssl.SSLEOFError", self.w_sslerror)
 
-def get_error(space):
-    return space.fromcache(Cache).w_error
+def get_exception_class(space, name):
+    return getattr(space.fromcache(Cache), name)
+
 
 @unwrap_spec(filename=str, verbose=bool)
 def _test_decode_cert(space, filename, verbose=True):
@@ -882,3 +809,76 @@
             libssl_X509_free(x)
     finally:
         libssl_BIO_free(cert)
+
+
+class _SSLContext(W_Root):
+    @staticmethod
+    @unwrap_spec(protocol=int)
+    def descr_new(space, w_subtype, protocol):
+        if protocol == PY_SSL_VERSION_TLS1:
+            method = libssl_TLSv1_method()
+        elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
+            method = libssl_SSLv3_method()
+        elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
+            method = libssl_SSLv2_method()
+        elif protocol == PY_SSL_VERSION_SSL23:
+            method = libssl_SSLv23_method()
+        else:
+            raise ssl_error(space, "invalid protocol version")
+        ctx = libssl_SSL_CTX_new(method)
+        if not ctx:
+            raise ssl_error(space, "failed to allocate SSL context")
+
+        self = space.allocate_instance(_SSLContext, w_subtype)
+        self.ctx = ctx
+        return self
+
+    @unwrap_spec(server_side=int)
+    def descr_wrap_socket(self, space, w_sock, server_side, w_server_hostname=None, w_ssl_sock=None):
+        return _SSLSocket.descr_new(space, self, w_sock, server_side, w_server_hostname, w_ssl_sock)
+
+    @unwrap_spec(cipherlist=str)
+    def descr_set_ciphers(self, space, cipherlist):
+        ret = libssl_SSL_CTX_set_cipher_list(self.ctx, cipherlist)
+        if ret == 0:
+            libssl_ERR_clear_error()
+            raise ssl_error(space, "No cipher can be selected.")
+
+_SSLContext.typedef = TypeDef("_SSLContext",
+    __module__ = "_ssl",
+    __new__ = interp2app(_SSLContext.descr_new),
+    _wrap_socket = interp2app(_SSLContext.descr_wrap_socket),
+    set_ciphers = interp2app(_SSLContext.descr_set_ciphers),
+)
+
+
+def _asn1obj2py(space, obj):
+    nid = libssl_OBJ_obj2nid(obj)
+    if nid == NID_undef:
+        raise oefmt(space.w_ValueError, "Unknown object")
+    with rffi.scoped_alloc_buffer(100) as buf:
+        buflen = libssl_OBJ_obj2txt(buf.raw, 100, obj, 1)
+        if buflen < 0:
+            raise _ssl_seterror(space, None, 0)
+        if buflen:
+            w_buf = space.wrap(buf.str(buflen))
+        else:
+            w_buf = space.w_None
+    w_sn = space.wrap(rffi.charp2str(libssl_OBJ_nid2sn(nid)))
+    w_ln = space.wrap(rffi.charp2str(libssl_OBJ_nid2ln(nid)))
+    return space.newtuple([space.wrap(nid), w_sn, w_ln, w_buf])
+
+
+ at unwrap_spec(txt=str, name=bool)
+def txt2obj(space, txt, name=False):
+    obj = libssl_OBJ_txt2obj(txt, name)
+    if not obj:
+        raise oefmt(space.w_ValueError, "unknown object '%s'", txt)
+    result = _asn1obj2py(space, obj)
+    libssl_ASN1_OBJECT_free(obj)
+    return result
+
+
+ at unwrap_spec(nid=int)
+def nid2obj(space, nid):
+    return space.newtuple([])
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -130,20 +130,6 @@
         self.s.close()
         del ss; gc.collect()
 
-    def test_server(self):
-        import socket, gc
-        ss = socket.ssl(self.s)
-        assert isinstance(ss.server(), str)
-        self.s.close()
-        del ss; gc.collect()
-
-    def test_issuer(self):
-        import socket, gc
-        ss = socket.ssl(self.s)
-        assert isinstance(ss.issuer(), str)
-        self.s.close()
-        del ss; gc.collect()
-
     def test_write(self):
         import socket, gc
         ss = socket.ssl(self.s)
@@ -157,9 +143,10 @@
     def test_read(self):
         import socket, gc
         ss = socket.ssl(self.s)
+        raises(TypeError, ss.read)
         raises(TypeError, ss.read, "foo")
         ss.write("hello\n")
-        data = ss.read()
+        data = ss.read(10)
         assert isinstance(data, str)
         self.s.close()
         del ss; gc.collect()
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -69,6 +69,8 @@
         "SSLEAY_VERSION", "SSLeay_version(SSLEAY_VERSION)")
     OPENSSL_NO_SSL2 = rffi_platform.Defined("OPENSSL_NO_SSL2")
     OPENSSL_NO_SSL3 = rffi_platform.Defined("OPENSSL_NO_SSL3")
+    OPENSSL_NO_ECDH = rffi_platform.Defined("OPENSSL_NO_ECDH")
+    OPENSSL_NPN_NEGOTIATED = rffi_platform.Defined("OPENSSL_NPN_NEGOTIATED")
     SSL_FILETYPE_PEM = rffi_platform.ConstantInteger("SSL_FILETYPE_PEM")
     SSL_OP_ALL = rffi_platform.ConstantInteger("SSL_OP_ALL")
     SSL_OP_NO_SSLv2 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv2")
@@ -80,6 +82,9 @@
     SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE")
     SSL_VERIFY_PEER = rffi_platform.ConstantInteger("SSL_VERIFY_PEER")
     SSL_VERIFY_FAIL_IF_NO_PEER_CERT = rffi_platform.ConstantInteger("SSL_VERIFY_FAIL_IF_NO_PEER_CERT")
+    X509_V_FLAG_CRL_CHECK = rffi_platform.ConstantInteger("X509_V_FLAG_CRL_CHECK")
+    X509_V_FLAG_CRL_CHECK_ALL = rffi_platform.ConstantInteger("X509_V_FLAG_CRL_CHECK_ALL")
+    X509_V_FLAG_X509_STRICT = rffi_platform.ConstantInteger("X509_V_FLAG_X509_STRICT")
     SSL_ERROR_WANT_READ = rffi_platform.ConstantInteger(
         "SSL_ERROR_WANT_READ")
     SSL_ERROR_WANT_WRITE = rffi_platform.ConstantInteger(
@@ -97,6 +102,7 @@
     SSL_MODE_AUTO_RETRY = rffi_platform.ConstantInteger("SSL_MODE_AUTO_RETRY")
     SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER = rffi_platform.ConstantInteger("SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER")
 
+    NID_undef = rffi_platform.ConstantInteger("NID_undef")
     NID_subject_alt_name = rffi_platform.ConstantInteger("NID_subject_alt_name")
     GEN_DIRNAME = rffi_platform.ConstantInteger("GEN_DIRNAME")
     GEN_EMAIL = rffi_platform.ConstantInteger("GEN_EMAIL")
@@ -173,7 +179,10 @@
 OBJ_NAME = rffi.CArrayPtr(OBJ_NAME_st)
 
 HAVE_OPENSSL_RAND = OPENSSL_VERSION_NUMBER >= 0x0090500f
+HAVE_OPENSSL_FINISHED = OPENSSL_VERSION_NUMBER >= 0x0090500f
 HAVE_SSL_CTX_CLEAR_OPTIONS = OPENSSL_VERSION_NUMBER >= 0x009080df
+if OPENSSL_VERSION_NUMBER < 0x0090800f and not OPENSSL_NO_ECDH:
+    OPENSSL_NO_ECDH = True
 
 def external(name, argtypes, restype, **kw):
     kw['compilation_info'] = eci
@@ -265,6 +274,11 @@
 
 ssl_external('OBJ_obj2txt',
              [rffi.CCHARP, rffi.INT, ASN1_OBJECT, rffi.INT], rffi.INT)
+ssl_external('OBJ_obj2nid', [ASN1_OBJECT], rffi.INT)
+ssl_external('OBJ_nid2sn', [rffi.INT], rffi.CCHARP)
+ssl_external('OBJ_nid2ln', [rffi.INT], rffi.CCHARP)
+ssl_external('OBJ_txt2obj', [rffi.CCHARP, rffi.INT], ASN1_OBJECT)
+ssl_external('ASN1_OBJECT_free', [ASN1_OBJECT], lltype.Void)
 ssl_external('ASN1_STRING_data', [ASN1_STRING], rffi.CCHARP)
 ssl_external('ASN1_STRING_length', [ASN1_STRING], rffi.INT)
 ssl_external('ASN1_STRING_to_UTF8', [rffi.CCHARPP, ASN1_STRING], rffi.INT)

From noreply at buildbot.pypy.org  Sat Dec 20 22:36:01 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sat, 20 Dec 2014 22:36:01 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: update a few ssl tests
Message-ID: <20141220213601.5A5D91D2D59@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75041:f1afe76b15ce
Date: 2014-12-20 16:35 -0500
http://bitbucket.org/pypy/pypy/changeset/f1afe76b15ce/

Log:	update a few ssl tests

diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -65,7 +65,10 @@
         if sys.platform == 'darwin' or 'freebsd' in sys.platform:
             skip("hangs indefinitely on OSX & FreeBSD (also on CPython)")
         s = _socket.socket()
-        ss = _ssl.sslwrap(s, 0)
+        if sys.version_info < (2, 7, 9):
+            ss = _ssl.sslwrap(s, 0)
+        else:
+            ss = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)._wrap_socket(s, 0)
         exc = raises(_socket.error, ss.do_handshake)
         if sys.platform == 'win32':
             assert exc.value.errno == 10057 # WSAENOTCONN
@@ -75,13 +78,16 @@
         gc.collect()     # force the destructor() to be called now
 
     def test_async_closed(self):
-        import _ssl, _socket, gc
+        import _ssl, _socket, sys, gc
         s = _socket.socket()
         s.settimeout(3)
-        ss = _ssl.sslwrap(s, 0)
+        if sys.version_info < (2, 7, 9):
+            ss = _ssl.sslwrap(s, 0)
+        else:
+            ss = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)._wrap_socket(s, 0)
         s.close()
         exc = raises(_ssl.SSLError, ss.write, "data")
-        assert exc.value.strerror == "Underlying socket has been closed."
+        assert exc.value.message == 'Underlying socket has been closed.'
         del exc, ss, s
         gc.collect()     # force the destructor() to be called now
 

From noreply at buildbot.pypy.org  Sun Dec 21 03:02:17 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sun, 21 Dec 2014 03:02:17 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: provide SSLContext.options
Message-ID: <20141221020217.A52871D2961@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75042:e8524db1d39c
Date: 2014-12-20 21:00 -0500
http://bitbucket.org/pypy/pypy/changeset/e8524db1d39c/

Log:	provide SSLContext.options

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module._socket import interp_socket
 
 
@@ -844,11 +844,15 @@
             libssl_ERR_clear_error()
             raise ssl_error(space, "No cipher can be selected.")
 
+    def descr_get_options(self, space):
+        return space.newlong(libssl_SSL_CTX_get_options(self.ctx))
+
 _SSLContext.typedef = TypeDef("_SSLContext",
     __module__ = "_ssl",
     __new__ = interp2app(_SSLContext.descr_new),
     _wrap_socket = interp2app(_SSLContext.descr_wrap_socket),
     set_ciphers = interp2app(_SSLContext.descr_set_ciphers),
+    options = GetSetProperty(_SSLContext.descr_get_options),
 )
 
 
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -110,6 +110,10 @@
              ('IP Address', '192.0.2.1'),
              ('IP Address', '2001:DB8:0:0:0:0:0:1\n'))
 
+    def test_context(self):
+        import _ssl
+        s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        assert type(s.options) is long
 
 class AppTestConnectedSSL:
     spaceconfig = {

From noreply at buildbot.pypy.org  Sun Dec 21 11:34:16 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 21 Dec 2014 11:34:16 +0100 (CET)
Subject: [pypy-commit] pypy default: Store values in pure_operations instead
 of ops (I wonder what it changes, 
Message-ID: <20141221103416.65FD21C0399@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75043:0d5b714ce13a
Date: 2014-12-19 15:48 +0200
http://bitbucket.org/pypy/pypy/changeset/0d5b714ce13a/

Log:	Store values in pure_operations instead of ops (I wonder what it
	changes, actually)

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -299,8 +299,8 @@
     def make_constant_int(self, box, intconst):
         return self.optimizer.make_constant_int(box, intconst)
 
-    def make_equal_to(self, box, value):
-        return self.optimizer.make_equal_to(box, value)
+    def make_equal_to(self, box, value, replace=False):
+        return self.optimizer.make_equal_to(box, value, replace=replace)
 
     def get_constant_box(self, box):
         return self.optimizer.get_constant_box(box)
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -25,6 +25,7 @@
         else:
             nextop = None
 
+        args = None
         if canfold:
             for i in range(op.numargs()):
                 if self.get_constant_box(op.getarg(i)) is None:
@@ -39,15 +40,11 @@
 
             # did we do the exact same operation already?
             args = self.optimizer.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.optimizer.make_equal_to(op.result,
-                                             self.getvalue(oldop.result),
-                                             True)
+            oldvalue = self.pure_operations.get(args, None)
+            if oldvalue is not None:
+                self.optimizer.make_equal_to(op.result, oldvalue, True)
                 return
             else:
-                self.pure_operations[args] = op
                 self.remember_emitting_pure(op)
 
         # otherwise, the operation remains
@@ -56,6 +53,8 @@
             self.optimizer.bool_boxes[self.getvalue(op.result)] = None
         if nextop:
             self.emit_operation(nextop)
+        if args is not None:
+            self.pure_operations[args] = self.getvalue(op.result)
 
     def optimize_CALL_PURE(self, op):
         # Step 1: check if all arguments are constant
@@ -69,16 +68,15 @@
         # Step 2: check if all arguments are the same as a previous
         # CALL_PURE.
         args = self.optimizer.make_args_key(op)
-        oldop = self.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
+        oldvalue = self.pure_operations.get(args, None)
+        if oldvalue is not None:
             # this removes a CALL_PURE that has the same (non-constant)
             # arguments as a previous CALL_PURE.
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            self.make_equal_to(op.result, oldvalue)
             self.last_emitted_operation = REMOVED
             return
         else:
-            self.pure_operations[args] = op
+            self.pure_operations[args] = self.getvalue(op.result)
             self.remember_emitting_pure(op)
 
         # replace CALL_PURE with just CALL
@@ -103,15 +101,12 @@
         op = ResOperation(opnum, args, result)
         key = self.optimizer.make_args_key(op)
         if key not in self.pure_operations:
-            self.pure_operations[key] = op
+            self.pure_operations[key] = self.getvalue(op.result)
 
     def has_pure_result(self, opnum, args, descr):
         op = ResOperation(opnum, args, None, descr)
         key = self.optimizer.make_args_key(op)
-        op = self.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        return self.pure_operations.get(key, None) is not None
 
     def get_pure_result(self, key):
         return self.pure_operations.get(key, None)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -33,9 +33,8 @@
         dispatch_opt(self, op)
 
     def try_boolinvers(self, op, targs):
-        oldop = self.get_pure_result(targs)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            value = self.getvalue(oldop.result)
+        value = self.get_pure_result(targs)
+        if value is not None:
             if value.is_constant():
                 if value.box.same_constant(CONST_1):
                     self.make_constant(op.result, CONST_0)
@@ -59,9 +58,9 @@
         if oldopnum != -1:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.get_pure_result(targs)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                self.make_equal_to(op.result, self.getvalue(oldop.result))
+            value = self.get_pure_result(targs)
+            if value is not None:
+                self.optimizer.make_equal_to(op.result, value, True)
                 return True
 
         if op.boolreflex == -1:

From noreply at buildbot.pypy.org  Sun Dec 21 11:34:17 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 21 Dec 2014 11:34:17 +0100 (CET)
Subject: [pypy-commit] pypy default: essential optimization,
 don't store None in producer (and update it all the time too)
Message-ID: <20141221103417.8F9631C0399@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75044:fd1489733b1e
Date: 2014-12-19 15:52 +0200
http://bitbucket.org/pypy/pypy/changeset/fd1489733b1e/

Log:	essential optimization, don't store None in producer (and update it
	all the time too)

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -541,7 +541,8 @@
         self.first_optimization.propagate_forward(op)
 
     def propagate_forward(self, op):
-        self.producer[op.result] = op
+        if op.result is not None:
+            self.producer[op.result] = op
         dispatch_opt(self, op)
 
     def emit_operation(self, op):

From noreply at buildbot.pypy.org  Sun Dec 21 11:34:18 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 21 Dec 2014 11:34:18 +0100 (CET)
Subject: [pypy-commit] pypy default: Try to slim optvalues by splitting them
 into IntOptValue and PtrOptValue
Message-ID: <20141221103418.D99BB1C0399@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75045:e0c704d662ec
Date: 2014-12-21 12:34 +0200
http://bitbucket.org/pypy/pypy/changeset/e0c704d662ec/

Log:	Try to slim optvalues by splitting them into IntOptValue and
	PtrOptValue

diff --git a/rpython/jit/metainterp/optimizeopt/generalize.py b/rpython/jit/metainterp/optimizeopt/generalize.py
--- a/rpython/jit/metainterp/optimizeopt/generalize.py
+++ b/rpython/jit/metainterp/optimizeopt/generalize.py
@@ -1,4 +1,5 @@
-from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT
+from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT,\
+     IntOptValue
 
 
 class GeneralizationStrategy(object):
@@ -14,7 +15,8 @@
         for v in self.optimizer.values.values():
             if v.is_constant():
                 continue
-            if v.intbound.lower < MININT / 2:
-                v.intbound.lower = MININT
-            if v.intbound.upper > MAXINT / 2:
-                v.intbound.upper = MAXINT
+            if isinstance(v, IntOptValue):
+                if v.intbound.lower < MININT / 2:
+                    v.intbound.lower = MININT
+                if v.intbound.upper > MAXINT / 2:
+                    v.intbound.upper = MAXINT
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -383,7 +383,7 @@
         except KeyError:
             return
         for idx, cf in submap.iteritems():
-            if indexvalue is None or indexvalue.intbound.contains(idx):
+            if indexvalue is None or indexvalue.getintbound().contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
     def _assert_valid_cf(self, cf):
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt.intutils import (IntBound, IntLowerBound,
     IntUpperBound)
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1,
-    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE)
+    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE, IntOptValue)
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.backend.llsupport import symbolic
@@ -54,7 +54,9 @@
         #        but the bounds produced by all instructions where box is
         #        an argument might also be tighten
         v = self.getvalue(box)
-        b = v.intbound
+        b = v.getintbound()
+        if b is None:
+            return # pointer
         if b.has_lower and b.has_upper and b.lower == b.upper:
             v.make_constant(ConstInt(b.lower))
 
@@ -81,11 +83,13 @@
                 self.make_constant_int(op.result, 0)
             return
         self.emit_operation(op)
-        if v1.intbound.known_ge(IntBound(0, 0)) and \
-           v2.intbound.known_ge(IntBound(0, 0)):
-            r = self.getvalue(op.result)
-            mostsignificant = v1.intbound.upper | v2.intbound.upper
-            r.intbound.intersect(IntBound(0, next_pow2_m1(mostsignificant)))
+        bound1 = v1.getintbound()
+        bound2 = v2.getintbound()
+        if bound1.known_ge(IntBound(0, 0)) and \
+           bound2.known_ge(IntBound(0, 0)):
+            r = self.getvalue(op.result).getintbound()
+            mostsignificant = bound1.upper | bound2.upper
+            r.intersect(IntBound(0, next_pow2_m1(mostsignificant)))
 
     optimize_INT_OR = optimize_INT_OR_or_XOR
     optimize_INT_XOR = optimize_INT_OR_or_XOR
@@ -99,55 +103,55 @@
         if v2.is_constant():
             val = v2.box.getint()
             if val >= 0:
-                r.intbound.intersect(IntBound(0, val))
+                r.getintbound().intersect(IntBound(0, val))
         elif v1.is_constant():
             val = v1.box.getint()
             if val >= 0:
-                r.intbound.intersect(IntBound(0, val))
-        elif v1.intbound.known_ge(IntBound(0, 0)) and \
-          v2.intbound.known_ge(IntBound(0, 0)):
-            lesser = min(v1.intbound.upper, v2.intbound.upper)
-            r.intbound.intersect(IntBound(0, next_pow2_m1(lesser)))
+                r.getintbound().intersect(IntBound(0, val))
+        elif v1.getintbound().known_ge(IntBound(0, 0)) and \
+          v2.getintbound().known_ge(IntBound(0, 0)):
+            lesser = min(v1.getintbound().upper, v2.getintbound().upper)
+            r.getintbound().intersect(IntBound(0, next_pow2_m1(lesser)))
 
     def optimize_INT_SUB(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        b = v1.intbound.sub_bound(v2.intbound)
+        b = v1.getintbound().sub_bound(v2.getintbound())
         if b.bounded():
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        b = v1.intbound.add_bound(v2.intbound)
+        b = v1.getintbound().add_bound(v2.getintbound())
         if b.bounded():
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        b = v1.intbound.mul_bound(v2.intbound)
+        b = v1.getintbound().mul_bound(v2.getintbound())
         if b.bounded():
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        r.intbound.intersect(v1.intbound.div_bound(v2.intbound))
+        r.getintbound().intersect(v1.getintbound().div_bound(v2.getintbound()))
 
     def optimize_INT_MOD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        known_nonneg = (v1.intbound.known_ge(IntBound(0, 0)) and
-                        v2.intbound.known_ge(IntBound(0, 0)))
+        known_nonneg = (v1.getintbound().known_ge(IntBound(0, 0)) and
+                        v2.getintbound().known_ge(IntBound(0, 0)))
         if known_nonneg and v2.is_constant():
             val = v2.box.getint()
             if (val & (val-1)) == 0:
@@ -164,18 +168,18 @@
                     return     # give up
                 val = -val
             if known_nonneg:
-                r.intbound.make_ge(IntBound(0, 0))
+                r.getintbound().make_ge(IntBound(0, 0))
             else:
-                r.intbound.make_gt(IntBound(-val, -val))
-            r.intbound.make_lt(IntBound(val, val))
+                r.getintbound().make_gt(IntBound(-val, -val))
+            r.getintbound().make_lt(IntBound(val, val))
 
     def optimize_INT_LSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        b = v1.intbound.lshift_bound(v2.intbound)
-        r.intbound.intersect(b)
+        b = v1.getintbound().lshift_bound(v2.getintbound())
+        r.getintbound().intersect(b)
         # intbound.lshift_bound checks for an overflow and if the
         # lshift can be proven not to overflow sets b.has_upper and
         # b.has_lower
@@ -186,14 +190,14 @@
     def optimize_INT_RSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        b = v1.intbound.rshift_bound(v2.intbound)
+        b = v1.getintbound().rshift_bound(v2.getintbound())
         if b.has_lower and b.has_upper and b.lower == b.upper:
             # constant result (likely 0, for rshifts that kill all bits)
             self.make_constant_int(op.result, b.lower)
         else:
             self.emit_operation(op)
             r = self.getvalue(op.result)
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_GUARD_NO_OVERFLOW(self, op):
         lastop = self.last_emitted_operation
@@ -238,7 +242,7 @@
     def optimize_INT_ADD_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        resbound = v1.intbound.add_bound(v2.intbound)
+        resbound = v1.getintbound().add_bound(v2.getintbound())
         if resbound.bounded():
             # Transform into INT_ADD.  The following guard will be killed
             # by optimize_GUARD_NO_OVERFLOW; if we see instead an
@@ -246,7 +250,7 @@
             op = op.copy_and_change(rop.INT_ADD)
         self.emit_operation(op) # emit the op
         r = self.getvalue(op.result)
-        r.intbound.intersect(resbound)
+        r.getintbound().intersect(resbound)
 
     def optimize_INT_SUB_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -254,29 +258,29 @@
         if v1 is v2:
             self.make_constant_int(op.result, 0)
             return
-        resbound = v1.intbound.sub_bound(v2.intbound)
+        resbound = v1.getintbound().sub_bound(v2.getintbound())
         if resbound.bounded():
             op = op.copy_and_change(rop.INT_SUB)
         self.emit_operation(op) # emit the op
         r = self.getvalue(op.result)
-        r.intbound.intersect(resbound)
+        r.getintbound().intersect(resbound)
 
     def optimize_INT_MUL_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        resbound = v1.intbound.mul_bound(v2.intbound)
+        resbound = v1.getintbound().mul_bound(v2.getintbound())
         if resbound.bounded():
             op = op.copy_and_change(rop.INT_MUL)
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        r.intbound.intersect(resbound)
+        r.getintbound().intersect(resbound)
 
     def optimize_INT_LT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_lt(v2.intbound):
+        if v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op.result, 1)
-        elif v1.intbound.known_ge(v2.intbound) or v1 is v2:
+        elif v1.getintbound().known_ge(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
@@ -284,9 +288,9 @@
     def optimize_INT_GT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_gt(v2.intbound):
+        if v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op.result, 1)
-        elif v1.intbound.known_le(v2.intbound) or v1 is v2:
+        elif v1.getintbound().known_le(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
@@ -294,9 +298,9 @@
     def optimize_INT_LE(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_le(v2.intbound) or v1 is v2:
+        if v1.getintbound().known_le(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op.result, 1)
-        elif v1.intbound.known_gt(v2.intbound):
+        elif v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
@@ -304,9 +308,9 @@
     def optimize_INT_GE(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_ge(v2.intbound) or v1 is v2:
+        if v1.getintbound().known_ge(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op.result, 1)
-        elif v1.intbound.known_lt(v2.intbound):
+        elif v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
@@ -314,9 +318,9 @@
     def optimize_INT_EQ(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_gt(v2.intbound):
+        if v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op.result, 0)
-        elif v1.intbound.known_lt(v2.intbound):
+        elif v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op.result, 0)
         elif v1 is v2:
             self.make_constant_int(op.result, 1)
@@ -326,9 +330,9 @@
     def optimize_INT_NE(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_gt(v2.intbound):
+        if v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op.result, 1)
-        elif v1.intbound.known_lt(v2.intbound):
+        elif v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op.result, 1)
         elif v1 is v2:
             self.make_constant_int(op.result, 0)
@@ -337,7 +341,7 @@
 
     def optimize_INT_FORCE_GE_ZERO(self, op):
         value = self.getvalue(op.getarg(0))
-        if value.intbound.known_ge(IntBound(0, 0)):
+        if value.getintbound().known_ge(IntBound(0, 0)):
             self.make_equal_to(op.result, value)
         else:
             self.emit_operation(op)
@@ -348,50 +352,53 @@
         start = -(1 << (numbits - 1))
         stop = 1 << (numbits - 1)
         bounds = IntBound(start, stop - 1)
-        if bounds.contains_bound(value.intbound):
+        if bounds.contains_bound(value.getintbound()):
             self.make_equal_to(op.result, value)
         else:
             self.emit_operation(op)
             vres = self.getvalue(op.result)
-            vres.intbound.intersect(bounds)
+            vres.getintbound().intersect(bounds)
 
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
         result = self.getvalue(op.result)
         array.make_len_gt(MODE_ARRAY, op.getdescr(), -1)
-        array.lenbound.bound.intersect(result.intbound)
-        result.intbound = array.lenbound.bound
+        array.getlenbound().bound.intersect(result.getintbound())
+        assert isinstance(result, IntOptValue)
+        result.intbound = array.getlenbound().bound
 
     def optimize_STRLEN(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
         result = self.getvalue(op.result)
         array.make_len_gt(MODE_STR, op.getdescr(), -1)
-        array.lenbound.bound.intersect(result.intbound)
-        result.intbound = array.lenbound.bound
+        array.getlenbound().bound.intersect(result.getintbound())
+        assert isinstance(result, IntOptValue)
+        result.intbound = array.getlenbound().bound
 
     def optimize_UNICODELEN(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
         result = self.getvalue(op.result)
         array.make_len_gt(MODE_UNICODE, op.getdescr(), -1)
-        array.lenbound.bound.intersect(result.intbound)
-        result.intbound = array.lenbound.bound
+        array.getlenbound().bound.intersect(result.getintbound())
+        assert isinstance(result, IntOptValue)        
+        result.intbound = array.getlenbound().bound
 
     def optimize_STRGETITEM(self, op):
         self.emit_operation(op)
         v1 = self.getvalue(op.result)
-        v1.intbound.make_ge(IntLowerBound(0))
-        v1.intbound.make_lt(IntUpperBound(256))
+        v1.getintbound().make_ge(IntLowerBound(0))
+        v1.getintbound().make_lt(IntUpperBound(256))
 
     def optimize_GETFIELD_RAW(self, op):
         self.emit_operation(op)
         descr = op.getdescr()
         if descr.is_integer_bounded():
             v1 = self.getvalue(op.result)
-            v1.intbound.make_ge(IntLowerBound(descr.get_integer_min()))
-            v1.intbound.make_le(IntUpperBound(descr.get_integer_max()))
+            v1.getintbound().make_ge(IntLowerBound(descr.get_integer_min()))
+            v1.getintbound().make_le(IntUpperBound(descr.get_integer_max()))
 
     optimize_GETFIELD_GC = optimize_GETFIELD_RAW
 
@@ -402,30 +409,30 @@
         descr = op.getdescr()
         if descr and descr.is_item_integer_bounded():
             v1 = self.getvalue(op.result)
-            v1.intbound.make_ge(IntLowerBound(descr.get_item_integer_min()))
-            v1.intbound.make_le(IntUpperBound(descr.get_item_integer_max()))
+            v1.getintbound().make_ge(IntLowerBound(descr.get_item_integer_min()))
+            v1.getintbound().make_le(IntUpperBound(descr.get_item_integer_max()))
 
     optimize_GETARRAYITEM_GC = optimize_GETARRAYITEM_RAW
 
     def optimize_UNICODEGETITEM(self, op):
         self.emit_operation(op)
         v1 = self.getvalue(op.result)
-        v1.intbound.make_ge(IntLowerBound(0))
+        v1.getintbound().make_ge(IntLowerBound(0))
 
     def make_int_lt(self, box1, box2):
         v1 = self.getvalue(box1)
         v2 = self.getvalue(box2)
-        if v1.intbound.make_lt(v2.intbound):
+        if v1.getintbound().make_lt(v2.getintbound()):
             self.propagate_bounds_backward(box1)
-        if v2.intbound.make_gt(v1.intbound):
+        if v2.getintbound().make_gt(v1.getintbound()):
             self.propagate_bounds_backward(box2)
 
     def make_int_le(self, box1, box2):
         v1 = self.getvalue(box1)
         v2 = self.getvalue(box2)
-        if v1.intbound.make_le(v2.intbound):
+        if v1.getintbound().make_le(v2.getintbound()):
             self.propagate_bounds_backward(box1)
-        if v2.intbound.make_ge(v1.intbound):
+        if v2.getintbound().make_ge(v1.getintbound()):
             self.propagate_bounds_backward(box2)
 
     def make_int_gt(self, box1, box2):
@@ -472,9 +479,9 @@
             if r.box.same_constant(CONST_1):
                 v1 = self.getvalue(op.getarg(0))
                 v2 = self.getvalue(op.getarg(1))
-                if v1.intbound.intersect(v2.intbound):
+                if v1.getintbound().intersect(v2.getintbound()):
                     self.propagate_bounds_backward(op.getarg(0))
-                if v2.intbound.intersect(v1.intbound):
+                if v2.getintbound().intersect(v1.getintbound()):
                     self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_NE(self, op):
@@ -483,9 +490,9 @@
             if r.box.same_constant(CONST_0):
                 v1 = self.getvalue(op.getarg(0))
                 v2 = self.getvalue(op.getarg(1))
-                if v1.intbound.intersect(v2.intbound):
+                if v1.getintbound().intersect(v2.getintbound()):
                     self.propagate_bounds_backward(op.getarg(0))
-                if v2.intbound.intersect(v1.intbound):
+                if v2.getintbound().intersect(v1.getintbound()):
                     self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_IS_TRUE(self, op):
@@ -493,8 +500,8 @@
         if r.is_constant():
             if r.box.same_constant(CONST_1):
                 v1 = self.getvalue(op.getarg(0))
-                if v1.intbound.known_ge(IntBound(0, 0)):
-                    v1.intbound.make_gt(IntBound(0, 0))
+                if v1.getintbound().known_ge(IntBound(0, 0)):
+                    v1.getintbound().make_gt(IntBound(0, 0))
                     self.propagate_bounds_backward(op.getarg(0))
 
     def propagate_bounds_INT_IS_ZERO(self, op):
@@ -505,49 +512,49 @@
                 # Clever hack, we can't use self.make_constant_int yet because
                 # the args aren't in the values dictionary yet so it runs into
                 # an assert, this is a clever way of expressing the same thing.
-                v1.intbound.make_ge(IntBound(0, 0))
-                v1.intbound.make_lt(IntBound(1, 1))
+                v1.getintbound().make_ge(IntBound(0, 0))
+                v1.getintbound().make_lt(IntBound(1, 1))
                 self.propagate_bounds_backward(op.getarg(0))
 
     def propagate_bounds_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         r = self.getvalue(op.result)
-        b = r.intbound.sub_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        b = r.getintbound().sub_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
-        b = r.intbound.sub_bound(v1.intbound)
-        if v2.intbound.intersect(b):
+        b = r.getintbound().sub_bound(v1.getintbound())
+        if v2.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_SUB(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         r = self.getvalue(op.result)
-        b = r.intbound.add_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        b = r.getintbound().add_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
-        b = r.intbound.sub_bound(v1.intbound).mul(-1)
-        if v2.intbound.intersect(b):
+        b = r.getintbound().sub_bound(v1.getintbound()).mul(-1)
+        if v2.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         r = self.getvalue(op.result)
-        b = r.intbound.div_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        b = r.getintbound().div_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
-        b = r.intbound.div_bound(v1.intbound)
-        if v2.intbound.intersect(b):
+        b = r.getintbound().div_bound(v1.getintbound())
+        if v2.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_LSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         r = self.getvalue(op.result)
-        b = r.intbound.rshift_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        b = r.getintbound().rshift_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
 
     propagate_bounds_INT_ADD_OVF = propagate_bounds_INT_ADD
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -1,9 +1,11 @@
 from rpython.jit.metainterp import jitprof, resume, compile
 from rpython.jit.metainterp.executor import execute_nonspec
-from rpython.jit.metainterp.history import BoxInt, BoxFloat, Const, ConstInt, REF
-from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded, \
+from rpython.jit.metainterp.history import BoxInt, BoxFloat, Const, ConstInt,\
+     REF, BoxPtr, ConstPtr, ConstFloat
+from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded,\
                                                      ImmutableIntUnbounded, \
-                                                     IntLowerBound, MININT, MAXINT
+                                                     IntLowerBound, MININT,\
+                                                     MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractResOp
 from rpython.jit.metainterp.typesystem import llhelper
@@ -39,69 +41,19 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
-    last_guard = None
+    _attrs_ = ('box', 'level')
 
     level = LEVEL_UNKNOWN
-    known_class = None
-    intbound = ImmutableIntUnbounded()
-    lenbound = None
 
     def __init__(self, box, level=None, known_class=None, intbound=None):
         self.box = box
         if level is not None:
             self.level = level
-        self.known_class = known_class
-        if intbound:
-            self.intbound = intbound
-        else:
-            if isinstance(box, BoxInt):
-                self.intbound = IntBound(MININT, MAXINT)
-            else:
-                self.intbound = IntUnbounded()
 
         if isinstance(box, Const):
             self.make_constant(box)
         # invariant: box is a Const if and only if level == LEVEL_CONSTANT
 
-    def make_len_gt(self, mode, descr, val):
-        if self.lenbound:
-            assert self.lenbound.mode == mode
-            assert self.lenbound.descr == descr
-            self.lenbound.bound.make_gt(IntBound(val, val))
-        else:
-            self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))
-
-    def make_guards(self, box):
-        guards = []
-        if self.level == LEVEL_CONSTANT:
-            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
-            guards.append(op)
-        elif self.level == LEVEL_KNOWNCLASS:
-            op = ResOperation(rop.GUARD_NONNULL, [box], None)
-            guards.append(op)
-            op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
-            guards.append(op)
-        else:
-            if self.level == LEVEL_NONNULL:
-                op = ResOperation(rop.GUARD_NONNULL, [box], None)
-                guards.append(op)
-            self.intbound.make_guards(box, guards)
-            if self.lenbound:
-                lenbox = BoxInt()
-                if self.lenbound.mode == MODE_ARRAY:
-                    op = ResOperation(rop.ARRAYLEN_GC, [box], lenbox, self.lenbound.descr)
-                elif self.lenbound.mode == MODE_STR:
-                    op = ResOperation(rop.STRLEN, [box], lenbox, self.lenbound.descr)
-                elif self.lenbound.mode == MODE_UNICODE:
-                    op = ResOperation(rop.UNICODELEN, [box], lenbox, self.lenbound.descr)
-                else:
-                    debug_print("Unknown lenbound mode")
-                    assert False
-                guards.append(op)
-                self.lenbound.bound.make_guards(lenbox, guards)
-        return guards
-
     def import_from(self, other, optimizer):
         if self.level == LEVEL_CONSTANT:
             assert other.level == LEVEL_CONSTANT
@@ -112,19 +64,16 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, None)
+            self.make_constant_class(other.get_known_class(), None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
-            self.intbound.intersect(other.intbound)
-            if other.lenbound:
-                if self.lenbound:
-                    assert other.lenbound.mode == self.lenbound.mode
-                    assert other.lenbound.descr == self.lenbound.descr
-                    self.lenbound.bound.intersect(other.lenbound.bound)
-                else:
-                    self.lenbound = other.lenbound.clone()
 
+    def make_guards(self, box):
+        if self.level == LEVEL_CONSTANT:
+            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
+            return [op]
+        return []
 
     def force_box(self, optforce):
         return self.box
@@ -168,38 +117,6 @@
             return self.box.same_constant(other.box)
         return self is other
 
-    def make_constant(self, constbox):
-        """Replace 'self.box' with a Const box."""
-        assert isinstance(constbox, Const)
-        self.box = constbox
-        self.level = LEVEL_CONSTANT
-
-        if isinstance(constbox, ConstInt):
-            val = constbox.getint()
-            self.intbound = IntBound(val, val)
-        else:
-            self.intbound = IntUnbounded()
-
-    def get_constant_class(self, cpu):
-        level = self.level
-        if level == LEVEL_KNOWNCLASS:
-            return self.known_class
-        elif level == LEVEL_CONSTANT:
-            return cpu.ts.cls_of_box(self.box)
-        else:
-            return None
-
-    def make_constant_class(self, classbox, guardop):
-        assert self.level < LEVEL_KNOWNCLASS
-        self.known_class = classbox
-        self.level = LEVEL_KNOWNCLASS
-        self.last_guard = guardop
-
-    def make_nonnull(self, guardop):
-        assert self.level < LEVEL_NONNULL
-        self.level = LEVEL_NONNULL
-        self.last_guard = guardop
-
     def is_nonnull(self):
         level = self.level
         if level == LEVEL_NONNULL or level == LEVEL_KNOWNCLASS:
@@ -208,12 +125,6 @@
             box = self.box
             assert isinstance(box, Const)
             return box.nonnull()
-        elif self.intbound:
-            if self.intbound.known_gt(IntBound(0, 0)) or \
-               self.intbound.known_lt(IntBound(0, 0)):
-                return True
-            else:
-                return False
         else:
             return False
 
@@ -260,8 +171,202 @@
     def get_missing_null_value(self):
         raise NotImplementedError    # only for VArrayValue
 
+    def make_constant(self, constbox):
+        """Replace 'self.box' with a Const box."""
+        assert isinstance(constbox, Const)
+        self.box = constbox
+        self.level = LEVEL_CONSTANT
 
-class ConstantValue(OptValue):
+    def get_last_guard(self):
+        return None
+
+    def get_known_class(self):
+        return None
+
+    def getlenbound(self):
+        return None
+
+    def getintbound(self):
+        return None
+
+    def get_constant_class(self, cpu):
+        return None
+
+class PtrOptValue(OptValue):
+    _attrs_ = ('known_class', 'last_guard', 'lenbound')
+
+    known_class = None
+    last_guard = None
+    lenbound = None
+
+    def __init__(self, box, level=None, known_class=None, intbound=None):
+        OptValue.__init__(self, box, level, None, intbound)
+        if not isinstance(box, Const):
+            self.known_class = known_class
+
+    def make_len_gt(self, mode, descr, val):
+        if self.lenbound:
+            assert self.lenbound.mode == mode
+            assert self.lenbound.descr == descr
+            self.lenbound.bound.make_gt(IntBound(val, val))
+        else:
+            self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))
+
+    def make_nonnull(self, guardop):
+        assert self.level < LEVEL_NONNULL
+        self.level = LEVEL_NONNULL
+        self.last_guard = guardop
+
+    def make_constant_class(self, classbox, guardop):
+        assert self.level < LEVEL_KNOWNCLASS
+        self.known_class = classbox
+        self.level = LEVEL_KNOWNCLASS
+        self.last_guard = guardop
+
+    def import_from(self, other, optimizer):
+        OptValue.import_from(self, other, optimizer)
+        if self.level != LEVEL_CONSTANT:
+            if other.getlenbound():
+                if self.lenbound:
+                    assert other.getlenbound().mode == self.lenbound.mode
+                    assert other.getlenbound().descr == self.lenbound.descr
+                    self.lenbound.bound.intersect(other.getlenbound().bound)
+                else:
+                    self.lenbound = other.getlenbound().clone()
+
+    def make_guards(self, box):
+        guards = []
+        if self.level == LEVEL_CONSTANT:
+            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
+            guards.append(op)
+        elif self.level == LEVEL_KNOWNCLASS:
+            op = ResOperation(rop.GUARD_NONNULL, [box], None)
+            guards.append(op)
+            op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
+            guards.append(op)
+        else:
+            if self.level == LEVEL_NONNULL:
+                op = ResOperation(rop.GUARD_NONNULL, [box], None)
+                guards.append(op)
+            if self.lenbound:
+                lenbox = BoxInt()
+                if self.lenbound.mode == MODE_ARRAY:
+                    op = ResOperation(rop.ARRAYLEN_GC, [box], lenbox, self.lenbound.descr)
+                elif self.lenbound.mode == MODE_STR:
+                    op = ResOperation(rop.STRLEN, [box], lenbox, self.lenbound.descr)
+                elif self.lenbound.mode == MODE_UNICODE:
+                    op = ResOperation(rop.UNICODELEN, [box], lenbox, self.lenbound.descr)
+                else:
+                    debug_print("Unknown lenbound mode")
+                    assert False
+                guards.append(op)
+                self.lenbound.bound.make_guards(lenbox, guards)
+        return guards
+
+    def get_constant_class(self, cpu):
+        level = self.level
+        if level == LEVEL_KNOWNCLASS:
+            return self.known_class
+        elif level == LEVEL_CONSTANT:
+            return cpu.ts.cls_of_box(self.box)
+        else:
+            return None
+
+    def getlenbound(self):
+        return self.lenbound
+
+    def get_last_guard(self):
+        return self.last_guard
+
+    def get_known_class(self):
+        return self.known_class
+
+class IntOptValue(OptValue):
+    _attrs_ = ('intbound',)
+
+    intbound = ImmutableIntUnbounded()
+
+    def __init__(self, box, level=None, known_class=None, intbound=None):
+        OptValue.__init__(self, box, level, None, None)
+        if isinstance(box, Const):
+            return
+        if intbound:
+            self.intbound = intbound
+        else:
+            if isinstance(box, BoxInt):
+                self.intbound = IntBound(MININT, MAXINT)
+            else:
+                self.intbound = IntUnbounded()
+
+    def make_constant(self, constbox):
+        """Replace 'self.box' with a Const box."""
+        assert isinstance(constbox, ConstInt)
+        self.box = constbox
+        self.level = LEVEL_CONSTANT
+        val = constbox.getint()
+        self.intbound = IntBound(val, val)
+
+    def is_nonnull(self):
+        if OptValue.is_nonnull(self):
+            return True
+        if self.intbound:
+            if self.intbound.known_gt(IntBound(0, 0)) or \
+               self.intbound.known_lt(IntBound(0, 0)):
+                return True
+        return False
+
+    def make_nonnull(self, guardop):
+        assert self.level < LEVEL_NONNULL
+        self.level = LEVEL_NONNULL
+
+    def import_from(self, other, optimizer):
+        OptValue.import_from(self, other, optimizer)
+        if self.level != LEVEL_CONSTANT:
+            if other.getintbound() is not None: # VRawBufferValue
+                self.intbound.intersect(other.getintbound())
+
+    def make_guards(self, box):
+        guards = []
+        if self.level == LEVEL_CONSTANT:
+            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
+            guards.append(op)
+        elif self.level == LEVEL_KNOWNCLASS:
+            op = ResOperation(rop.GUARD_NONNULL, [box], None)
+            guards.append(op)
+        else:
+            if self.level == LEVEL_NONNULL:
+                op = ResOperation(rop.GUARD_NONNULL, [box], None)
+                guards.append(op)
+            self.intbound.make_guards(box, guards)
+        return guards
+
+    def getintbound(self):
+        return self.intbound
+
+    def get_last_guard(self):
+        return None
+
+    def get_known_class(self):
+        return None
+
+    def getlenbound(self):
+        return None
+
+class ConstantFloatValue(OptValue):
+    def __init__(self, box):
+        self.make_constant(box)
+
+    def __repr__(self):
+        return 'Constant(%r)' % (self.box,)
+
+class ConstantIntValue(IntOptValue):
+    def __init__(self, box):
+        self.make_constant(box)
+
+    def __repr__(self):
+        return 'Constant(%r)' % (self.box,)
+
+class ConstantPtrValue(PtrOptValue):
     def __init__(self, box):
         self.make_constant(box)
 
@@ -270,9 +375,9 @@
 
 CONST_0      = ConstInt(0)
 CONST_1      = ConstInt(1)
-CVAL_ZERO    = ConstantValue(CONST_0)
-CVAL_ZERO_FLOAT = ConstantValue(Const._new(0.0))
-llhelper.CVAL_NULLREF = ConstantValue(llhelper.CONST_NULL)
+CVAL_ZERO    = ConstantIntValue(CONST_0)
+CVAL_ZERO_FLOAT = ConstantFloatValue(Const._new(0.0))
+llhelper.CVAL_NULLREF = ConstantPtrValue(llhelper.CONST_NULL)
 REMOVED = AbstractResOp(None)
 
 
@@ -451,7 +556,12 @@
         try:
             value = self.values[box]
         except KeyError:
-            value = self.values[box] = OptValue(box)
+            if isinstance(box, BoxPtr) or isinstance(box, ConstPtr):
+                value = self.values[box] = PtrOptValue(box)
+            elif isinstance(box, BoxInt) or isinstance(box, ConstInt):
+                value = self.values[box] = IntOptValue(box)
+            else:
+                value = self.values[box] = OptValue(box)
         self.ensure_imported(value)
         return value
 
@@ -487,7 +597,14 @@
         self.values[box] = value
 
     def make_constant(self, box, constbox):
-        self.make_equal_to(box, ConstantValue(constbox))
+        if isinstance(constbox, ConstInt):
+            self.make_equal_to(box, ConstantIntValue(constbox))
+        elif isinstance(constbox, ConstPtr):
+            self.make_equal_to(box, ConstantPtrValue(constbox))
+        elif isinstance(constbox, ConstFloat):
+            self.make_equal_to(box, ConstantFloatValue(constbox))
+        else:
+            assert False
 
     def make_constant_int(self, box, intvalue):
         self.make_constant(box, ConstInt(intvalue))
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
-    CONST_0, CONST_1)
+    CONST_0, CONST_1, PtrOptValue)
 from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses
 from rpython.rlib.rarithmetic import highest_bit
@@ -82,14 +82,14 @@
             return
         elif v2.is_constant():
             val = v2.box.getint()
-            if val == -1 or v1.intbound.lower >= 0 \
-                and v1.intbound.upper <= val & ~(val + 1):
+            if val == -1 or v1.getintbound().lower >= 0 \
+                and v1.getintbound().upper <= val & ~(val + 1):
                 self.make_equal_to(op.result, v1)
                 return
         elif v1.is_constant():
             val = v1.box.getint()
-            if val == -1 or v2.intbound.lower >= 0 \
-                and v2.intbound.upper <= val & ~(val + 1):
+            if val == -1 or v2.getintbound().lower >= 0 \
+                and v2.getintbound().upper <= val & ~(val + 1):
                 self.make_equal_to(op.result, v2)
                 return
 
@@ -295,11 +295,11 @@
             else:
                 name = "<unknown>"
             raise InvalidLoop('A promote of a virtual %s (a recently allocated object) never makes sense!' % name)
-        if value.last_guard:
+        if value.get_last_guard():
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = value.last_guard
+            old_guard_op = value.get_last_guard()
             if old_guard_op.getopnum() != rop.GUARD_NONNULL:
                 # This is only safe if the class of the guard_value matches the
                 # class of the guard_*_class, otherwise the intermediate ops might
@@ -322,7 +322,8 @@
             descr.guard_opnum = rop.GUARD_VALUE
             descr.make_a_counter_per_value(op)
             # to be safe
-            value.last_guard = None
+            if isinstance(value, PtrOptValue):
+                value.last_guard = None
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
         self.optimize_guard(op, constbox)
@@ -354,6 +355,7 @@
             r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
             raise InvalidLoop('A GUARD_CLASS (%s) was proven to always fail'
                               % r)
+        assert isinstance(value, PtrOptValue)
         if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
@@ -563,7 +565,7 @@
         elif v1.is_constant() and v1.box.getint() == 0:
             self.make_constant_int(op.result, 0)
             return
-        if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
+        if v1.getintbound().known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
                 op = op.copy_and_change(rop.INT_RSHIFT,
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -3,7 +3,8 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo, VStructStateInfo, \
      VArrayStateInfo, NotVirtualStateInfo, VirtualState, ShortBoxes, GenerateGuardState, \
      VirtualStatesCantMatch, VArrayStructStateInfo
-from rpython.jit.metainterp.optimizeopt.optimizer import OptValue
+from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, PtrOptValue,\
+      IntOptValue
 from rpython.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, ConstInt, ConstPtr
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin, BaseTest, \
@@ -121,13 +122,13 @@
         assert isgeneral(OptValue(BoxInt()), OptValue(ConstInt(7)))
         assert not isgeneral(OptValue(ConstInt(7)), OptValue(BoxInt()))
 
-        ptr = OptValue(BoxPtr())
-        nonnull = OptValue(BoxPtr())
+        ptr = PtrOptValue(BoxPtr())
+        nonnull = PtrOptValue(BoxPtr())
         nonnull.make_nonnull(0)
-        knownclass = OptValue(BoxPtr())
+        knownclass = PtrOptValue(BoxPtr())
         clsbox = self.cpu.ts.cls_of_box(BoxPtr(self.myptr))
         knownclass.make_constant_class(clsbox, 0)
-        const = OptValue(BoxPtr)
+        const = PtrOptValue(BoxPtr)
         const.make_constant_class(clsbox, 0)
         const.make_constant(ConstPtr(self.myptr))
         inorder = [ptr, nonnull, knownclass, const]
@@ -137,8 +138,8 @@
                 if i != j:
                     assert not isgeneral(inorder[j], inorder[i])
 
-        value1 = OptValue(BoxInt())
-        value2 = OptValue(BoxInt())
+        value1 = IntOptValue(BoxInt())
+        value2 = IntOptValue(BoxInt())
         value2.intbound.make_lt(IntBound(10, 10))
         assert isgeneral(value1, value2)
         assert not isgeneral(value2, value1)
@@ -150,9 +151,9 @@
         assert isgeneral(OptValue(ConstPtr(fooref)),
                          OptValue(ConstPtr(fooref)))
 
-        value1 = OptValue(BoxPtr())
+        value1 = PtrOptValue(BoxPtr())
         value1.make_nonnull(None)
-        value2 = OptValue(ConstPtr(self.nullptr))
+        value2 = PtrOptValue(ConstPtr(self.nullptr))
         assert not isgeneral(value1, value2)
 
     def test_field_matching_generalization(self):
@@ -178,18 +179,18 @@
         fldtst(VArrayStructStateInfo(fakedescr, [[fielddescr]]), VArrayStructStateInfo(fakedescr, [[fielddescr]]))
 
     def test_known_class_generalization(self):
-        knownclass1 = OptValue(BoxPtr())
+        knownclass1 = PtrOptValue(BoxPtr())
         knownclass1.make_constant_class(ConstPtr(self.myptr), 0)
         info1 = NotVirtualStateInfo(knownclass1)
         info1.position = 0
-        knownclass2 = OptValue(BoxPtr())
+        knownclass2 = PtrOptValue(BoxPtr())
         knownclass2.make_constant_class(ConstPtr(self.myptr), 0)
         info2 = NotVirtualStateInfo(knownclass2)
         info2.position = 0
         self.check_no_guards(info1, info2)
         self.check_no_guards(info2, info1)
 
-        knownclass3 = OptValue(BoxPtr())
+        knownclass3 = PtrOptValue(BoxPtr())
         knownclass3.make_constant_class(ConstPtr(self.myptr2), 0)
         info3 = NotVirtualStateInfo(knownclass3)
         info3.position = 0
@@ -209,33 +210,33 @@
 
     def test_generate_guards_nonvirtual_all_combinations(self):
         # set up infos
-        unknown_val = OptValue(self.nodebox)
-        unknownnull_val = OptValue(BoxPtr(self.nullptr))
+        unknown_val = PtrOptValue(self.nodebox)
+        unknownnull_val = PtrOptValue(BoxPtr(self.nullptr))
         unknown_info = NotVirtualStateInfo(unknown_val)
 
-        nonnull_val = OptValue(self.nodebox)
+        nonnull_val = PtrOptValue(self.nodebox)
         nonnull_val.make_nonnull(None)
         nonnull_info = NotVirtualStateInfo(nonnull_val)
 
-        knownclass_val = OptValue(self.nodebox)
+        knownclass_val = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         knownclass_val.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(knownclass_val)
-        knownclass2_val = OptValue(self.nodebox2)
+        knownclass2_val = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
         knownclass2_val.make_constant_class(classbox, -1)
         knownclass2_info = NotVirtualStateInfo(knownclass2_val)
 
-        constant_val = OptValue(BoxInt())
+        constant_val = IntOptValue(BoxInt())
         constant_val.make_constant(ConstInt(1))
         constant_info = NotVirtualStateInfo(constant_val)
-        constclass_val = OptValue(self.nodebox)
+        constclass_val = PtrOptValue(self.nodebox)
         constclass_val.make_constant(self.nodebox.constbox())
         constclass_info = NotVirtualStateInfo(constclass_val)
-        constclass2_val = OptValue(self.nodebox)
+        constclass2_val = PtrOptValue(self.nodebox)
         constclass2_val.make_constant(self.nodebox2.constbox())
         constclass2_info = NotVirtualStateInfo(constclass2_val)
-        constantnull_val = OptValue(ConstPtr(self.nullptr))
+        constantnull_val = PtrOptValue(ConstPtr(self.nullptr))
         constantnull_info = NotVirtualStateInfo(constantnull_val)
 
         # unknown unknown
@@ -354,11 +355,11 @@
 
 
     def test_intbounds(self):
-        value1 = OptValue(BoxInt(15))
+        value1 = IntOptValue(BoxInt(15))
         value1.intbound.make_ge(IntBound(0, 10))
         value1.intbound.make_le(IntBound(20, 30))
         info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(OptValue(BoxInt()))
+        info2 = NotVirtualStateInfo(IntOptValue(BoxInt()))
         expected = """
         [i0]
         i1 = int_ge(i0, 0)
@@ -370,22 +371,22 @@
         self.check_invalid(info1, info2, BoxInt(50))
 
     def test_intbounds_constant(self):
-        value1 = OptValue(BoxInt(15))
+        value1 = IntOptValue(BoxInt(15))
         value1.intbound.make_ge(IntBound(0, 10))
         value1.intbound.make_le(IntBound(20, 30))
         info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(OptValue(ConstInt(10000)))
+        info2 = NotVirtualStateInfo(IntOptValue(ConstInt(10000)))
         self.check_invalid(info1, info2)
         info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(OptValue(ConstInt(11)))
+        info2 = NotVirtualStateInfo(IntOptValue(ConstInt(11)))
         self.check_no_guards(info1, info2)
 
     def test_known_class(self):
-        value1 = OptValue(self.nodebox)
+        value1 = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value1.make_constant_class(classbox, -1)
         info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        info2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         expected = """
         [p0]
         guard_nonnull(p0) []        
@@ -395,7 +396,7 @@
         self.check_invalid(info1, info2, BoxPtr())
 
     def test_known_class_value(self):
-        value1 = OptValue(self.nodebox)
+        value1 = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value1.make_constant_class(classbox, -1)
         box = self.nodebox
@@ -408,7 +409,7 @@
         self.compare(guards, expected, [box])
 
     def test_known_value(self):
-        value1 = OptValue(self.nodebox)
+        value1 = PtrOptValue(self.nodebox)
         value1.make_constant(ConstInt(1))
         box = self.nodebox
         guards = value1.make_guards(box)
@@ -419,21 +420,21 @@
         self.compare(guards, expected, [box])
 
     def test_equal_inputargs(self):
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
         vstate1 = VirtualState([knownclass_info, knownclass_info])
         assert vstate1.generalization_of(vstate1)
 
-        unknown_info1 = NotVirtualStateInfo(OptValue(self.nodebox))
+        unknown_info1 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         vstate2 = VirtualState([unknown_info1, unknown_info1])
         assert vstate2.generalization_of(vstate2)
         assert not vstate1.generalization_of(vstate2)
         assert vstate2.generalization_of(vstate1)
 
-        unknown_info1 = NotVirtualStateInfo(OptValue(self.nodebox))
-        unknown_info2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        unknown_info1 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
+        unknown_info2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         vstate3 = VirtualState([unknown_info1, unknown_info2])
         assert vstate3.generalization_of(vstate2)
         assert vstate3.generalization_of(vstate1)
@@ -457,12 +458,12 @@
 
 
     def test_generate_guards_on_virtual_fields_matches_array(self):
-        innervalue1 = OptValue(self.nodebox)
+        innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
         innervalue1.make_constant_class(constclassbox, -1)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         innerinfo2.position = 1
 
         descr = object()
@@ -474,7 +475,7 @@
         info2.fieldstate = [innerinfo2]
 
         value1 = VArrayValue(descr, None, 1, self.nodebox)
-        value1._items[0] = OptValue(self.nodebox)
+        value1._items[0] = PtrOptValue(self.nodebox)
 
         expected = """
         [p0]
@@ -484,12 +485,12 @@
         self.guards(info1, info2, value1, expected, [self.nodebox])
 
     def test_generate_guards_on_virtual_fields_matches_instance(self):
-        innervalue1 = OptValue(self.nodebox)
+        innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
         innervalue1.make_constant_class(constclassbox, -1)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         innerinfo2.position = 1
 
         info1 = VirtualStateInfo(ConstInt(42), [1])
@@ -499,7 +500,7 @@
         info2.fieldstate = [innerinfo2]
 
         value1 = VirtualValue(self.cpu, constclassbox, self.nodebox)
-        value1._fields = {1: OptValue(self.nodebox)}
+        value1._fields = {1: PtrOptValue(self.nodebox)}
 
         expected = """
         [p0]
@@ -509,12 +510,12 @@
         self.guards(info1, info2, value1, expected, [self.nodebox])
 
     def test_generate_guards_on_virtual_fields_matches_struct(self):
-        innervalue1 = OptValue(self.nodebox)
+        innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
         innervalue1.make_constant_class(constclassbox, -1)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         innerinfo2.position = 1
 
         structdescr = object()
@@ -526,7 +527,7 @@
         info2.fieldstate = [innerinfo2]
 
         value1 = VStructValue(self.cpu, structdescr, self.nodebox)
-        value1._fields = {1: OptValue(self.nodebox)}
+        value1._fields = {1: PtrOptValue(self.nodebox)}
 
         expected = """
         [p0]
@@ -536,12 +537,12 @@
         self.guards(info1, info2, value1, expected, [self.nodebox])
 
     def test_generate_guards_on_virtual_fields_matches_arraystruct(self):
-        innervalue1 = OptValue(self.nodebox)
+        innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
         innervalue1.make_constant_class(constclassbox, -1)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         innerinfo2.position = 1
 
         arraydescr = object()
@@ -554,7 +555,7 @@
         info2.fieldstate = [innerinfo2]
 
         value1 = VArrayStructValue(arraydescr, 1, self.nodebox)
-        value1._items[0][fielddescr] = OptValue(self.nodebox)
+        value1._items[0][fielddescr] = PtrOptValue(self.nodebox)
 
         expected = """
         [p0]
@@ -568,7 +569,7 @@
 
     def test_virtuals_with_equal_fields(self):
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -596,7 +597,7 @@
 
     def test_virtuals_with_nonmatching_fields(self):
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -605,7 +606,7 @@
         assert vstate1.generalization_of(vstate1)
 
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox2)
+        value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -618,7 +619,7 @@
 
     def test_virtuals_with_nonmatching_descrs(self):
         info1 = VirtualStateInfo(ConstInt(42), [10, 20])
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -627,7 +628,7 @@
         assert vstate1.generalization_of(vstate1)
 
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox2)
+        value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -640,7 +641,7 @@
         
     def test_virtuals_with_nonmatching_classes(self):
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -649,7 +650,7 @@
         assert vstate1.generalization_of(vstate1)
 
         info2 = VirtualStateInfo(ConstInt(7), [1, 2])
-        value = OptValue(self.nodebox2)
+        value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -662,7 +663,7 @@
         
     def test_nonvirtual_is_not_virtual(self):
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -679,7 +680,7 @@
 
     def test_arrays_with_nonmatching_fields(self):
         info1 = VArrayStateInfo(42)
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -688,7 +689,7 @@
         assert vstate1.generalization_of(vstate1)
 
         info2 = VArrayStateInfo(42)
-        value = OptValue(self.nodebox2)
+        value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -701,7 +702,7 @@
 
     def test_arrays_of_different_sizes(self):
         info1 = VArrayStateInfo(42)
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -710,7 +711,7 @@
         assert vstate1.generalization_of(vstate1)
 
         info2 = VArrayStateInfo(42)
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -723,7 +724,7 @@
 
     def test_arrays_with_nonmatching_types(self):
         info1 = VArrayStateInfo(42)
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -732,7 +733,7 @@
         assert vstate1.generalization_of(vstate1)
 
         info2 = VArrayStateInfo(7)
-        value = OptValue(self.nodebox2)
+        value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -745,7 +746,7 @@
         
     def test_nonvirtual_is_not_array(self):
         info1 = VArrayStateInfo(42)
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -762,7 +763,7 @@
         
 
     def test_crash_varay_clear(self):
-        innervalue1 = OptValue(self.nodebox)
+        innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
         innervalue1.make_constant_class(constclassbox, -1)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -11,9 +11,9 @@
 from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.rlib.objectmodel import we_are_translated, specialize
+from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 
-
-class AbstractVirtualValue(optimizer.OptValue):
+class AbstractVirtualValue(optimizer.PtrOptValue):
     _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
@@ -433,6 +433,9 @@
         self.size = size
         self.buffer = RawBuffer(cpu, logops)
 
+    def getintbound(self):
+        return IntUnbounded()
+
     def getlength(self):
         return len(self.buffer.values)
 
@@ -492,6 +495,9 @@
         self.rawbuffer_value = rawbuffer_value
         self.offset = offset
 
+    def getintbound(self):
+        return IntUnbounded()
+
     def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -280,18 +280,18 @@
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     def __init__(self, value, is_opaque=False):
         self.is_opaque = is_opaque
-        self.known_class = value.known_class
+        self.known_class = value.get_known_class()
         self.level = value.level
-        if value.intbound is None:
+        if value.getintbound() is None:
             self.intbound = IntUnbounded()
         else:
-            self.intbound = value.intbound.clone()
+            self.intbound = value.getintbound().clone()
         if value.is_constant():
             self.constbox = value.box
         else:
             self.constbox = None
         self.position_in_notvirtuals = -1
-        self.lenbound = value.lenbound
+        self.lenbound = value.getlenbound()
 
 
     def _generate_guards(self, other, value, state):
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -472,7 +472,7 @@
                 if index < len1:
                     return self.strgetitem(value.left, vindex, mode)
                 else:
-                    vindex = optimizer.ConstantValue(ConstInt(index - len1))
+                    vindex = optimizer.ConstantIntValue(ConstInt(index - len1))
                     return self.strgetitem(value.right, vindex, mode)
         #
         resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode, resbox)
@@ -521,7 +521,7 @@
             dst_start = dststart.force_box(self).getint()
             actual_length = length.force_box(self).getint()
             for index in range(actual_length):
-                vresult = self.strgetitem(src, optimizer.ConstantValue(ConstInt(index + src_start)), mode)
+                vresult = self.strgetitem(src, optimizer.ConstantIntValue(ConstInt(index + src_start)), mode)
                 if dst_virtual:
                     dst.setitem(index + dst_start, vresult)
                 else:

From noreply at buildbot.pypy.org  Sun Dec 21 16:34:31 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 21 Dec 2014 16:34:31 +0100 (CET)
Subject: [pypy-commit] pypy default: update value in-place instead of
 creating a new one (for one relatively obscure
Message-ID: <20141221153431.90B841C3354@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75046:9a145bb36ced
Date: 2014-12-21 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/9a145bb36ced/

Log:	update value in-place instead of creating a new one (for one
	relatively obscure case of promoting to constant anyway)

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt.util import args_dict
 from rpython.jit.metainterp.history import Const
 from rpython.jit.metainterp.jitexc import JitException
-from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED
+from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED, LEVEL_CONSTANT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation
@@ -63,6 +63,17 @@
             # cancelling its previous effects with no side effect.
             self._lazy_setfield = None
 
+    def value_updated(self, oldvalue, newvalue):
+        try:
+            fieldvalue = self._cached_fields[oldvalue]
+        except KeyError:
+            pass
+        else:
+            self._cached_fields[newvalue] = fieldvalue
+            op = self._cached_fields_getfield_op[oldvalue].clone()
+            op.setarg(0, newvalue.box)
+            self._cached_fields_getfield_op[newvalue] = op
+
     def possible_aliasing(self, optheap, structvalue):
         # If lazy_setfield is set and contains a setfield on a different
         # structvalue, then we are annoyed, because it may point to either
@@ -117,18 +128,6 @@
         self._cached_fields.clear()
         self._cached_fields_getfield_op.clear()
 
-    def turned_constant(self, newvalue, value):
-        if newvalue not in self._cached_fields and value in self._cached_fields:
-            self._cached_fields[newvalue] = self._cached_fields[value]
-            op = self._cached_fields_getfield_op[value].clone()
-            constbox = value.box
-            assert isinstance(constbox, Const)
-            op.setarg(0, constbox)
-            self._cached_fields_getfield_op[newvalue] = op
-        for structvalue in self._cached_fields.keys():
-            if self._cached_fields[structvalue] is value:
-                self._cached_fields[structvalue] = newvalue
-
     def produce_potential_short_preamble_ops(self, optimizer, shortboxes, descr):
         if self._lazy_setfield is not None:
             return
@@ -186,6 +185,17 @@
         self._seen_guard_not_invalidated = False
         self.postponed_op = None
 
+    def setup(self):
+        self.optimizer.optheap = self
+
+    def value_updated(self, oldvalue, newvalue):
+        # XXXX very unhappy about that
+        for cf in self.cached_fields.itervalues():
+            cf.value_updated(oldvalue, newvalue)
+        for submap in self.cached_arrayitems.itervalues():
+            for cf in submap.itervalues():
+                cf.value_updated(oldvalue, newvalue)
+
     def force_at_end_of_preamble(self):
         self.cached_dict_reads.clear()
         self.corresponding_array_descrs.clear()
@@ -360,16 +370,6 @@
             # ^^^ we only need to force this field; the other fields
             # of virtualref_info and virtualizable_info are not gcptrs.
 
-    def turned_constant(self, value):
-        assert value.is_constant()
-        newvalue = self.getvalue(value.box)
-        if value is not newvalue:
-            for cf in self.cached_fields.itervalues():
-                cf.turned_constant(newvalue, value)
-            for submap in self.cached_arrayitems.itervalues():
-                for cf in submap.itervalues():
-                    cf.turned_constant(newvalue, value)
-
     def force_lazy_setfield(self, descr, can_cache=True):
         try:
             cf = self.cached_fields[descr]
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -62,7 +62,6 @@
         assert self.level <= LEVEL_NONNULL
         if other.level == LEVEL_CONSTANT:
             self.make_constant(other.get_key_box())
-            optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
             self.make_constant_class(other.get_known_class(), None)
         else:
@@ -75,6 +74,11 @@
             return [op]
         return []
 
+    def copy_from(self, other_value):
+        assert isinstance(other_value, OptValue)
+        self.box = other_value.box
+        self.level = other_value.level
+
     def force_box(self, optforce):
         return self.box
 
@@ -204,6 +208,14 @@
         if not isinstance(box, Const):
             self.known_class = known_class
 
+    def copy_from(self, other_value):
+        assert isinstance(other_value, PtrOptValue)
+        self.box = other_value.box
+        self.known_class = other_value.known_class
+        self.level = other_value.level
+        self.last_guard = other_value.last_guard
+        self.lenbound = other_value.lenbound
+
     def make_len_gt(self, mode, descr, val):
         if self.lenbound:
             assert self.lenbound.mode == mode
@@ -298,6 +310,12 @@
             else:
                 self.intbound = IntUnbounded()
 
+    def copy_from(self, other_value):
+        assert isinstance(other_value, IntOptValue)
+        self.box = other_value.box
+        self.intbound = other_value.intbound
+        self.level = other_value.level
+
     def make_constant(self, constbox):
         """Replace 'self.box' with a Const box."""
         assert isinstance(constbox, ConstInt)
@@ -439,9 +457,6 @@
     def setup(self):
         pass
 
-    def turned_constant(self, value):
-        pass
-
     def force_at_end_of_preamble(self):
         pass
 
@@ -492,6 +507,7 @@
         self.seen_results = {}
         self.optimizer = self
         self.optpure = None
+        self.optheap = None
         self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
@@ -527,10 +543,6 @@
         for opt in self.optimizations:
             opt.produce_potential_short_preamble_ops(sb)
 
-    def turned_constant(self, value):
-        for o in self.optimizations:
-            o.turned_constant(value)
-
     def forget_numberings(self, virtualbox):
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_FORCINGS)
         self.resumedata_memo.forget_numberings(virtualbox)
@@ -593,7 +605,17 @@
 
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
-        assert replace or box not in self.values
+        if replace:
+            try:
+                cur_value = self.values[box]
+            except KeyError:
+                pass
+            else:
+                assert value.level != LEVEL_CONSTANT
+                assert cur_value.level != LEVEL_CONSTANT
+                # replacing with a different box
+                cur_value.copy_from(value)
+                return
         self.values[box] = value
 
     def make_constant(self, box, constbox):
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -261,7 +261,8 @@
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
-        self.optimizer.turned_constant(value)
+        if self.optimizer.optheap:
+            self.optimizer.optheap.value_updated(value, self.getvalue(constbox))
 
     def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))

From noreply at buildbot.pypy.org  Sun Dec 21 16:41:01 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 21 Dec 2014 16:41:01 +0100 (CET)
Subject: [pypy-commit] pypy default: be concise here
Message-ID: <20141221154101.30B911C3354@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75047:7a8eb8dd3607
Date: 2014-12-21 17:40 +0200
http://bitbucket.org/pypy/pypy/changeset/7a8eb8dd3607/

Log:	be concise here

diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -101,7 +101,7 @@
         op = ResOperation(opnum, args, result)
         key = self.optimizer.make_args_key(op)
         if key not in self.pure_operations:
-            self.pure_operations[key] = self.getvalue(op.result)
+            self.pure_operations[key] = self.getvalue(result)
 
     def has_pure_result(self, opnum, args, descr):
         op = ResOperation(opnum, args, None, descr)

From noreply at buildbot.pypy.org  Sun Dec 21 16:45:08 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 21 Dec 2014 16:45:08 +0100 (CET)
Subject: [pypy-commit] pypy default: Kill seen_results. The rationale is
 that we'll get a KeyError in ShortBoxes
Message-ID: <20141221154508.41B681C3354@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75048:0422bc7c7898
Date: 2014-12-21 17:45 +0200
http://bitbucket.org/pypy/pypy/changeset/0422bc7c7898/

Log:	Kill seen_results. The rationale is that we'll get a KeyError in
	ShortBoxes anyway (so seen_results will only be visibile with
	unrolling disabled). Backend also checks it.

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -504,7 +504,6 @@
         self.opaque_pointers = {}
         self.replaces_guard = {}
         self._newoperations = []
-        self.seen_results = {}
         self.optimizer = self
         self.optpure = None
         self.optheap = None
@@ -601,7 +600,6 @@
 
     def clear_newoperations(self):
         self._newoperations = []
-        self.seen_results = {}
 
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
@@ -714,10 +712,6 @@
                 op = self.store_final_boxes_in_guard(op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        if op.result:
-            if op.result in self.seen_results:
-                raise ValueError, "invalid optimization"
-            self.seen_results[op.result] = None
         self._newoperations.append(op)
 
     def replace_op(self, old_op, new_op):

From noreply at buildbot.pypy.org  Sun Dec 21 17:05:27 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sun, 21 Dec 2014 17:05:27 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: allow setting SSLContext options
Message-ID: <20141221160527.803761D2549@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75049:f9c6ea503c21
Date: 2014-12-21 10:50 -0500
http://bitbucket.org/pypy/pypy/changeset/f9c6ea503c21/

Log:	allow setting SSLContext options

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -67,6 +67,11 @@
 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 
+constants["OP_ALL"] = SSL_OP_ALL &~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
+constants["OP_NO_SSLv2"] = SSL_OP_NO_SSLv2
+constants["OP_NO_SSLv3"] = SSL_OP_NO_SSLv3
+constants["OP_NO_TLSv1"] = SSL_OP_NO_TLSv1
+
 constants["OPENSSL_VERSION_NUMBER"] = OPENSSL_VERSION_NUMBER
 ver = OPENSSL_VERSION_NUMBER
 ver, status = divmod(ver, 16)
@@ -831,6 +836,10 @@
 
         self = space.allocate_instance(_SSLContext, w_subtype)
         self.ctx = ctx
+        options = SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
+        if protocol != PY_SSL_VERSION_SSL2:
+            options |= SSL_OP_NO_SSLv2
+        libssl_SSL_CTX_set_options(ctx, options)
         return self
 
     @unwrap_spec(server_side=int)
@@ -847,12 +856,27 @@
     def descr_get_options(self, space):
         return space.newlong(libssl_SSL_CTX_get_options(self.ctx))
 
+    def descr_set_options(self, space, w_new_opts):
+        new_opts = space.int_w(w_new_opts)
+        opts = libssl_SSL_CTX_get_options(self.ctx)
+        clear = opts & ~new_opts
+        set = ~opts & new_opts
+        if clear:
+            if HAVE_SSL_CTX_CLEAR_OPTIONS:
+                libssl_SSL_CTX_clear_options(self.ctx, clear)
+            else:
+                raise oefmt(space.w_ValueError,
+                            "can't clear options before OpenSSL 0.9.8m")
+        if set:
+            libssl_SSL_CTX_set_options(self.ctx, set)
+
 _SSLContext.typedef = TypeDef("_SSLContext",
     __module__ = "_ssl",
     __new__ = interp2app(_SSLContext.descr_new),
     _wrap_socket = interp2app(_SSLContext.descr_wrap_socket),
     set_ciphers = interp2app(_SSLContext.descr_set_ciphers),
-    options = GetSetProperty(_SSLContext.descr_get_options),
+    options = GetSetProperty(_SSLContext.descr_get_options,
+                             _SSLContext.descr_set_options),
 )
 
 
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -114,6 +114,10 @@
         import _ssl
         s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         assert type(s.options) is long
+        assert s.options & _ssl.OP_NO_SSLv2
+        s.options &= ~_ssl.OP_NO_SSLv2
+        assert not s.options & _ssl.OP_NO_SSLv2
+        raises(TypeError, "s.options = 2.5")
 
 class AppTestConnectedSSL:
     spaceconfig = {
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -180,7 +180,8 @@
 
 HAVE_OPENSSL_RAND = OPENSSL_VERSION_NUMBER >= 0x0090500f
 HAVE_OPENSSL_FINISHED = OPENSSL_VERSION_NUMBER >= 0x0090500f
-HAVE_SSL_CTX_CLEAR_OPTIONS = OPENSSL_VERSION_NUMBER >= 0x009080df
+HAVE_SSL_CTX_CLEAR_OPTIONS = OPENSSL_VERSION_NUMBER >= 0x009080df and \
+                             OPENSSL_VERSION_NUMBER != 0x00909000
 if OPENSSL_VERSION_NUMBER < 0x0090800f and not OPENSSL_NO_ECDH:
     OPENSSL_NO_ECDH = True
 

From noreply at buildbot.pypy.org  Sun Dec 21 17:05:28 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sun, 21 Dec 2014 17:05:28 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: provide SSLContext.verify_mode
Message-ID: <20141221160528.B53BA1D2549@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75050:a155733a8665
Date: 2014-12-21 11:05 -0500
http://bitbucket.org/pypy/pypy/changeset/a155733a8665/

Log:	provide SSLContext.verify_mode

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -870,6 +870,29 @@
         if set:
             libssl_SSL_CTX_set_options(self.ctx, set)
 
+    def descr_get_verify_mode(self, space):
+        mode = libssl_SSL_CTX_get_verify_mode(self.ctx)
+        if mode == SSL_VERIFY_NONE:
+            return space.newlong(PY_SSL_CERT_NONE)
+        elif mode == SSL_VERIFY_PEER:
+            return space.newlong(PY_SSL_CERT_OPTIONAL)
+        elif mode == SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT:
+            return space.newlong(PY_SSL_CERT_REQUIRED)
+        raise ssl_error(space, "invalid return value from SSL_CTX_get_verify_mode")
+
+    def descr_set_verify_mode(self, space, w_mode):
+        n = space.int_w(w_mode)
+        if n == PY_SSL_CERT_NONE:
+            mode = SSL_VERIFY_NONE
+        elif n == PY_SSL_CERT_OPTIONAL:
+            mode = SSL_VERIFY_PEER
+        elif n == PY_SSL_CERT_REQUIRED:
+            mode = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT
+        else:
+            raise oefmt(space.w_ValueError,
+                        "invalid value for verify_mode")
+        libssl_SSL_CTX_set_verify(self.ctx, mode, None)
+
 _SSLContext.typedef = TypeDef("_SSLContext",
     __module__ = "_ssl",
     __new__ = interp2app(_SSLContext.descr_new),
@@ -877,6 +900,8 @@
     set_ciphers = interp2app(_SSLContext.descr_set_ciphers),
     options = GetSetProperty(_SSLContext.descr_get_options,
                              _SSLContext.descr_set_options),
+    verify_mode = GetSetProperty(_SSLContext.descr_get_verify_mode,
+                                 _SSLContext.descr_set_verify_mode),
 )
 
 
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -113,12 +113,20 @@
     def test_context(self):
         import _ssl
         s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+
         assert type(s.options) is long
         assert s.options & _ssl.OP_NO_SSLv2
         s.options &= ~_ssl.OP_NO_SSLv2
         assert not s.options & _ssl.OP_NO_SSLv2
         raises(TypeError, "s.options = 2.5")
 
+        assert s.verify_mode == _ssl.CERT_NONE
+        s.verify_mode = _ssl.CERT_REQUIRED
+        assert s.verify_mode == _ssl.CERT_REQUIRED
+        exc = raises(ValueError, "s.verify_mode = 1234")
+        assert str(exc.value) == "invalid value for verify_mode"
+
+
 class AppTestConnectedSSL:
     spaceconfig = {
         "usemodules": ['_ssl', '_socket', 'struct', 'binascii'],

From noreply at buildbot.pypy.org  Sun Dec 21 17:16:17 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sun, 21 Dec 2014 17:16:17 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: fix construction of ssl exceptions
	without errno
Message-ID: <20141221161617.9DCA11C0399@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75051:21765b094705
Date: 2014-12-21 11:16 -0500
http://bitbucket.org/pypy/pypy/changeset/21765b094705/

Log:	fix construction of ssl exceptions without errno

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -86,8 +86,11 @@
 
 def ssl_error(space, msg, errno=0, exc='w_sslerror'):
     w_exception_class = get_exception_class(space, exc)
-    w_exception = space.call_function(w_exception_class,
-                                      space.wrap(errno), space.wrap(msg))
+    if not errno:
+        w_exception = space.call_function(w_exception_class, space.wrap(msg))
+    else:
+        w_exception = space.call_function(w_exception_class,
+                                          space.wrap(errno), space.wrap(msg))
     return OperationError(w_exception_class, w_exception)
 
 if HAVE_OPENSSL_RAND:

From noreply at buildbot.pypy.org  Sun Dec 21 18:32:53 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Sun, 21 Dec 2014 18:32:53 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: provide SSLContext.check_hostname
Message-ID: <20141221173253.0624D1D2961@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75052:fcae7464203b
Date: 2014-12-21 12:32 -0500
http://bitbucket.org/pypy/pypy/changeset/fcae7464203b/

Log:	provide SSLContext.check_hostname

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -839,6 +839,7 @@
 
         self = space.allocate_instance(_SSLContext, w_subtype)
         self.ctx = ctx
+        self.check_hostname = False
         options = SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS
         if protocol != PY_SSL_VERSION_SSL2:
             options |= SSL_OP_NO_SSLv2
@@ -894,8 +895,23 @@
         else:
             raise oefmt(space.w_ValueError,
                         "invalid value for verify_mode")
+        if mode == SSL_VERIFY_NONE and self.check_hostname:
+            raise oefmt(space.w_ValueError,
+                        "Cannot set verify_mode to CERT_NONE when "
+                        "check_hostname is enabled.")
         libssl_SSL_CTX_set_verify(self.ctx, mode, None)
 
+    def descr_get_check_hostname(self, space):
+        return space.newbool(self.check_hostname)
+
+    def descr_set_check_hostname(self, space, w_obj):
+        check_hostname = space.is_true(w_obj)
+        if check_hostname and libssl_SSL_CTX_get_verify_mode(self.ctx) == SSL_VERIFY_NONE:
+            raise oefmt(space.w_ValueError,
+                        "check_hostname needs a SSL context with either "
+                        "CERT_OPTIONAL or CERT_REQUIRED")
+        self.check_hostname = check_hostname
+
 _SSLContext.typedef = TypeDef("_SSLContext",
     __module__ = "_ssl",
     __new__ = interp2app(_SSLContext.descr_new),
@@ -905,6 +921,8 @@
                              _SSLContext.descr_set_options),
     verify_mode = GetSetProperty(_SSLContext.descr_get_verify_mode,
                                  _SSLContext.descr_set_verify_mode),
+    check_hostname = GetSetProperty(_SSLContext.descr_get_check_hostname,
+                                    _SSLContext.descr_set_check_hostname),
 )
 
 
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -120,12 +120,24 @@
         assert not s.options & _ssl.OP_NO_SSLv2
         raises(TypeError, "s.options = 2.5")
 
+        assert not s.check_hostname
+        exc = raises(ValueError, "s.check_hostname = True")
+        assert str(exc.value) == "check_hostname needs a SSL context with " \
+                                 "either CERT_OPTIONAL or CERT_REQUIRED"
+
         assert s.verify_mode == _ssl.CERT_NONE
         s.verify_mode = _ssl.CERT_REQUIRED
         assert s.verify_mode == _ssl.CERT_REQUIRED
         exc = raises(ValueError, "s.verify_mode = 1234")
         assert str(exc.value) == "invalid value for verify_mode"
 
+        s.check_hostname = True
+        assert s.check_hostname
+
+        exc = raises(ValueError, "s.verify_mode = _ssl.CERT_NONE")
+        assert str(exc.value) == "Cannot set verify_mode to CERT_NONE " \
+                                 "when check_hostname is enabled."
+
 
 class AppTestConnectedSSL:
     spaceconfig = {

From noreply at buildbot.pypy.org  Sun Dec 21 20:05:15 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 21 Dec 2014 20:05:15 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Implemented
	set_default_verify_paths on SSLContext
Message-ID: <20141221190515.384741D3640@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75053:7660237a949e
Date: 2014-12-21 11:05 -0800
http://bitbucket.org/pypy/pypy/changeset/7660237a949e/

Log:	Implemented set_default_verify_paths on SSLContext

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -912,6 +912,11 @@
                         "CERT_OPTIONAL or CERT_REQUIRED")
         self.check_hostname = check_hostname
 
+    def descr_set_default_verify_paths(self, space):
+        if not libssl_SSL_CTX_set_default_verify_paths(self.ctx):
+            raise ssl_error(space, "")
+
+
 _SSLContext.typedef = TypeDef("_SSLContext",
     __module__ = "_ssl",
     __new__ = interp2app(_SSLContext.descr_new),
@@ -923,6 +928,7 @@
                                  _SSLContext.descr_set_verify_mode),
     check_hostname = GetSetProperty(_SSLContext.descr_get_check_hostname,
                                     _SSLContext.descr_set_check_hostname),
+    set_default_verify_paths=interp2app(_SSLContext.descr_set_default_verify_paths),
 )
 
 
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -138,6 +138,11 @@
         assert str(exc.value) == "Cannot set verify_mode to CERT_NONE " \
                                  "when check_hostname is enabled."
 
+    def test_set_default_verify_paths(self):
+        import _ssl
+        s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        s.set_default_verify_paths()
+
 
 class AppTestConnectedSSL:
     spaceconfig = {

From noreply at buildbot.pypy.org  Mon Dec 22 09:03:34 2014
From: noreply at buildbot.pypy.org (dalcinl)
Date: Mon, 22 Dec 2014 09:03:34 +0100 (CET)
Subject: [pypy-commit] cffi default: Silence GCC/Clang -Wunused-parameter
	warnings
Message-ID: <20141222080334.BFFCF1C3354@cobra.cs.uni-duesseldorf.de>

Author: Lisandro Dalcin <dalcinl at gmail.com>
Branch: 
Changeset: r1595:25bdcb810dc0
Date: 2014-11-24 21:31 +0300
http://bitbucket.org/cffi/cffi/changeset/25bdcb810dc0/

Log:	Silence GCC/Clang -Wunused-parameter warnings

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -65,7 +65,7 @@
         # The following two 'chained_list_constants' items contains
         # the head of these two chained lists, as a string that gives the
         # call to do, if any.
-        self._chained_list_constants = ['0', '0']
+        self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)']
         #
         prnt = self._prnt
         # first paste some standard set of lines that are mostly '#define'
@@ -337,7 +337,7 @@
         prnt = self._prnt
         numargs = len(tp.args)
         if numargs == 0:
-            argname = 'no_arg'
+            argname = 'noarg'
         elif numargs == 1:
             argname = 'arg0'
         else:
@@ -387,6 +387,9 @@
         prnt('  Py_END_ALLOW_THREADS')
         prnt()
         #
+        prnt('  (void)self; /* unused */')
+        if numargs == 0:
+            prnt('  (void)noarg; /* unused */')
         if result_code:
             prnt('  return %s;' %
                  self._convert_expr_from_c(tp.result, 'result', 'result type'))
@@ -453,6 +456,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
@@ -483,6 +487,8 @@
                 prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
+        prnt('  (void)self; /* unused */')
+        prnt('  (void)noarg; /* unused */')
         prnt('  return _cffi_get_struct_layout(nums);')
         prnt('  /* the next line is not executed, but compiled */')
         prnt('  %s(0);' % (checkfuncname,))
@@ -911,6 +917,7 @@
 {
     PyObject *library;
     int was_alive = (_cffi_types != NULL);
+    (void)self; /* unused */
     if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError,
                                        &library))
         return NULL;
diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -235,6 +235,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -430,7 +430,7 @@
     ffi = FFI()
     ffi.cdef("typedef struct foo_s foo_t; int bar(foo_t *);")
     lib = ffi.verify("typedef struct foo_s foo_t;\n"
-                     "int bar(foo_t *f) { return 42; }\n")
+                     "int bar(foo_t *f) { (void)f; return 42; }\n")
     assert lib.bar(ffi.NULL) == 42
 
 def test_ffi_full_struct():
@@ -1810,7 +1810,7 @@
 def test_typeof_function():
     ffi = FFI()
     ffi.cdef("int foo(int, char);")
-    lib = ffi.verify("int foo(int x, char y) { return 42; }")
+    lib = ffi.verify("int foo(int x, char y) { (void)x; (void)y; return 42; }")
     ctype = ffi.typeof(lib.foo)
     assert len(ctype.args) == 2
     assert ctype.result == ffi.typeof("int")

From noreply at buildbot.pypy.org  Mon Dec 22 09:03:35 2014
From: noreply at buildbot.pypy.org (dalcinl)
Date: Mon, 22 Dec 2014 09:03:35 +0100 (CET)
Subject: [pypy-commit] cffi default: Add GCC/Clang -Wall -Wextra flags in
	tests
Message-ID: <20141222080335.E21761C3354@cobra.cs.uni-duesseldorf.de>

Author: Lisandro Dalcin <dalcinl at gmail.com>
Branch: 
Changeset: r1596:03f384b183d1
Date: 2014-11-28 12:05 +0300
http://bitbucket.org/cffi/cffi/changeset/03f384b183d1/

Log:	Add GCC/Clang -Wall -Wextra flags in tests

diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -15,12 +15,12 @@
     if (sys.platform == 'darwin' and
           [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
         # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wconversion']
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
         # special things for clang
         extra_compile_args.append('-Qunused-arguments')
     else:
         # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wconversion']
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
 
     class FFI(FFI):
         def verify(self, *args, **kwds):

From noreply at buildbot.pypy.org  Mon Dec 22 09:03:37 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 09:03:37 +0100 (CET)
Subject: [pypy-commit] cffi default: Merged in dalcinl/python-cffi (pull
	request #52)
Message-ID: <20141222080337.0433B1C3354@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r1597:c095d7b92108
Date: 2014-12-22 09:04 +0100
http://bitbucket.org/cffi/cffi/changeset/c095d7b92108/

Log:	Merged in dalcinl/python-cffi (pull request #52)

	Silence GCC/Clang -Wunused-parameter warnings

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -65,7 +65,7 @@
         # The following two 'chained_list_constants' items contains
         # the head of these two chained lists, as a string that gives the
         # call to do, if any.
-        self._chained_list_constants = ['0', '0']
+        self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)']
         #
         prnt = self._prnt
         # first paste some standard set of lines that are mostly '#define'
@@ -344,7 +344,7 @@
         prnt = self._prnt
         numargs = len(tp.args)
         if numargs == 0:
-            argname = 'no_arg'
+            argname = 'noarg'
         elif numargs == 1:
             argname = 'arg0'
         else:
@@ -394,6 +394,9 @@
         prnt('  Py_END_ALLOW_THREADS')
         prnt()
         #
+        prnt('  (void)self; /* unused */')
+        if numargs == 0:
+            prnt('  (void)noarg; /* unused */')
         if result_code:
             prnt('  return %s;' %
                  self._convert_expr_from_c(tp.result, 'result', 'result type'))
@@ -460,6 +463,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
@@ -490,6 +494,8 @@
                 prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
+        prnt('  (void)self; /* unused */')
+        prnt('  (void)noarg; /* unused */')
         prnt('  return _cffi_get_struct_layout(nums);')
         prnt('  /* the next line is not executed, but compiled */')
         prnt('  %s(0);' % (checkfuncname,))
@@ -918,6 +924,7 @@
 {
     PyObject *library;
     int was_alive = (_cffi_types != NULL);
+    (void)self; /* unused */
     if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError,
                                        &library))
         return NULL;
diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -235,6 +235,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -15,12 +15,12 @@
     if (sys.platform == 'darwin' and
           [int(x) for x in os.uname()[2].split('.')] >= [11, 0, 0]):
         # assume a standard clang or gcc
-        extra_compile_args = ['-Werror', '-Wconversion']
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
         # special things for clang
         extra_compile_args.append('-Qunused-arguments')
     else:
         # assume a standard gcc
-        extra_compile_args = ['-Werror', '-Wconversion']
+        extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion']
 
     class FFI(FFI):
         def verify(self, *args, **kwds):
@@ -430,7 +430,7 @@
     ffi = FFI()
     ffi.cdef("typedef struct foo_s foo_t; int bar(foo_t *);")
     lib = ffi.verify("typedef struct foo_s foo_t;\n"
-                     "int bar(foo_t *f) { return 42; }\n")
+                     "int bar(foo_t *f) { (void)f; return 42; }\n")
     assert lib.bar(ffi.NULL) == 42
 
 def test_ffi_full_struct():
@@ -1810,7 +1810,7 @@
 def test_typeof_function():
     ffi = FFI()
     ffi.cdef("int foo(int, char);")
-    lib = ffi.verify("int foo(int x, char y) { return 42; }")
+    lib = ffi.verify("int foo(int x, char y) { (void)x; (void)y; return 42; }")
     ctype = ffi.typeof(lib.foo)
     assert len(ctype.args) == 2
     assert ctype.result == ffi.typeof("int")

From noreply at buildbot.pypy.org  Mon Dec 22 12:19:53 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 12:19:53 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: (ltratt, arigo)
Message-ID: <20141222111953.9CA781C046A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r75054:7972bae6fec9
Date: 2014-12-22 11:50 +0100
http://bitbucket.org/pypy/pypy/changeset/7972bae6fec9/

Log:	(ltratt, arigo)

	in-progress: simplify OrderedDict

diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -35,84 +35,19 @@
     # The remaining methods are order-aware.
     # Big-O running times for all methods are the same as regular dictionaries.
 
+    # XXX FIX THIS COMMENT
     # The internal self.__map dict maps keys to links in a doubly linked list.
     # The circular doubly linked list starts and ends with a sentinel element.
     # The sentinel element never gets deleted (this simplifies the algorithm).
     # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
 
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  The signature is the same as
-        regular dictionaries, but keyword arguments are not recommended because
-        their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__root = root = []                     # sentinel node
-            root[:] = [root, root, None]
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link at the end of the linked list,
-        # and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            root = self.__root
-            last = root[0]
-            last[1] = root[0] = self.__map[key] = [last, root, key]
-        return dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which gets
-        # removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link_prev, link_next, _ = self.__map.pop(key)
-        link_prev[1] = link_next                        # update link_prev[NEXT]
-        link_next[0] = link_prev                        # update link_next[PREV]
-
     def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        # Traverse the linked list in order.
-        root = self.__root
-        curr = root[1]                                  # start at the first node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[1]                              # move to next node
+        for x in self._pypy_mut_safe_iter():
+            yield x
 
     def __reversed__(self):
         'od.__reversed__() <==> reversed(od)'
-        # Traverse the linked list in reverse order.
-        root = self.__root
-        curr = root[0]                                  # start at the last node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[0]                              # move to previous node
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        root = self.__root
-        root[:] = [root, root, None]
-        self.__map.clear()
-        dict.clear(self)
-
-    # -- the following methods do not depend on the internal structure --
-
-    def keys(self):
-        'od.keys() -> list of keys in od'
-        return list(self)
-
-    def values(self):
-        'od.values() -> list of values in od'
-        return [self[key] for key in self]
-
-    def items(self):
-        'od.items() -> list of (key, value) pairs in od'
-        return [(key, self[key]) for key in self]
+        return XXX
 
     def iterkeys(self):
         'od.iterkeys() -> an iterator over the keys in od'
@@ -128,43 +63,18 @@
         for k in self:
             yield (k, self[k])
 
-    update = MutableMapping.update
-
-    __update = update # let subclasses override update without breaking __init__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
-        value.  If key is not found, d is returned if given, otherwise KeyError
-        is raised.
-
-        '''
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
-
     def popitem(self, last=True):
         '''od.popitem() -> (k, v), return and remove a (key, value) pair.
         Pairs are returned in LIFO order if last is true or FIFO order if false.
 
         '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        key = next(reversed(self) if last else iter(self))
-        value = self.pop(key)
-        return key, value
+        if last:
+            return dict.popitem(self)
+        else:
+            if not self:
+                raise KeyError('dictionary is empty')
+            k = iter(self).next()
+            return (k, self.pop(k))
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
@@ -183,8 +93,6 @@
         'Return state information for pickling'
         items = [[k, self[k]] for k in self]
         inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)

From noreply at buildbot.pypy.org  Mon Dec 22 12:19:54 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 12:19:54 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: Trying to finish this,
 using a slightly sub-efficient __iter__() method.
Message-ID: <20141222111954.E34761C046A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r75055:c07c71e5d2be
Date: 2014-12-22 12:19 +0100
http://bitbucket.org/pypy/pypy/changeset/c07c71e5d2be/

Log:	Trying to finish this, using a slightly sub-efficient __iter__()
	method. Too bad I suppose; the speed-up gained by using the
	simplified OrderedDict class should largely make up for it.

diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -29,25 +29,33 @@
 ################################################################################
 
 class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as regular dictionaries.
+    '''Dictionary that remembers insertion order.
 
-    # XXX FIX THIS COMMENT
-    # The internal self.__map dict maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+    In PyPy all dicts are ordered anyway.  This is mostly useful as a
+    placeholder to mean "this dict must be ordered even on CPython.'''
 
     def __iter__(self):
-        for x in self._pypy_mut_safe_iter():
-            yield x
+        # This method allows some concurrent changes to the dictionary
+        # while iterating.  The annoying part is that the exact allowed
+        # changes are messy to define and different than CPython's own
+        # messy definition (which the docs have nothing to say about).
+        # For now, we'll suppose it is good enough.  Precisely: we
+        # iterate over the list of keys grabbed at the start; we return
+        # all keys that are still in the dictionary at the time we
+        # reach them.  This is a simple rule, but if a key is deleted
+        # and re-added, this method will return it in its old position,
+        # which is arguably wrong.  Also, any newly-added key is never
+        # returned, unlike CPython (which usually returns them, but not
+        # always).
+        for k in dict.keys(self):
+            if k in self:
+                yield k
 
     def __reversed__(self):
         'od.__reversed__() <==> reversed(od)'
-        return XXX
+        for k in reversed(dict.keys(self)):
+            if k in self:
+                yield k
 
     def iterkeys(self):
         'od.iterkeys() -> an iterator over the keys in od'
@@ -71,9 +79,11 @@
         if last:
             return dict.popitem(self)
         else:
-            if not self:
+            it = dict.__iter__(self)
+            try:
+                k = it.next()
+            except StopIteration:
                 raise KeyError('dictionary is empty')
-            k = iter(self).next()
             return (k, self.pop(k))
 
     def __repr__(self, _repr_running={}):

From noreply at buildbot.pypy.org  Mon Dec 22 13:41:32 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 13:41:32 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: Fix,
	and tweak the tests with comments.
Message-ID: <20141222124132.32B701C243B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r75056:6c2e77d6ceb6
Date: 2014-12-22 13:41 +0100
http://bitbucket.org/pypy/pypy/changeset/6c2e77d6ceb6/

Log:	Fix, and tweak the tests with comments.

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -2,7 +2,7 @@
 
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.optimizeopt.util import args_dict
-from rpython.jit.metainterp.history import Const
+from rpython.jit.metainterp.history import Const, ConstInt
 from rpython.jit.metainterp.jitexc import JitException
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
@@ -307,6 +307,9 @@
         self.emit_operation(op)
 
     def _optimize_CALL_DICT_LOOKUP(self, op):
+        from rpython.rtyper.lltypesystem.rordereddict import FLAG_LOOKUP
+        if not op.getarg(4).same_box(ConstInt(FLAG_LOOKUP)):
+            return False
         descrs = op.getdescr().get_extra_info().extradescrs
         assert descrs        # translation hint
         descr1 = descrs[0]
diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -181,15 +181,21 @@
                 n = d[y]
             return d[Wrapper(str(n + 1))]
 
+        # XXX <arigo> unsure I see the point of this test: the repeated
+        # dict lookup is *not* elided so far, and the test happens to
+        # check this...  with rdict.py, it's a write followed by a read,
+        # where the dict cache is thrown away after the first lookup
+        # (correctly: we don't want the two lookups to return the exact
+        # same result!).  With rordereddict.py, FLAG_STORE lookups are
+        # not cached anyway.
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
         self.check_resops({'new_array_clear': 2, 'getfield_gc': 2,
-                           'guard_true': 2, 'jump': 1,
+                           'guard_true': 4, 'jump': 1,
                            'new_with_vtable': 2, 'getinteriorfield_gc': 2,
-                           'setfield_gc': 8, 'int_gt': 2, 'int_sub': 2,
-                           'call': 10, 'int_and': 2,
-                           'guard_no_exception': 8, 'new': 2,
-                           'guard_false': 2, 'int_is_true': 2})
+                           'setfield_gc': 14, 'int_gt': 2, 'int_sub': 2,
+                           'call': 10, 'int_ne': 2,
+                           'guard_no_exception': 8, 'new': 2})
 
     def test_unrolling_of_dict_iter(self):
         driver = JitDriver(greens = [], reds = ['n'])
@@ -223,7 +229,7 @@
             return s
 
         self.meta_interp(f, [10])
-        # XXX should be one getinteriorfield_gc
+        # XXX should be one getinteriorfield_gc.  At least it's one call.
         self.check_simple_loop(call=1, getinteriorfield_gc=2,
                                guard_no_exception=1)
 
@@ -244,7 +250,7 @@
             return s
 
         self.meta_interp(f, [10])
-        # XXX should be one getinteriorfield_gc
+        # XXX should be one getinteriorfield_gc.  At least it's one call.
         self.check_simple_loop(call=1, getinteriorfield_gc=2,
                                guard_no_exception=1)
 
@@ -259,7 +265,7 @@
                 driver.jit_merge_point()
                 index = indexes[n & 1]
                 s += d[index]
-                d['aa'] += 1 # this will invalidate the index
+                d['aa'] = 13 # this will invalidate the index
                 s += d[index]
                 n -= 1
             return s
@@ -291,6 +297,10 @@
         self.check_simple_loop(call=7)
 
     def test_dict_double_lookup_2(self):
+        py.test.skip("xxx reimplement me")
+        # one read and one write at the same key should be jitted as only
+        # one lookup, but it's a bit harder now with rordereddict.py
+
         driver = JitDriver(greens = [], reds = 'auto')
         indexes = ['aa', 'b', 'cc']
 
@@ -355,7 +365,8 @@
                     if n in mdict:
                         raise Exception
         self.meta_interp(f, [10])
-        self.check_simple_loop(call_may_force=0, call=3)
+        self.check_simple_loop(call_may_force=0, call=4)
+          # XXX should be call=3, same reason as test_dict_double_lookup_2
 
     def test_dict_virtual(self):
         myjitdriver = JitDriver(greens = [], reds = 'auto')
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -42,6 +42,7 @@
 #
 
 @jit.look_inside_iff(lambda d, key, hash, flag: jit.isvirtual(d))
+ at jit.oopspec('ordereddict.lookup(d, key, hash, flag)')
 def ll_call_lookup_function(d, key, hash, flag):
     fun = d.lookup_function_no
     if fun == FUNC_BYTE:

From noreply at buildbot.pypy.org  Mon Dec 22 14:51:19 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 14:51:19 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: OrderedDicts with the JIT:
 use intbounds to cache a pair "getitem /
Message-ID: <20141222135119.31AD91C243B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r75057:3e5bb8df5d0f
Date: 2014-12-22 14:50 +0100
http://bitbucket.org/pypy/pypy/changeset/3e5bb8df5d0f/

Log:	OrderedDicts with the JIT: use intbounds to cache a pair "getitem /
	setitem" on the same key.

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -6,6 +6,7 @@
 from rpython.jit.metainterp.jitexc import JitException
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
+from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.rlib.objectmodel import we_are_translated
@@ -307,9 +308,29 @@
         self.emit_operation(op)
 
     def _optimize_CALL_DICT_LOOKUP(self, op):
+        # Cache consecutive lookup() calls on the same dict and key,
+        # depending on the 'flag_store' argument passed:
+        # FLAG_LOOKUP: always cache and use the cached result.
+        # FLAG_STORE:  don't cache (it might return -1, which would be
+        #                incorrect for future lookups); but if found in
+        #                the cache and the cached value was already checked
+        #                non-negative, then we can reuse it.
+        # FLAG_DELETE: never cache, never use the cached result (because
+        #                if there is a cached result, the FLAG_DELETE call
+        #                is needed for its side-effect of removing it).
+        #                In theory we could cache a -1 for the case where
+        #                the delete is immediately followed by a lookup,
+        #                but too obscure.
+        #
         from rpython.rtyper.lltypesystem.rordereddict import FLAG_LOOKUP
-        if not op.getarg(4).same_box(ConstInt(FLAG_LOOKUP)):
+        from rpython.rtyper.lltypesystem.rordereddict import FLAG_STORE
+        flag_value = self.getvalue(op.getarg(4))
+        if not flag_value.is_constant():
             return False
+        flag = flag_value.get_constant_int()
+        if flag != FLAG_LOOKUP and flag != FLAG_STORE:
+            return False
+        #
         descrs = op.getdescr().get_extra_info().extradescrs
         assert descrs        # translation hint
         descr1 = descrs[0]
@@ -318,13 +339,20 @@
         except KeyError:
             d = self.cached_dict_reads[descr1] = args_dict()
             self.corresponding_array_descrs[descrs[1]] = descr1
-        args = self.optimizer.make_args_key(op)
+        #
+        key = [self.optimizer.get_arg_key(op.getarg(1)),   # dict
+               self.optimizer.get_arg_key(op.getarg(2))]   # key
+               # other args can be ignored here (hash, store_flag)
         try:
-            res_v = d[args]
+            res_v = d[key]
         except KeyError:
-            d[args] = self.getvalue(op.result)
+            if flag == FLAG_LOOKUP:
+                d[key] = self.getvalue(op.result)
             return False
         else:
+            if flag != FLAG_LOOKUP:
+                if not res_v.intbound.known_ge(IntBound(0, 0)):
+                    return False
             self.make_equal_to(op.result, res_v)
             self.last_emitted_operation = REMOVED
             return True
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -221,6 +221,12 @@
         if self.level < LEVEL_NONNULL:
             self.level = LEVEL_NONNULL
 
+    def get_constant_int(self):
+        assert self.is_constant()
+        box = self.box
+        assert isinstance(box, ConstInt)
+        return box.getint()
+
     def is_virtual(self):
         # Don't check this with 'isinstance(_, VirtualValue)'!
         # Even if it is a VirtualValue, the 'box' can be non-None,
@@ -625,18 +631,20 @@
                 descr.make_a_counter_per_value(op)
         return op
 
+    def get_arg_key(self, box):
+        try:
+            value = self.values[box]
+        except KeyError:
+            pass
+        else:
+            box = value.get_key_box()
+        return box
+
     def make_args_key(self, op):
         n = op.numargs()
         args = [None] * (n + 2)
         for i in range(n):
-            arg = op.getarg(i)
-            try:
-                value = self.values[arg]
-            except KeyError:
-                pass
-            else:
-                arg = value.get_key_box()
-            args[i] = arg
+            args[i] = self.get_arg_key(op.getarg(i))
         args[n] = ConstInt(op.getopnum())
         args[n + 1] = op.getdescr()
         return args
diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -194,7 +194,7 @@
                            'guard_true': 4, 'jump': 1,
                            'new_with_vtable': 2, 'getinteriorfield_gc': 2,
                            'setfield_gc': 14, 'int_gt': 2, 'int_sub': 2,
-                           'call': 10, 'int_ne': 2,
+                           'call': 10, 'int_ge': 2,
                            'guard_no_exception': 8, 'new': 2})
 
     def test_unrolling_of_dict_iter(self):
@@ -297,10 +297,6 @@
         self.check_simple_loop(call=7)
 
     def test_dict_double_lookup_2(self):
-        py.test.skip("xxx reimplement me")
-        # one read and one write at the same key should be jitted as only
-        # one lookup, but it's a bit harder now with rordereddict.py
-
         driver = JitDriver(greens = [], reds = 'auto')
         indexes = ['aa', 'b', 'cc']
 
@@ -366,7 +362,6 @@
                         raise Exception
         self.meta_interp(f, [10])
         self.check_simple_loop(call_may_force=0, call=4)
-          # XXX should be call=3, same reason as test_dict_double_lookup_2
 
     def test_dict_virtual(self):
         myjitdriver = JitDriver(greens = [], reds = 'auto')
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -526,7 +526,7 @@
 
 def ll_dict_getitem(d, key):
     index = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
-    if index != -1:
+    if index >= 0:
         return d.entries[index].value
     else:
         raise KeyError
@@ -721,7 +721,7 @@
 
 def ll_dict_delitem(d, key):
     index = d.lookup_function(d, key, d.keyhash(key), FLAG_DELETE)
-    if index == -1:
+    if index < 0:
         raise KeyError
     _ll_dict_del(d, index)
 
@@ -1024,7 +1024,7 @@
 
 def ll_dict_get(dict, key, default):
     index = dict.lookup_function(dict, key, dict.keyhash(key), FLAG_LOOKUP)
-    if index == -1:
+    if index < 0:
         return default
     else:
         return dict.entries[index].value
@@ -1032,7 +1032,7 @@
 def ll_dict_setdefault(dict, key, default):
     hash = dict.keyhash(key)
     index = dict.lookup_function(dict, key, hash, FLAG_STORE)
-    if index == -1:
+    if index < 0:
         _ll_dict_setitem_lookup_done(dict, key, default, hash, -1)
         return default
     else:
@@ -1159,7 +1159,7 @@
 
 def ll_dict_contains(d, key):
     i = d.lookup_function(d, key, d.keyhash(key), FLAG_LOOKUP)
-    return i != -1
+    return i >= 0
 
 def _ll_getnextitem(dic):
     if dic.num_live_items == 0:
@@ -1192,7 +1192,7 @@
 
 def ll_dict_pop(dic, key):
     index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
-    if index == -1:
+    if index < 0:
         raise KeyError
     value = dic.entries[index].value
     _ll_dict_del(dic, index)
@@ -1200,7 +1200,7 @@
 
 def ll_dict_pop_default(dic, key, dfl):
     index = dic.lookup_function(dic, key, dic.keyhash(key), FLAG_DELETE)
-    if index == -1:
+    if index < 0:
         return dfl
     value = dic.entries[index].value
     _ll_dict_del(dic, index)

From noreply at buildbot.pypy.org  Mon Dec 22 14:53:06 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 14:53:06 +0100 (CET)
Subject: [pypy-commit] pypy default: update test
Message-ID: <20141222135306.2EFD11D2961@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75058:a47e4969db25
Date: 2014-12-22 14:52 +0100
http://bitbucket.org/pypy/pypy/changeset/a47e4969db25/

Log:	update test

diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -397,7 +397,7 @@
 def test_invalid_indexing():
     p = new_primitive_type("int")
     x = cast(p, 42)
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises(TypeError, "x[0]")
 
 def test_default_str():
     BChar = new_primitive_type("char")

From noreply at buildbot.pypy.org  Mon Dec 22 14:59:57 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 14:59:57 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix: dict ordering issue
Message-ID: <20141222135957.D726D1D2961@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75059:218ec8282891
Date: 2014-12-22 14:59 +0100
http://bitbucket.org/pypy/pypy/changeset/218ec8282891/

Log:	Fix: dict ordering issue

diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -15,14 +15,15 @@
         assert NotEmpty.__dict__.get("b") is None
         raises(TypeError, 'NotEmpty.__dict__[15] = "y"')
         raises(KeyError, 'del NotEmpty.__dict__[15]')
+
+        key, value = NotEmpty.__dict__.popitem()
+        assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
+
         assert NotEmpty.__dict__.setdefault("string", 1) == 1
         assert NotEmpty.__dict__.setdefault("string", 2) == 1
         assert NotEmpty.string == 1
         raises(TypeError, 'NotEmpty.__dict__.setdefault(15, 1)')
 
-        key, value = NotEmpty.__dict__.popitem()
-        assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
-
     def test_dictproxy_getitem(self):
         class NotEmpty(object):
             a = 1

From noreply at buildbot.pypy.org  Mon Dec 22 15:28:47 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 15:28:47 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: fix test
Message-ID: <20141222142847.1BA021C046A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r75060:bc4e9c652d8d
Date: 2014-12-22 15:28 +0100
http://bitbucket.org/pypy/pypy/changeset/bc4e9c652d8d/

Log:	fix test

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -1877,7 +1877,7 @@
                 return None
         a = self.RPythonAnnotator()
         s = a.build_types(f, [int])
-        assert s.knowntype == dict
+        assert s.knowntype == annmodel.SomeOrderedDict.knowntype
 
     def test_const_list_and_none(self):
         def g(l=None):

From noreply at buildbot.pypy.org  Mon Dec 22 15:59:40 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 15:59:40 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: fix, skip, or kill tests
Message-ID: <20141222145940.90B2C1C243B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r75061:33f58b3c63f5
Date: 2014-12-22 15:42 +0100
http://bitbucket.org/pypy/pypy/changeset/33f58b3c63f5/

Log:	fix, skip, or kill tests

diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -1005,6 +1005,7 @@
 
 
     def test_dict_resize(self):
+        py.test.skip("test written for non-ordered dicts, update or kill")
         # XXX we no longer automatically resize on 'del'.  We need to
         # hack a bit in this test to trigger a resize by continuing to
         # fill the dict's table while keeping the actual size very low
@@ -1025,7 +1026,7 @@
         res = self.interpret(func, [1])
         assert len(res.entries) == rdict.DICT_INITSIZE
 
-    def test_opt_nullkeymarker(self):
+    def test_opt_dummykeymarker(self):
         def f():
             d = {"hello": None}
             d["world"] = None
@@ -1033,10 +1034,9 @@
         res = self.interpret(f, [])
         assert res.item0 == True
         DICT = lltype.typeOf(res.item1).TO
-        assert not hasattr(DICT.entries.TO.OF, 'f_everused')# non-None string keys
-        assert not hasattr(DICT.entries.TO.OF, 'f_valid')   # strings have a dummy
+        assert not hasattr(DICT.entries.TO.OF, 'f_valid')   # strs have a dummy
 
-    def test_opt_nullvaluemarker(self):
+    def test_opt_dummyvaluemarker(self):
         def f(n):
             d = {-5: "abcd"}
             d[123] = "def"
@@ -1044,29 +1044,8 @@
         res = self.interpret(f, [-5])
         assert res.item0 == 4
         DICT = lltype.typeOf(res.item1).TO
-        assert not hasattr(DICT.entries.TO.OF, 'f_everused')# non-None str values
         assert not hasattr(DICT.entries.TO.OF, 'f_valid')   # strs have a dummy
 
-    def test_opt_nonullmarker(self):
-        class A:
-            pass
-        def f(n):
-            if n > 5:
-                a = A()
-            else:
-                a = None
-            d = {a: -5441}
-            d[A()] = n+9872
-            return d[a], d
-        res = self.interpret(f, [-5])
-        assert res.item0 == -5441
-        DICT = lltype.typeOf(res.item1).TO
-        assert hasattr(DICT.entries.TO.OF, 'f_everused') # can-be-None A instances
-        assert not hasattr(DICT.entries.TO.OF, 'f_valid')# with a dummy A instance
-
-        res = self.interpret(f, [6])
-        assert res.item0 == -5441
-
     def test_opt_nonnegint_dummy(self):
         def f(n):
             d = {n: 12}
@@ -1077,7 +1056,6 @@
         assert res.item0 == 1
         assert res.item1 == 24
         DICT = lltype.typeOf(res.item2).TO
-        assert hasattr(DICT.entries.TO.OF, 'f_everused') # all ints can be zero
         assert not hasattr(DICT.entries.TO.OF, 'f_valid')# nonneg int: dummy -1
 
     def test_opt_no_dummy(self):
@@ -1090,7 +1068,6 @@
         assert res.item0 == 1
         assert res.item1 == -24
         DICT = lltype.typeOf(res.item2).TO
-        assert hasattr(DICT.entries.TO.OF, 'f_everused') # all ints can be zero
         assert hasattr(DICT.entries.TO.OF, 'f_valid')    # no dummy available
 
     def test_opt_boolean_has_no_dummy(self):
@@ -1103,7 +1080,6 @@
         assert res.item0 == 1
         assert res.item1 is True
         DICT = lltype.typeOf(res.item2).TO
-        assert hasattr(DICT.entries.TO.OF, 'f_everused') # all ints can be zero
         assert hasattr(DICT.entries.TO.OF, 'f_valid')    # no dummy available
 
     def test_opt_multiple_identical_dicts(self):
@@ -1142,6 +1118,7 @@
         assert sorted(DICT.TO.entries.TO.OF._flds) == ['f_hash', 'key', 'value']
 
     def test_deleted_entry_reusage_with_colliding_hashes(self):
+        py.test.skip("test written for non-ordered dicts, update or kill")
         def lowlevelhash(value):
             p = rstr.mallocstr(len(value))
             for i in range(len(value)):

From noreply at buildbot.pypy.org  Mon Dec 22 17:23:35 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 17:23:35 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: hg merge default
Message-ID: <20141222162335.85FC01C0399@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r75062:6b50206e647d
Date: 2014-12-22 17:18 +0100
http://bitbucket.org/pypy/pypy/changeset/6b50206e647d/

Log:	hg merge default

diff too long, truncating to 2000 out of 2244 lines

diff --git a/lib_pypy/readline.py b/lib_pypy/readline.py
--- a/lib_pypy/readline.py
+++ b/lib_pypy/readline.py
@@ -6,4 +6,11 @@
 are only stubs at the moment.
 """
 
-from pyrepl.readline import *
+try:
+    from pyrepl.readline import *
+except ImportError:
+    import sys
+    if sys.platform == 'win32':
+        raise ImportError("the 'readline' module is not available on Windows"
+                          " (on either PyPy or CPython)")
+    raise
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -397,7 +397,7 @@
 def test_invalid_indexing():
     p = new_primitive_type("int")
     x = cast(p, 42)
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises(TypeError, "x[0]")
 
 def test_default_str():
     BChar = new_primitive_type("char")
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -618,8 +618,8 @@
         try:
             i = self.find(w_value, 0, sys.maxint)
         except ValueError:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("list.remove(x): x not in list"))
+            raise oefmt(space.w_ValueError,
+                        "list.remove(): %R is not in list", w_value)
         if i < self.length():  # otherwise list was mutated
             self.pop(i)
 
@@ -633,8 +633,7 @@
         try:
             i = self.find(w_value, i, stop)
         except ValueError:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("list.index(x): x not in list"))
+            raise oefmt(space.w_ValueError, "%R is not in list", w_value)
         return space.wrap(i)
 
     @unwrap_spec(reverse=bool)
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -15,14 +15,15 @@
         assert NotEmpty.__dict__.get("b") is None
         raises(TypeError, 'NotEmpty.__dict__[15] = "y"')
         raises(KeyError, 'del NotEmpty.__dict__[15]')
+
+        key, value = NotEmpty.__dict__.popitem()
+        assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
+
         assert NotEmpty.__dict__.setdefault("string", 1) == 1
         assert NotEmpty.__dict__.setdefault("string", 2) == 1
         assert NotEmpty.string == 1
         raises(TypeError, 'NotEmpty.__dict__.setdefault(15, 1)')
 
-        key, value = NotEmpty.__dict__.popitem()
-        assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
-
     def test_dictproxy_getitem(self):
         class NotEmpty(object):
             a = 1
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -976,7 +976,10 @@
 
         c = [0.0, 2.2, 4.4]
         assert c.index(0) == 0.0
-        raises(ValueError, c.index, 3)
+        e = raises(ValueError, c.index, 3)
+        import sys
+        if sys.version_info[:2] == (2, 7):     # CPython 2.7, PyPy
+            assert str(e.value) == '3 is not in list'
 
     def test_index_cpython_bug(self):
         if self.on_cpython:
@@ -1228,7 +1231,9 @@
         assert l == [0.0, 1.1, 3.3, 4.4]
         l = [0.0, 3.3, 5.5]
         raises(ValueError, c.remove, 2)
-        raises(ValueError, c.remove, 2.2)
+        e = raises(ValueError, c.remove, 2.2)
+        if not self.on_cpython:
+            assert str(e.value) == 'list.remove(): 2.2 is not in list'
 
     def test_reverse(self):
         c = list('hello world')
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -389,7 +389,7 @@
         assert isinstance(dct2, SomeOrderedDict), "OrderedDict.update(dict) not allowed"
         dct1.dictdef.union(dct2.dictdef)
 
-SomeDict=SomeOrderedDict
+SomeDict = SomeOrderedDict      # all dicts are ordered!
 
 
 class SomeIterator(SomeObject):
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -128,7 +128,8 @@
                       [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
 
     try:
-        start_state = optimize_trace(metainterp_sd, part, enable_opts)
+        start_state = optimize_trace(metainterp_sd, part, enable_opts,
+                                     export_state=True)
     except InvalidLoop:
         return None
     target_token = part.operations[0].getdescr()
@@ -156,7 +157,7 @@
 
         try:
             optimize_trace(metainterp_sd, part, enable_opts,
-                           start_state=start_state)
+                           start_state=start_state, export_state=False)
         except InvalidLoop:
             return None
 
@@ -209,7 +210,7 @@
     assert label.getopnum() == rop.LABEL
     try:
         optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
-                       start_state=start_state)
+                       start_state=start_state, export_state=False)
     except InvalidLoop:
         # Fall back on jumping to preamble
         target_token = label.getdescr()
@@ -220,7 +221,8 @@
         try:
             optimize_trace(metainterp_sd, part,
                            jitdriver_sd.warmstate.enable_opts,
-                           inline_short_preamble=False, start_state=start_state)
+                           inline_short_preamble=False, start_state=start_state,
+                           export_state=False)
         except InvalidLoop:
             return None
     assert part.operations[-1].getopnum() != rop.LABEL
@@ -789,7 +791,8 @@
     else:
         inline_short_preamble = True
     try:
-        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts, inline_short_preamble)
+        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts,
+                               inline_short_preamble, export_state=True)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -48,7 +48,8 @@
     return optimizations, unroll
 
 def optimize_trace(metainterp_sd, loop, enable_opts,
-                   inline_short_preamble=True, start_state=None):
+                   inline_short_preamble=True, start_state=None,
+                   export_state=True):
     """Optimize loop.operations to remove internal overheadish operations.
     """
 
@@ -59,7 +60,8 @@
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
             return optimize_unroll(metainterp_sd, loop, optimizations,
-                                   inline_short_preamble, start_state)
+                                   inline_short_preamble, start_state,
+                                   export_state)
         else:
             optimizer = Optimizer(metainterp_sd, loop, optimizations)
             optimizer.propagate_all_forward()
diff --git a/rpython/jit/metainterp/optimizeopt/generalize.py b/rpython/jit/metainterp/optimizeopt/generalize.py
--- a/rpython/jit/metainterp/optimizeopt/generalize.py
+++ b/rpython/jit/metainterp/optimizeopt/generalize.py
@@ -1,4 +1,5 @@
-from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT
+from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT,\
+     IntOptValue
 
 
 class GeneralizationStrategy(object):
@@ -14,7 +15,8 @@
         for v in self.optimizer.values.values():
             if v.is_constant():
                 continue
-            if v.intbound.lower < MININT / 2:
-                v.intbound.lower = MININT
-            if v.intbound.upper > MAXINT / 2:
-                v.intbound.upper = MAXINT
+            if isinstance(v, IntOptValue):
+                if v.intbound.lower < MININT / 2:
+                    v.intbound.lower = MININT
+                if v.intbound.upper > MAXINT / 2:
+                    v.intbound.upper = MAXINT
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt.util import args_dict
 from rpython.jit.metainterp.history import Const, ConstInt
 from rpython.jit.metainterp.jitexc import JitException
-from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED
+from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED, LEVEL_CONSTANT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimize import InvalidLoop
@@ -64,6 +64,17 @@
             # cancelling its previous effects with no side effect.
             self._lazy_setfield = None
 
+    def value_updated(self, oldvalue, newvalue):
+        try:
+            fieldvalue = self._cached_fields[oldvalue]
+        except KeyError:
+            pass
+        else:
+            self._cached_fields[newvalue] = fieldvalue
+            op = self._cached_fields_getfield_op[oldvalue].clone()
+            op.setarg(0, newvalue.box)
+            self._cached_fields_getfield_op[newvalue] = op
+
     def possible_aliasing(self, optheap, structvalue):
         # If lazy_setfield is set and contains a setfield on a different
         # structvalue, then we are annoyed, because it may point to either
@@ -118,18 +129,6 @@
         self._cached_fields.clear()
         self._cached_fields_getfield_op.clear()
 
-    def turned_constant(self, newvalue, value):
-        if newvalue not in self._cached_fields and value in self._cached_fields:
-            self._cached_fields[newvalue] = self._cached_fields[value]
-            op = self._cached_fields_getfield_op[value].clone()
-            constbox = value.box
-            assert isinstance(constbox, Const)
-            op.setarg(0, constbox)
-            self._cached_fields_getfield_op[newvalue] = op
-        for structvalue in self._cached_fields.keys():
-            if self._cached_fields[structvalue] is value:
-                self._cached_fields[structvalue] = newvalue
-
     def produce_potential_short_preamble_ops(self, optimizer, shortboxes, descr):
         if self._lazy_setfield is not None:
             return
@@ -187,6 +186,17 @@
         self._seen_guard_not_invalidated = False
         self.postponed_op = None
 
+    def setup(self):
+        self.optimizer.optheap = self
+
+    def value_updated(self, oldvalue, newvalue):
+        # XXXX very unhappy about that
+        for cf in self.cached_fields.itervalues():
+            cf.value_updated(oldvalue, newvalue)
+        for submap in self.cached_arrayitems.itervalues():
+            for cf in submap.itervalues():
+                cf.value_updated(oldvalue, newvalue)
+
     def force_at_end_of_preamble(self):
         self.cached_dict_reads.clear()
         self.corresponding_array_descrs.clear()
@@ -391,16 +401,6 @@
             # ^^^ we only need to force this field; the other fields
             # of virtualref_info and virtualizable_info are not gcptrs.
 
-    def turned_constant(self, value):
-        assert value.is_constant()
-        newvalue = self.getvalue(value.box)
-        if value is not newvalue:
-            for cf in self.cached_fields.itervalues():
-                cf.turned_constant(newvalue, value)
-            for submap in self.cached_arrayitems.itervalues():
-                for cf in submap.itervalues():
-                    cf.turned_constant(newvalue, value)
-
     def force_lazy_setfield(self, descr, can_cache=True):
         try:
             cf = self.cached_fields[descr]
@@ -414,7 +414,7 @@
         except KeyError:
             return
         for idx, cf in submap.iteritems():
-            if indexvalue is None or indexvalue.intbound.contains(idx):
+            if indexvalue is None or indexvalue.getintbound().contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
     def _assert_valid_cf(self, cf):
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt.intutils import (IntBound, IntLowerBound,
     IntUpperBound)
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1,
-    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE)
+    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE, IntOptValue)
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.backend.llsupport import symbolic
@@ -54,7 +54,9 @@
         #        but the bounds produced by all instructions where box is
         #        an argument might also be tighten
         v = self.getvalue(box)
-        b = v.intbound
+        b = v.getintbound()
+        if b is None:
+            return # pointer
         if b.has_lower and b.has_upper and b.lower == b.upper:
             v.make_constant(ConstInt(b.lower))
 
@@ -81,11 +83,13 @@
                 self.make_constant_int(op.result, 0)
             return
         self.emit_operation(op)
-        if v1.intbound.known_ge(IntBound(0, 0)) and \
-           v2.intbound.known_ge(IntBound(0, 0)):
-            r = self.getvalue(op.result)
-            mostsignificant = v1.intbound.upper | v2.intbound.upper
-            r.intbound.intersect(IntBound(0, next_pow2_m1(mostsignificant)))
+        bound1 = v1.getintbound()
+        bound2 = v2.getintbound()
+        if bound1.known_ge(IntBound(0, 0)) and \
+           bound2.known_ge(IntBound(0, 0)):
+            r = self.getvalue(op.result).getintbound()
+            mostsignificant = bound1.upper | bound2.upper
+            r.intersect(IntBound(0, next_pow2_m1(mostsignificant)))
 
     optimize_INT_OR = optimize_INT_OR_or_XOR
     optimize_INT_XOR = optimize_INT_OR_or_XOR
@@ -99,55 +103,55 @@
         if v2.is_constant():
             val = v2.box.getint()
             if val >= 0:
-                r.intbound.intersect(IntBound(0, val))
+                r.getintbound().intersect(IntBound(0, val))
         elif v1.is_constant():
             val = v1.box.getint()
             if val >= 0:
-                r.intbound.intersect(IntBound(0, val))
-        elif v1.intbound.known_ge(IntBound(0, 0)) and \
-          v2.intbound.known_ge(IntBound(0, 0)):
-            lesser = min(v1.intbound.upper, v2.intbound.upper)
-            r.intbound.intersect(IntBound(0, next_pow2_m1(lesser)))
+                r.getintbound().intersect(IntBound(0, val))
+        elif v1.getintbound().known_ge(IntBound(0, 0)) and \
+          v2.getintbound().known_ge(IntBound(0, 0)):
+            lesser = min(v1.getintbound().upper, v2.getintbound().upper)
+            r.getintbound().intersect(IntBound(0, next_pow2_m1(lesser)))
 
     def optimize_INT_SUB(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        b = v1.intbound.sub_bound(v2.intbound)
+        b = v1.getintbound().sub_bound(v2.getintbound())
         if b.bounded():
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        b = v1.intbound.add_bound(v2.intbound)
+        b = v1.getintbound().add_bound(v2.getintbound())
         if b.bounded():
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        b = v1.intbound.mul_bound(v2.intbound)
+        b = v1.getintbound().mul_bound(v2.getintbound())
         if b.bounded():
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        r.intbound.intersect(v1.intbound.div_bound(v2.intbound))
+        r.getintbound().intersect(v1.getintbound().div_bound(v2.getintbound()))
 
     def optimize_INT_MOD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        known_nonneg = (v1.intbound.known_ge(IntBound(0, 0)) and
-                        v2.intbound.known_ge(IntBound(0, 0)))
+        known_nonneg = (v1.getintbound().known_ge(IntBound(0, 0)) and
+                        v2.getintbound().known_ge(IntBound(0, 0)))
         if known_nonneg and v2.is_constant():
             val = v2.box.getint()
             if (val & (val-1)) == 0:
@@ -164,18 +168,18 @@
                     return     # give up
                 val = -val
             if known_nonneg:
-                r.intbound.make_ge(IntBound(0, 0))
+                r.getintbound().make_ge(IntBound(0, 0))
             else:
-                r.intbound.make_gt(IntBound(-val, -val))
-            r.intbound.make_lt(IntBound(val, val))
+                r.getintbound().make_gt(IntBound(-val, -val))
+            r.getintbound().make_lt(IntBound(val, val))
 
     def optimize_INT_LSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        b = v1.intbound.lshift_bound(v2.intbound)
-        r.intbound.intersect(b)
+        b = v1.getintbound().lshift_bound(v2.getintbound())
+        r.getintbound().intersect(b)
         # intbound.lshift_bound checks for an overflow and if the
         # lshift can be proven not to overflow sets b.has_upper and
         # b.has_lower
@@ -186,14 +190,14 @@
     def optimize_INT_RSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        b = v1.intbound.rshift_bound(v2.intbound)
+        b = v1.getintbound().rshift_bound(v2.getintbound())
         if b.has_lower and b.has_upper and b.lower == b.upper:
             # constant result (likely 0, for rshifts that kill all bits)
             self.make_constant_int(op.result, b.lower)
         else:
             self.emit_operation(op)
             r = self.getvalue(op.result)
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_GUARD_NO_OVERFLOW(self, op):
         lastop = self.last_emitted_operation
@@ -238,7 +242,7 @@
     def optimize_INT_ADD_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        resbound = v1.intbound.add_bound(v2.intbound)
+        resbound = v1.getintbound().add_bound(v2.getintbound())
         if resbound.bounded():
             # Transform into INT_ADD.  The following guard will be killed
             # by optimize_GUARD_NO_OVERFLOW; if we see instead an
@@ -246,7 +250,7 @@
             op = op.copy_and_change(rop.INT_ADD)
         self.emit_operation(op) # emit the op
         r = self.getvalue(op.result)
-        r.intbound.intersect(resbound)
+        r.getintbound().intersect(resbound)
 
     def optimize_INT_SUB_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -254,29 +258,29 @@
         if v1 is v2:
             self.make_constant_int(op.result, 0)
             return
-        resbound = v1.intbound.sub_bound(v2.intbound)
+        resbound = v1.getintbound().sub_bound(v2.getintbound())
         if resbound.bounded():
             op = op.copy_and_change(rop.INT_SUB)
         self.emit_operation(op) # emit the op
         r = self.getvalue(op.result)
-        r.intbound.intersect(resbound)
+        r.getintbound().intersect(resbound)
 
     def optimize_INT_MUL_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        resbound = v1.intbound.mul_bound(v2.intbound)
+        resbound = v1.getintbound().mul_bound(v2.getintbound())
         if resbound.bounded():
             op = op.copy_and_change(rop.INT_MUL)
         self.emit_operation(op)
         r = self.getvalue(op.result)
-        r.intbound.intersect(resbound)
+        r.getintbound().intersect(resbound)
 
     def optimize_INT_LT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_lt(v2.intbound):
+        if v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op.result, 1)
-        elif v1.intbound.known_ge(v2.intbound) or v1 is v2:
+        elif v1.getintbound().known_ge(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
@@ -284,9 +288,9 @@
     def optimize_INT_GT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_gt(v2.intbound):
+        if v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op.result, 1)
-        elif v1.intbound.known_le(v2.intbound) or v1 is v2:
+        elif v1.getintbound().known_le(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
@@ -294,9 +298,9 @@
     def optimize_INT_LE(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_le(v2.intbound) or v1 is v2:
+        if v1.getintbound().known_le(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op.result, 1)
-        elif v1.intbound.known_gt(v2.intbound):
+        elif v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
@@ -304,9 +308,9 @@
     def optimize_INT_GE(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_ge(v2.intbound) or v1 is v2:
+        if v1.getintbound().known_ge(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op.result, 1)
-        elif v1.intbound.known_lt(v2.intbound):
+        elif v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op.result, 0)
         else:
             self.emit_operation(op)
@@ -314,9 +318,9 @@
     def optimize_INT_EQ(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_gt(v2.intbound):
+        if v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op.result, 0)
-        elif v1.intbound.known_lt(v2.intbound):
+        elif v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op.result, 0)
         elif v1 is v2:
             self.make_constant_int(op.result, 1)
@@ -326,9 +330,9 @@
     def optimize_INT_NE(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_gt(v2.intbound):
+        if v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op.result, 1)
-        elif v1.intbound.known_lt(v2.intbound):
+        elif v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op.result, 1)
         elif v1 is v2:
             self.make_constant_int(op.result, 0)
@@ -337,7 +341,7 @@
 
     def optimize_INT_FORCE_GE_ZERO(self, op):
         value = self.getvalue(op.getarg(0))
-        if value.intbound.known_ge(IntBound(0, 0)):
+        if value.getintbound().known_ge(IntBound(0, 0)):
             self.make_equal_to(op.result, value)
         else:
             self.emit_operation(op)
@@ -348,50 +352,53 @@
         start = -(1 << (numbits - 1))
         stop = 1 << (numbits - 1)
         bounds = IntBound(start, stop - 1)
-        if bounds.contains_bound(value.intbound):
+        if bounds.contains_bound(value.getintbound()):
             self.make_equal_to(op.result, value)
         else:
             self.emit_operation(op)
             vres = self.getvalue(op.result)
-            vres.intbound.intersect(bounds)
+            vres.getintbound().intersect(bounds)
 
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
         result = self.getvalue(op.result)
         array.make_len_gt(MODE_ARRAY, op.getdescr(), -1)
-        array.lenbound.bound.intersect(result.intbound)
-        result.intbound = array.lenbound.bound
+        array.getlenbound().bound.intersect(result.getintbound())
+        assert isinstance(result, IntOptValue)
+        result.intbound = array.getlenbound().bound
 
     def optimize_STRLEN(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
         result = self.getvalue(op.result)
         array.make_len_gt(MODE_STR, op.getdescr(), -1)
-        array.lenbound.bound.intersect(result.intbound)
-        result.intbound = array.lenbound.bound
+        array.getlenbound().bound.intersect(result.getintbound())
+        assert isinstance(result, IntOptValue)
+        result.intbound = array.getlenbound().bound
 
     def optimize_UNICODELEN(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
         result = self.getvalue(op.result)
         array.make_len_gt(MODE_UNICODE, op.getdescr(), -1)
-        array.lenbound.bound.intersect(result.intbound)
-        result.intbound = array.lenbound.bound
+        array.getlenbound().bound.intersect(result.getintbound())
+        assert isinstance(result, IntOptValue)        
+        result.intbound = array.getlenbound().bound
 
     def optimize_STRGETITEM(self, op):
         self.emit_operation(op)
         v1 = self.getvalue(op.result)
-        v1.intbound.make_ge(IntLowerBound(0))
-        v1.intbound.make_lt(IntUpperBound(256))
+        v1.getintbound().make_ge(IntLowerBound(0))
+        v1.getintbound().make_lt(IntUpperBound(256))
 
     def optimize_GETFIELD_RAW(self, op):
         self.emit_operation(op)
         descr = op.getdescr()
         if descr.is_integer_bounded():
             v1 = self.getvalue(op.result)
-            v1.intbound.make_ge(IntLowerBound(descr.get_integer_min()))
-            v1.intbound.make_le(IntUpperBound(descr.get_integer_max()))
+            v1.getintbound().make_ge(IntLowerBound(descr.get_integer_min()))
+            v1.getintbound().make_le(IntUpperBound(descr.get_integer_max()))
 
     optimize_GETFIELD_GC = optimize_GETFIELD_RAW
 
@@ -402,30 +409,30 @@
         descr = op.getdescr()
         if descr and descr.is_item_integer_bounded():
             v1 = self.getvalue(op.result)
-            v1.intbound.make_ge(IntLowerBound(descr.get_item_integer_min()))
-            v1.intbound.make_le(IntUpperBound(descr.get_item_integer_max()))
+            v1.getintbound().make_ge(IntLowerBound(descr.get_item_integer_min()))
+            v1.getintbound().make_le(IntUpperBound(descr.get_item_integer_max()))
 
     optimize_GETARRAYITEM_GC = optimize_GETARRAYITEM_RAW
 
     def optimize_UNICODEGETITEM(self, op):
         self.emit_operation(op)
         v1 = self.getvalue(op.result)
-        v1.intbound.make_ge(IntLowerBound(0))
+        v1.getintbound().make_ge(IntLowerBound(0))
 
     def make_int_lt(self, box1, box2):
         v1 = self.getvalue(box1)
         v2 = self.getvalue(box2)
-        if v1.intbound.make_lt(v2.intbound):
+        if v1.getintbound().make_lt(v2.getintbound()):
             self.propagate_bounds_backward(box1)
-        if v2.intbound.make_gt(v1.intbound):
+        if v2.getintbound().make_gt(v1.getintbound()):
             self.propagate_bounds_backward(box2)
 
     def make_int_le(self, box1, box2):
         v1 = self.getvalue(box1)
         v2 = self.getvalue(box2)
-        if v1.intbound.make_le(v2.intbound):
+        if v1.getintbound().make_le(v2.getintbound()):
             self.propagate_bounds_backward(box1)
-        if v2.intbound.make_ge(v1.intbound):
+        if v2.getintbound().make_ge(v1.getintbound()):
             self.propagate_bounds_backward(box2)
 
     def make_int_gt(self, box1, box2):
@@ -472,9 +479,9 @@
             if r.box.same_constant(CONST_1):
                 v1 = self.getvalue(op.getarg(0))
                 v2 = self.getvalue(op.getarg(1))
-                if v1.intbound.intersect(v2.intbound):
+                if v1.getintbound().intersect(v2.getintbound()):
                     self.propagate_bounds_backward(op.getarg(0))
-                if v2.intbound.intersect(v1.intbound):
+                if v2.getintbound().intersect(v1.getintbound()):
                     self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_NE(self, op):
@@ -483,9 +490,9 @@
             if r.box.same_constant(CONST_0):
                 v1 = self.getvalue(op.getarg(0))
                 v2 = self.getvalue(op.getarg(1))
-                if v1.intbound.intersect(v2.intbound):
+                if v1.getintbound().intersect(v2.getintbound()):
                     self.propagate_bounds_backward(op.getarg(0))
-                if v2.intbound.intersect(v1.intbound):
+                if v2.getintbound().intersect(v1.getintbound()):
                     self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_IS_TRUE(self, op):
@@ -493,8 +500,8 @@
         if r.is_constant():
             if r.box.same_constant(CONST_1):
                 v1 = self.getvalue(op.getarg(0))
-                if v1.intbound.known_ge(IntBound(0, 0)):
-                    v1.intbound.make_gt(IntBound(0, 0))
+                if v1.getintbound().known_ge(IntBound(0, 0)):
+                    v1.getintbound().make_gt(IntBound(0, 0))
                     self.propagate_bounds_backward(op.getarg(0))
 
     def propagate_bounds_INT_IS_ZERO(self, op):
@@ -505,49 +512,49 @@
                 # Clever hack, we can't use self.make_constant_int yet because
                 # the args aren't in the values dictionary yet so it runs into
                 # an assert, this is a clever way of expressing the same thing.
-                v1.intbound.make_ge(IntBound(0, 0))
-                v1.intbound.make_lt(IntBound(1, 1))
+                v1.getintbound().make_ge(IntBound(0, 0))
+                v1.getintbound().make_lt(IntBound(1, 1))
                 self.propagate_bounds_backward(op.getarg(0))
 
     def propagate_bounds_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         r = self.getvalue(op.result)
-        b = r.intbound.sub_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        b = r.getintbound().sub_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
-        b = r.intbound.sub_bound(v1.intbound)
-        if v2.intbound.intersect(b):
+        b = r.getintbound().sub_bound(v1.getintbound())
+        if v2.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_SUB(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         r = self.getvalue(op.result)
-        b = r.intbound.add_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        b = r.getintbound().add_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
-        b = r.intbound.sub_bound(v1.intbound).mul(-1)
-        if v2.intbound.intersect(b):
+        b = r.getintbound().sub_bound(v1.getintbound()).mul(-1)
+        if v2.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         r = self.getvalue(op.result)
-        b = r.intbound.div_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        b = r.getintbound().div_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
-        b = r.intbound.div_bound(v1.intbound)
-        if v2.intbound.intersect(b):
+        b = r.getintbound().div_bound(v1.getintbound())
+        if v2.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_LSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         r = self.getvalue(op.result)
-        b = r.intbound.rshift_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        b = r.getintbound().rshift_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0))
 
     propagate_bounds_INT_ADD_OVF = propagate_bounds_INT_ADD
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -1,9 +1,11 @@
 from rpython.jit.metainterp import jitprof, resume, compile
 from rpython.jit.metainterp.executor import execute_nonspec
-from rpython.jit.metainterp.history import BoxInt, BoxFloat, Const, ConstInt, REF
-from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded, \
+from rpython.jit.metainterp.history import BoxInt, BoxFloat, Const, ConstInt,\
+     REF, BoxPtr, ConstPtr, ConstFloat
+from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded,\
                                                      ImmutableIntUnbounded, \
-                                                     IntLowerBound, MININT, MAXINT
+                                                     IntLowerBound, MININT,\
+                                                     MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractResOp
 from rpython.jit.metainterp.typesystem import llhelper
@@ -39,69 +41,19 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
-    last_guard = None
+    _attrs_ = ('box', 'level')
 
     level = LEVEL_UNKNOWN
-    known_class = None
-    intbound = ImmutableIntUnbounded()
-    lenbound = None
 
     def __init__(self, box, level=None, known_class=None, intbound=None):
         self.box = box
         if level is not None:
             self.level = level
-        self.known_class = known_class
-        if intbound:
-            self.intbound = intbound
-        else:
-            if isinstance(box, BoxInt):
-                self.intbound = IntBound(MININT, MAXINT)
-            else:
-                self.intbound = IntUnbounded()
 
         if isinstance(box, Const):
             self.make_constant(box)
         # invariant: box is a Const if and only if level == LEVEL_CONSTANT
 
-    def make_len_gt(self, mode, descr, val):
-        if self.lenbound:
-            assert self.lenbound.mode == mode
-            assert self.lenbound.descr == descr
-            self.lenbound.bound.make_gt(IntBound(val, val))
-        else:
-            self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))
-
-    def make_guards(self, box):
-        guards = []
-        if self.level == LEVEL_CONSTANT:
-            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
-            guards.append(op)
-        elif self.level == LEVEL_KNOWNCLASS:
-            op = ResOperation(rop.GUARD_NONNULL, [box], None)
-            guards.append(op)
-            op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
-            guards.append(op)
-        else:
-            if self.level == LEVEL_NONNULL:
-                op = ResOperation(rop.GUARD_NONNULL, [box], None)
-                guards.append(op)
-            self.intbound.make_guards(box, guards)
-            if self.lenbound:
-                lenbox = BoxInt()
-                if self.lenbound.mode == MODE_ARRAY:
-                    op = ResOperation(rop.ARRAYLEN_GC, [box], lenbox, self.lenbound.descr)
-                elif self.lenbound.mode == MODE_STR:
-                    op = ResOperation(rop.STRLEN, [box], lenbox, self.lenbound.descr)
-                elif self.lenbound.mode == MODE_UNICODE:
-                    op = ResOperation(rop.UNICODELEN, [box], lenbox, self.lenbound.descr)
-                else:
-                    debug_print("Unknown lenbound mode")
-                    assert False
-                guards.append(op)
-                self.lenbound.bound.make_guards(lenbox, guards)
-        return guards
-
     def import_from(self, other, optimizer):
         if self.level == LEVEL_CONSTANT:
             assert other.level == LEVEL_CONSTANT
@@ -110,21 +62,22 @@
         assert self.level <= LEVEL_NONNULL
         if other.level == LEVEL_CONSTANT:
             self.make_constant(other.get_key_box())
-            optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, None)
+            self.make_constant_class(other.get_known_class(), None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
-            self.intbound.intersect(other.intbound)
-            if other.lenbound:
-                if self.lenbound:
-                    assert other.lenbound.mode == self.lenbound.mode
-                    assert other.lenbound.descr == self.lenbound.descr
-                    self.lenbound.bound.intersect(other.lenbound.bound)
-                else:
-                    self.lenbound = other.lenbound.clone()
 
+    def make_guards(self, box):
+        if self.level == LEVEL_CONSTANT:
+            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
+            return [op]
+        return []
+
+    def copy_from(self, other_value):
+        assert isinstance(other_value, OptValue)
+        self.box = other_value.box
+        self.level = other_value.level
 
     def force_box(self, optforce):
         return self.box
@@ -168,38 +121,6 @@
             return self.box.same_constant(other.box)
         return self is other
 
-    def make_constant(self, constbox):
-        """Replace 'self.box' with a Const box."""
-        assert isinstance(constbox, Const)
-        self.box = constbox
-        self.level = LEVEL_CONSTANT
-
-        if isinstance(constbox, ConstInt):
-            val = constbox.getint()
-            self.intbound = IntBound(val, val)
-        else:
-            self.intbound = IntUnbounded()
-
-    def get_constant_class(self, cpu):
-        level = self.level
-        if level == LEVEL_KNOWNCLASS:
-            return self.known_class
-        elif level == LEVEL_CONSTANT:
-            return cpu.ts.cls_of_box(self.box)
-        else:
-            return None
-
-    def make_constant_class(self, classbox, guardop):
-        assert self.level < LEVEL_KNOWNCLASS
-        self.known_class = classbox
-        self.level = LEVEL_KNOWNCLASS
-        self.last_guard = guardop
-
-    def make_nonnull(self, guardop):
-        assert self.level < LEVEL_NONNULL
-        self.level = LEVEL_NONNULL
-        self.last_guard = guardop
-
     def is_nonnull(self):
         level = self.level
         if level == LEVEL_NONNULL or level == LEVEL_KNOWNCLASS:
@@ -208,12 +129,6 @@
             box = self.box
             assert isinstance(box, Const)
             return box.nonnull()
-        elif self.intbound:
-            if self.intbound.known_gt(IntBound(0, 0)) or \
-               self.intbound.known_lt(IntBound(0, 0)):
-                return True
-            else:
-                return False
         else:
             return False
 
@@ -266,8 +181,216 @@
     def get_missing_null_value(self):
         raise NotImplementedError    # only for VArrayValue
 
+    def make_constant(self, constbox):
+        """Replace 'self.box' with a Const box."""
+        assert isinstance(constbox, Const)
+        self.box = constbox
+        self.level = LEVEL_CONSTANT
 
-class ConstantValue(OptValue):
+    def get_last_guard(self):
+        return None
+
+    def get_known_class(self):
+        return None
+
+    def getlenbound(self):
+        return None
+
+    def getintbound(self):
+        return None
+
+    def get_constant_class(self, cpu):
+        return None
+
+class PtrOptValue(OptValue):
+    _attrs_ = ('known_class', 'last_guard', 'lenbound')
+
+    known_class = None
+    last_guard = None
+    lenbound = None
+
+    def __init__(self, box, level=None, known_class=None, intbound=None):
+        OptValue.__init__(self, box, level, None, intbound)
+        if not isinstance(box, Const):
+            self.known_class = known_class
+
+    def copy_from(self, other_value):
+        assert isinstance(other_value, PtrOptValue)
+        self.box = other_value.box
+        self.known_class = other_value.known_class
+        self.level = other_value.level
+        self.last_guard = other_value.last_guard
+        self.lenbound = other_value.lenbound
+
+    def make_len_gt(self, mode, descr, val):
+        if self.lenbound:
+            assert self.lenbound.mode == mode
+            assert self.lenbound.descr == descr
+            self.lenbound.bound.make_gt(IntBound(val, val))
+        else:
+            self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))
+
+    def make_nonnull(self, guardop):
+        assert self.level < LEVEL_NONNULL
+        self.level = LEVEL_NONNULL
+        self.last_guard = guardop
+
+    def make_constant_class(self, classbox, guardop):
+        assert self.level < LEVEL_KNOWNCLASS
+        self.known_class = classbox
+        self.level = LEVEL_KNOWNCLASS
+        self.last_guard = guardop
+
+    def import_from(self, other, optimizer):
+        OptValue.import_from(self, other, optimizer)
+        if self.level != LEVEL_CONSTANT:
+            if other.getlenbound():
+                if self.lenbound:
+                    assert other.getlenbound().mode == self.lenbound.mode
+                    assert other.getlenbound().descr == self.lenbound.descr
+                    self.lenbound.bound.intersect(other.getlenbound().bound)
+                else:
+                    self.lenbound = other.getlenbound().clone()
+
+    def make_guards(self, box):
+        guards = []
+        if self.level == LEVEL_CONSTANT:
+            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
+            guards.append(op)
+        elif self.level == LEVEL_KNOWNCLASS:
+            op = ResOperation(rop.GUARD_NONNULL, [box], None)
+            guards.append(op)
+            op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
+            guards.append(op)
+        else:
+            if self.level == LEVEL_NONNULL:
+                op = ResOperation(rop.GUARD_NONNULL, [box], None)
+                guards.append(op)
+            if self.lenbound:
+                lenbox = BoxInt()
+                if self.lenbound.mode == MODE_ARRAY:
+                    op = ResOperation(rop.ARRAYLEN_GC, [box], lenbox, self.lenbound.descr)
+                elif self.lenbound.mode == MODE_STR:
+                    op = ResOperation(rop.STRLEN, [box], lenbox, self.lenbound.descr)
+                elif self.lenbound.mode == MODE_UNICODE:
+                    op = ResOperation(rop.UNICODELEN, [box], lenbox, self.lenbound.descr)
+                else:
+                    debug_print("Unknown lenbound mode")
+                    assert False
+                guards.append(op)
+                self.lenbound.bound.make_guards(lenbox, guards)
+        return guards
+
+    def get_constant_class(self, cpu):
+        level = self.level
+        if level == LEVEL_KNOWNCLASS:
+            return self.known_class
+        elif level == LEVEL_CONSTANT:
+            return cpu.ts.cls_of_box(self.box)
+        else:
+            return None
+
+    def getlenbound(self):
+        return self.lenbound
+
+    def get_last_guard(self):
+        return self.last_guard
+
+    def get_known_class(self):
+        return self.known_class
+
+class IntOptValue(OptValue):
+    _attrs_ = ('intbound',)
+
+    intbound = ImmutableIntUnbounded()
+
+    def __init__(self, box, level=None, known_class=None, intbound=None):
+        OptValue.__init__(self, box, level, None, None)
+        if isinstance(box, Const):
+            return
+        if intbound:
+            self.intbound = intbound
+        else:
+            if isinstance(box, BoxInt):
+                self.intbound = IntBound(MININT, MAXINT)
+            else:
+                self.intbound = IntUnbounded()
+
+    def copy_from(self, other_value):
+        assert isinstance(other_value, IntOptValue)
+        self.box = other_value.box
+        self.intbound = other_value.intbound
+        self.level = other_value.level
+
+    def make_constant(self, constbox):
+        """Replace 'self.box' with a Const box."""
+        assert isinstance(constbox, ConstInt)
+        self.box = constbox
+        self.level = LEVEL_CONSTANT
+        val = constbox.getint()
+        self.intbound = IntBound(val, val)
+
+    def is_nonnull(self):
+        if OptValue.is_nonnull(self):
+            return True
+        if self.intbound:
+            if self.intbound.known_gt(IntBound(0, 0)) or \
+               self.intbound.known_lt(IntBound(0, 0)):
+                return True
+        return False
+
+    def make_nonnull(self, guardop):
+        assert self.level < LEVEL_NONNULL
+        self.level = LEVEL_NONNULL
+
+    def import_from(self, other, optimizer):
+        OptValue.import_from(self, other, optimizer)
+        if self.level != LEVEL_CONSTANT:
+            if other.getintbound() is not None: # VRawBufferValue
+                self.intbound.intersect(other.getintbound())
+
+    def make_guards(self, box):
+        guards = []
+        if self.level == LEVEL_CONSTANT:
+            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
+            guards.append(op)
+        elif self.level == LEVEL_KNOWNCLASS:
+            op = ResOperation(rop.GUARD_NONNULL, [box], None)
+            guards.append(op)
+        else:
+            if self.level == LEVEL_NONNULL:
+                op = ResOperation(rop.GUARD_NONNULL, [box], None)
+                guards.append(op)
+            self.intbound.make_guards(box, guards)
+        return guards
+
+    def getintbound(self):
+        return self.intbound
+
+    def get_last_guard(self):
+        return None
+
+    def get_known_class(self):
+        return None
+
+    def getlenbound(self):
+        return None
+
+class ConstantFloatValue(OptValue):
+    def __init__(self, box):
+        self.make_constant(box)
+
+    def __repr__(self):
+        return 'Constant(%r)' % (self.box,)
+
+class ConstantIntValue(IntOptValue):
+    def __init__(self, box):
+        self.make_constant(box)
+
+    def __repr__(self):
+        return 'Constant(%r)' % (self.box,)
+
+class ConstantPtrValue(PtrOptValue):
     def __init__(self, box):
         self.make_constant(box)
 
@@ -276,9 +399,9 @@
 
 CONST_0      = ConstInt(0)
 CONST_1      = ConstInt(1)
-CVAL_ZERO    = ConstantValue(CONST_0)
-CVAL_ZERO_FLOAT = ConstantValue(Const._new(0.0))
-llhelper.CVAL_NULLREF = ConstantValue(llhelper.CONST_NULL)
+CVAL_ZERO    = ConstantIntValue(CONST_0)
+CVAL_ZERO_FLOAT = ConstantFloatValue(Const._new(0.0))
+llhelper.CVAL_NULLREF = ConstantPtrValue(llhelper.CONST_NULL)
 REMOVED = AbstractResOp(None)
 
 
@@ -305,8 +428,8 @@
     def make_constant_int(self, box, intconst):
         return self.optimizer.make_constant_int(box, intconst)
 
-    def make_equal_to(self, box, value):
-        return self.optimizer.make_equal_to(box, value)
+    def make_equal_to(self, box, value, replace=False):
+        return self.optimizer.make_equal_to(box, value, replace=replace)
 
     def get_constant_box(self, box):
         return self.optimizer.get_constant_box(box)
@@ -340,9 +463,6 @@
     def setup(self):
         pass
 
-    def turned_constant(self, value):
-        pass
-
     def force_at_end_of_preamble(self):
         pass
 
@@ -390,9 +510,9 @@
         self.opaque_pointers = {}
         self.replaces_guard = {}
         self._newoperations = []
-        self.seen_results = {}
         self.optimizer = self
         self.optpure = None
+        self.optheap = None
         self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
@@ -428,10 +548,6 @@
         for opt in self.optimizations:
             opt.produce_potential_short_preamble_ops(sb)
 
-    def turned_constant(self, value):
-        for o in self.optimizations:
-            o.turned_constant(value)
-
     def forget_numberings(self, virtualbox):
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_FORCINGS)
         self.resumedata_memo.forget_numberings(virtualbox)
@@ -457,7 +573,12 @@
         try:
             value = self.values[box]
         except KeyError:
-            value = self.values[box] = OptValue(box)
+            if isinstance(box, BoxPtr) or isinstance(box, ConstPtr):
+                value = self.values[box] = PtrOptValue(box)
+            elif isinstance(box, BoxInt) or isinstance(box, ConstInt):
+                value = self.values[box] = IntOptValue(box)
+            else:
+                value = self.values[box] = OptValue(box)
         self.ensure_imported(value)
         return value
 
@@ -485,15 +606,31 @@
 
     def clear_newoperations(self):
         self._newoperations = []
-        self.seen_results = {}
 
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
-        assert replace or box not in self.values
+        if replace:
+            try:
+                cur_value = self.values[box]
+            except KeyError:
+                pass
+            else:
+                assert value.level != LEVEL_CONSTANT
+                assert cur_value.level != LEVEL_CONSTANT
+                # replacing with a different box
+                cur_value.copy_from(value)
+                return
         self.values[box] = value
 
     def make_constant(self, box, constbox):
-        self.make_equal_to(box, ConstantValue(constbox))
+        if isinstance(constbox, ConstInt):
+            self.make_equal_to(box, ConstantIntValue(constbox))
+        elif isinstance(constbox, ConstPtr):
+            self.make_equal_to(box, ConstantPtrValue(constbox))
+        elif isinstance(constbox, ConstFloat):
+            self.make_equal_to(box, ConstantFloatValue(constbox))
+        else:
+            assert False
 
     def make_constant_int(self, box, intvalue):
         self.make_constant(box, ConstInt(intvalue))
@@ -547,7 +684,8 @@
         self.first_optimization.propagate_forward(op)
 
     def propagate_forward(self, op):
-        self.producer[op.result] = op
+        if op.result is not None:
+            self.producer[op.result] = op
         dispatch_opt(self, op)
 
     def emit_operation(self, op):
@@ -580,10 +718,6 @@
                 op = self.store_final_boxes_in_guard(op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        if op.result:
-            if op.result in self.seen_results:
-                raise ValueError, "invalid optimization"
-            self.seen_results[op.result] = None
         self._newoperations.append(op)
 
     def replace_op(self, old_op, new_op):
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -25,6 +25,7 @@
         else:
             nextop = None
 
+        args = None
         if canfold:
             for i in range(op.numargs()):
                 if self.get_constant_box(op.getarg(i)) is None:
@@ -39,15 +40,11 @@
 
             # did we do the exact same operation already?
             args = self.optimizer.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.optimizer.make_equal_to(op.result,
-                                             self.getvalue(oldop.result),
-                                             True)
+            oldvalue = self.pure_operations.get(args, None)
+            if oldvalue is not None:
+                self.optimizer.make_equal_to(op.result, oldvalue, True)
                 return
             else:
-                self.pure_operations[args] = op
                 self.remember_emitting_pure(op)
 
         # otherwise, the operation remains
@@ -56,6 +53,8 @@
             self.optimizer.bool_boxes[self.getvalue(op.result)] = None
         if nextop:
             self.emit_operation(nextop)
+        if args is not None:
+            self.pure_operations[args] = self.getvalue(op.result)
 
     def optimize_CALL_PURE(self, op):
         # Step 1: check if all arguments are constant
@@ -69,16 +68,15 @@
         # Step 2: check if all arguments are the same as a previous
         # CALL_PURE.
         args = self.optimizer.make_args_key(op)
-        oldop = self.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
+        oldvalue = self.pure_operations.get(args, None)
+        if oldvalue is not None:
             # this removes a CALL_PURE that has the same (non-constant)
             # arguments as a previous CALL_PURE.
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            self.make_equal_to(op.result, oldvalue)
             self.last_emitted_operation = REMOVED
             return
         else:
-            self.pure_operations[args] = op
+            self.pure_operations[args] = self.getvalue(op.result)
             self.remember_emitting_pure(op)
 
         # replace CALL_PURE with just CALL
@@ -103,15 +101,12 @@
         op = ResOperation(opnum, args, result)
         key = self.optimizer.make_args_key(op)
         if key not in self.pure_operations:
-            self.pure_operations[key] = op
+            self.pure_operations[key] = self.getvalue(result)
 
     def has_pure_result(self, opnum, args, descr):
         op = ResOperation(opnum, args, None, descr)
         key = self.optimizer.make_args_key(op)
-        op = self.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        return self.pure_operations.get(key, None) is not None
 
     def get_pure_result(self, key):
         return self.pure_operations.get(key, None)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
-    CONST_0, CONST_1)
+    CONST_0, CONST_1, PtrOptValue)
 from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses
 from rpython.rlib.rarithmetic import highest_bit
@@ -33,9 +33,8 @@
         dispatch_opt(self, op)
 
     def try_boolinvers(self, op, targs):
-        oldop = self.get_pure_result(targs)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            value = self.getvalue(oldop.result)
+        value = self.get_pure_result(targs)
+        if value is not None:
             if value.is_constant():
                 if value.box.same_constant(CONST_1):
                     self.make_constant(op.result, CONST_0)
@@ -59,9 +58,9 @@
         if oldopnum != -1:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.get_pure_result(targs)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                self.make_equal_to(op.result, self.getvalue(oldop.result))
+            value = self.get_pure_result(targs)
+            if value is not None:
+                self.optimizer.make_equal_to(op.result, value, True)
                 return True
 
         if op.boolreflex == -1:
@@ -83,14 +82,14 @@
             return
         elif v2.is_constant():
             val = v2.box.getint()
-            if val == -1 or v1.intbound.lower >= 0 \
-                and v1.intbound.upper <= val & ~(val + 1):
+            if val == -1 or v1.getintbound().lower >= 0 \
+                and v1.getintbound().upper <= val & ~(val + 1):
                 self.make_equal_to(op.result, v1)
                 return
         elif v1.is_constant():
             val = v1.box.getint()
-            if val == -1 or v2.intbound.lower >= 0 \
-                and v2.intbound.upper <= val & ~(val + 1):
+            if val == -1 or v2.getintbound().lower >= 0 \
+                and v2.getintbound().upper <= val & ~(val + 1):
                 self.make_equal_to(op.result, v2)
                 return
 
@@ -262,7 +261,8 @@
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
-        self.optimizer.turned_constant(value)
+        if self.optimizer.optheap:
+            self.optimizer.optheap.value_updated(value, self.getvalue(constbox))
 
     def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
@@ -296,11 +296,11 @@
             else:
                 name = "<unknown>"
             raise InvalidLoop('A promote of a virtual %s (a recently allocated object) never makes sense!' % name)
-        if value.last_guard:
+        if value.get_last_guard():
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = value.last_guard
+            old_guard_op = value.get_last_guard()
             if old_guard_op.getopnum() != rop.GUARD_NONNULL:
                 # This is only safe if the class of the guard_value matches the
                 # class of the guard_*_class, otherwise the intermediate ops might
@@ -323,7 +323,8 @@
             descr.guard_opnum = rop.GUARD_VALUE
             descr.make_a_counter_per_value(op)
             # to be safe
-            value.last_guard = None
+            if isinstance(value, PtrOptValue):
+                value.last_guard = None
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
         self.optimize_guard(op, constbox)
@@ -355,6 +356,7 @@
             r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
             raise InvalidLoop('A GUARD_CLASS (%s) was proven to always fail'
                               % r)
+        assert isinstance(value, PtrOptValue)
         if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
@@ -564,7 +566,7 @@
         elif v1.is_constant() and v1.box.getint() == 0:
             self.make_constant_int(op.result, 0)
             return
-        if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
+        if v1.getintbound().known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
                 op = op.copy_and_change(rop.INT_RSHIFT,
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -45,7 +45,8 @@
             part.operations = operations
 
             self.add_guard_future_condition(part)
-            state = self._do_optimize_loop(part, None, state)
+            state = self._do_optimize_loop(part, None, state,
+                                           export_state=True)
             if part.operations[-1].getopnum() == rop.LABEL:
                 last_label = [part.operations.pop()]
             else:
@@ -497,7 +498,8 @@
 
 class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel):
 
-    def _do_optimize_loop(self, loop, call_pure_results, state):
+    def _do_optimize_loop(self, loop, call_pure_results, state,
+                          export_state=False):
         from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
         from rpython.jit.metainterp.optimizeopt.util import args_dict
         from rpython.jit.metainterp.optimizeopt.pure import OptPure
@@ -505,7 +507,7 @@
         self.loop = loop
         loop.call_pure_results = args_dict()
         metainterp_sd = FakeMetaInterpStaticData(self.cpu)
-        return optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True, state)
+        return optimize_unroll(metainterp_sd, loop, [OptRewrite(), OptRenameStrlen(), OptHeap(), OptPure()], True, state, export_state)
 
     def test_optimizer_renaming_boxes1(self):
         ops = """
@@ -543,8 +545,8 @@
         self.optimize_loop(ops, expected)
 
 
-class TestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin):
+class XxxTestLLtype(OptimizeoptTestMultiLabel, LLtypeMixin):
     pass
 
-class TestOptimizerRenamingBoxesLLtype(BaseTestOptimizerRenamingBoxes, LLtypeMixin):
+class XxxTestOptimizerRenamingBoxesLLtype(BaseTestOptimizerRenamingBoxes, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -104,7 +104,7 @@
         if loop.operations[-1].getopnum() == rop.JUMP:
             loop.operations[-1].setdescr(token)
         expected = convert_old_style_to_targets(self.parse(optops), jump=True)
-        self._do_optimize_loop(loop, call_pure_results)
+        self._do_optimize_loop(loop, call_pure_results, export_state=False)
         print '\n'.join([str(o) for o in loop.operations])
         self.assert_equal(loop, expected)
 
@@ -999,6 +999,40 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct_arraycopy(self):
+        ops = """
+        [f0, f1]
+        p0 = new_array_clear(3, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=compleximagdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=complexrealdescr)
+        call(0, p0, p0, 0, 2, 1, descr=complexarraycopydescr)
+        f2 = getinteriorfield_gc(p0, 2, descr=complexrealdescr)
+        f3 = getinteriorfield_gc(p0, 2, descr=compleximagdescr)
+        escape(f2)
+        escape(f3)
+        finish(1)
+        """
+        expected = """
+        [f0, f1]
+        escape(f1)
+        escape(f0)
+        finish(1)
+        """
+        self.optimize_loop(ops, ops)
+        py.test.skip("XXX missing optimization: ll_arraycopy(array-of-structs)")
+
+    def test_nonvirtual_array_of_struct_arraycopy(self):
+        ops = """
+        [p0]
+        call(0, p0, p0, 0, 2, 1, descr=complexarraycopydescr)
+        f2 = getinteriorfield_gc(p0, 2, descr=compleximagdescr)
+        f3 = getinteriorfield_gc(p0, 2, descr=complexrealdescr)
+        escape(f2)
+        escape(f3)
+        finish(1)
+        """
+        self.optimize_loop(ops, ops)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -223,6 +223,10 @@
     complexarraydescr = cpu.arraydescrof(complexarray)
     complexrealdescr = cpu.interiorfielddescrof(complexarray, "real")
     compleximagdescr = cpu.interiorfielddescrof(complexarray, "imag")
+    complexarraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+            EffectInfo([], [complexarraydescr], [], [], [complexarraydescr], [],
+                       EffectInfo.EF_CANNOT_RAISE,
+                       oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     rawarraydescr = cpu.arraydescrof(lltype.Array(lltype.Signed,
                                                   hints={'nolength': True}))
@@ -390,7 +394,8 @@
         assert equaloplists(optimized.operations,
                             expected.operations, False, remap, text_right)
 
-    def _do_optimize_loop(self, loop, call_pure_results, start_state=None):
+    def _do_optimize_loop(self, loop, call_pure_results, start_state=None,
+                          export_state=False):
         from rpython.jit.metainterp.optimizeopt import optimize_trace
         from rpython.jit.metainterp.optimizeopt.util import args_dict
 
@@ -406,7 +411,8 @@
             metainterp_sd.callinfocollection = self.callinfocollection
         #
         return optimize_trace(metainterp_sd, loop, self.enable_opts,
-                              start_state=start_state)
+                              start_state=start_state,
+                              export_state=export_state)
 
     def unroll_and_optimize(self, loop, call_pure_results=None):
         self.add_guard_future_condition(loop)
@@ -426,7 +432,8 @@
         preamble.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(token))] + \
                               operations +  \
                               [ResOperation(rop.LABEL, jump_args, None, descr=token)]
-        start_state = self._do_optimize_loop(preamble, call_pure_results)
+        start_state = self._do_optimize_loop(preamble, call_pure_results,
+                                             export_state=True)
 
         assert preamble.operations[-1].getopnum() == rop.LABEL
 
@@ -440,7 +447,8 @@
         assert loop.operations[0].getopnum() == rop.LABEL
         loop.inputargs = loop.operations[0].getarglist()
 
-        self._do_optimize_loop(loop, call_pure_results, start_state)
+        self._do_optimize_loop(loop, call_pure_results, start_state,
+                               export_state=False)
         extra_same_as = []
         while loop.operations[0].getopnum() != rop.LABEL:
             extra_same_as.append(loop.operations[0])
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -3,7 +3,8 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo, VStructStateInfo, \
      VArrayStateInfo, NotVirtualStateInfo, VirtualState, ShortBoxes, GenerateGuardState, \
      VirtualStatesCantMatch, VArrayStructStateInfo
-from rpython.jit.metainterp.optimizeopt.optimizer import OptValue
+from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, PtrOptValue,\
+      IntOptValue
 from rpython.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, ConstInt, ConstPtr
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin, BaseTest, \
@@ -121,13 +122,13 @@
         assert isgeneral(OptValue(BoxInt()), OptValue(ConstInt(7)))
         assert not isgeneral(OptValue(ConstInt(7)), OptValue(BoxInt()))
 
-        ptr = OptValue(BoxPtr())
-        nonnull = OptValue(BoxPtr())
+        ptr = PtrOptValue(BoxPtr())
+        nonnull = PtrOptValue(BoxPtr())
         nonnull.make_nonnull(0)
-        knownclass = OptValue(BoxPtr())
+        knownclass = PtrOptValue(BoxPtr())
         clsbox = self.cpu.ts.cls_of_box(BoxPtr(self.myptr))
         knownclass.make_constant_class(clsbox, 0)
-        const = OptValue(BoxPtr)
+        const = PtrOptValue(BoxPtr)
         const.make_constant_class(clsbox, 0)
         const.make_constant(ConstPtr(self.myptr))
         inorder = [ptr, nonnull, knownclass, const]
@@ -137,8 +138,8 @@
                 if i != j:
                     assert not isgeneral(inorder[j], inorder[i])
 
-        value1 = OptValue(BoxInt())
-        value2 = OptValue(BoxInt())
+        value1 = IntOptValue(BoxInt())
+        value2 = IntOptValue(BoxInt())
         value2.intbound.make_lt(IntBound(10, 10))
         assert isgeneral(value1, value2)
         assert not isgeneral(value2, value1)
@@ -150,9 +151,9 @@
         assert isgeneral(OptValue(ConstPtr(fooref)),
                          OptValue(ConstPtr(fooref)))
 
-        value1 = OptValue(BoxPtr())
+        value1 = PtrOptValue(BoxPtr())
         value1.make_nonnull(None)
-        value2 = OptValue(ConstPtr(self.nullptr))
+        value2 = PtrOptValue(ConstPtr(self.nullptr))
         assert not isgeneral(value1, value2)
 
     def test_field_matching_generalization(self):
@@ -178,18 +179,18 @@
         fldtst(VArrayStructStateInfo(fakedescr, [[fielddescr]]), VArrayStructStateInfo(fakedescr, [[fielddescr]]))
 
     def test_known_class_generalization(self):
-        knownclass1 = OptValue(BoxPtr())
+        knownclass1 = PtrOptValue(BoxPtr())
         knownclass1.make_constant_class(ConstPtr(self.myptr), 0)
         info1 = NotVirtualStateInfo(knownclass1)
         info1.position = 0
-        knownclass2 = OptValue(BoxPtr())
+        knownclass2 = PtrOptValue(BoxPtr())
         knownclass2.make_constant_class(ConstPtr(self.myptr), 0)
         info2 = NotVirtualStateInfo(knownclass2)
         info2.position = 0
         self.check_no_guards(info1, info2)
         self.check_no_guards(info2, info1)
 
-        knownclass3 = OptValue(BoxPtr())
+        knownclass3 = PtrOptValue(BoxPtr())
         knownclass3.make_constant_class(ConstPtr(self.myptr2), 0)
         info3 = NotVirtualStateInfo(knownclass3)
         info3.position = 0
@@ -209,33 +210,33 @@
 
     def test_generate_guards_nonvirtual_all_combinations(self):
         # set up infos
-        unknown_val = OptValue(self.nodebox)
-        unknownnull_val = OptValue(BoxPtr(self.nullptr))
+        unknown_val = PtrOptValue(self.nodebox)
+        unknownnull_val = PtrOptValue(BoxPtr(self.nullptr))
         unknown_info = NotVirtualStateInfo(unknown_val)
 
-        nonnull_val = OptValue(self.nodebox)
+        nonnull_val = PtrOptValue(self.nodebox)
         nonnull_val.make_nonnull(None)
         nonnull_info = NotVirtualStateInfo(nonnull_val)
 
-        knownclass_val = OptValue(self.nodebox)
+        knownclass_val = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         knownclass_val.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(knownclass_val)
-        knownclass2_val = OptValue(self.nodebox2)
+        knownclass2_val = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
         knownclass2_val.make_constant_class(classbox, -1)
         knownclass2_info = NotVirtualStateInfo(knownclass2_val)
 
-        constant_val = OptValue(BoxInt())
+        constant_val = IntOptValue(BoxInt())
         constant_val.make_constant(ConstInt(1))
         constant_info = NotVirtualStateInfo(constant_val)
-        constclass_val = OptValue(self.nodebox)
+        constclass_val = PtrOptValue(self.nodebox)
         constclass_val.make_constant(self.nodebox.constbox())
         constclass_info = NotVirtualStateInfo(constclass_val)
-        constclass2_val = OptValue(self.nodebox)
+        constclass2_val = PtrOptValue(self.nodebox)
         constclass2_val.make_constant(self.nodebox2.constbox())
         constclass2_info = NotVirtualStateInfo(constclass2_val)
-        constantnull_val = OptValue(ConstPtr(self.nullptr))
+        constantnull_val = PtrOptValue(ConstPtr(self.nullptr))
         constantnull_info = NotVirtualStateInfo(constantnull_val)
 
         # unknown unknown
@@ -354,11 +355,11 @@
 
 
     def test_intbounds(self):
-        value1 = OptValue(BoxInt(15))
+        value1 = IntOptValue(BoxInt(15))
         value1.intbound.make_ge(IntBound(0, 10))
         value1.intbound.make_le(IntBound(20, 30))
         info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(OptValue(BoxInt()))
+        info2 = NotVirtualStateInfo(IntOptValue(BoxInt()))
         expected = """
         [i0]
         i1 = int_ge(i0, 0)
@@ -370,22 +371,22 @@
         self.check_invalid(info1, info2, BoxInt(50))
 
     def test_intbounds_constant(self):
-        value1 = OptValue(BoxInt(15))
+        value1 = IntOptValue(BoxInt(15))
         value1.intbound.make_ge(IntBound(0, 10))
         value1.intbound.make_le(IntBound(20, 30))
         info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(OptValue(ConstInt(10000)))
+        info2 = NotVirtualStateInfo(IntOptValue(ConstInt(10000)))
         self.check_invalid(info1, info2)
         info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(OptValue(ConstInt(11)))
+        info2 = NotVirtualStateInfo(IntOptValue(ConstInt(11)))
         self.check_no_guards(info1, info2)
 
     def test_known_class(self):
-        value1 = OptValue(self.nodebox)
+        value1 = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value1.make_constant_class(classbox, -1)
         info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        info2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         expected = """
         [p0]
         guard_nonnull(p0) []        
@@ -395,7 +396,7 @@
         self.check_invalid(info1, info2, BoxPtr())
 
     def test_known_class_value(self):
-        value1 = OptValue(self.nodebox)
+        value1 = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value1.make_constant_class(classbox, -1)
         box = self.nodebox
@@ -408,7 +409,7 @@
         self.compare(guards, expected, [box])
 
     def test_known_value(self):
-        value1 = OptValue(self.nodebox)
+        value1 = PtrOptValue(self.nodebox)
         value1.make_constant(ConstInt(1))
         box = self.nodebox
         guards = value1.make_guards(box)
@@ -419,21 +420,21 @@
         self.compare(guards, expected, [box])
 
     def test_equal_inputargs(self):
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
         vstate1 = VirtualState([knownclass_info, knownclass_info])
         assert vstate1.generalization_of(vstate1)
 
-        unknown_info1 = NotVirtualStateInfo(OptValue(self.nodebox))
+        unknown_info1 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         vstate2 = VirtualState([unknown_info1, unknown_info1])
         assert vstate2.generalization_of(vstate2)
         assert not vstate1.generalization_of(vstate2)
         assert vstate2.generalization_of(vstate1)
 
-        unknown_info1 = NotVirtualStateInfo(OptValue(self.nodebox))
-        unknown_info2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        unknown_info1 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
+        unknown_info2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         vstate3 = VirtualState([unknown_info1, unknown_info2])
         assert vstate3.generalization_of(vstate2)
         assert vstate3.generalization_of(vstate1)
@@ -457,12 +458,12 @@
 
 
     def test_generate_guards_on_virtual_fields_matches_array(self):
-        innervalue1 = OptValue(self.nodebox)
+        innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
         innervalue1.make_constant_class(constclassbox, -1)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         innerinfo2.position = 1
 
         descr = object()
@@ -474,7 +475,7 @@
         info2.fieldstate = [innerinfo2]
 
         value1 = VArrayValue(descr, None, 1, self.nodebox)
-        value1._items[0] = OptValue(self.nodebox)
+        value1._items[0] = PtrOptValue(self.nodebox)
 
         expected = """
         [p0]
@@ -484,12 +485,12 @@
         self.guards(info1, info2, value1, expected, [self.nodebox])
 
     def test_generate_guards_on_virtual_fields_matches_instance(self):
-        innervalue1 = OptValue(self.nodebox)
+        innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
         innervalue1.make_constant_class(constclassbox, -1)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         innerinfo2.position = 1
 
         info1 = VirtualStateInfo(ConstInt(42), [1])
@@ -499,7 +500,7 @@
         info2.fieldstate = [innerinfo2]
 
         value1 = VirtualValue(self.cpu, constclassbox, self.nodebox)
-        value1._fields = {1: OptValue(self.nodebox)}
+        value1._fields = {1: PtrOptValue(self.nodebox)}
 
         expected = """
         [p0]
@@ -509,12 +510,12 @@
         self.guards(info1, info2, value1, expected, [self.nodebox])
 
     def test_generate_guards_on_virtual_fields_matches_struct(self):
-        innervalue1 = OptValue(self.nodebox)
+        innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
         innervalue1.make_constant_class(constclassbox, -1)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         innerinfo2.position = 1
 
         structdescr = object()
@@ -526,7 +527,7 @@
         info2.fieldstate = [innerinfo2]
 
         value1 = VStructValue(self.cpu, structdescr, self.nodebox)
-        value1._fields = {1: OptValue(self.nodebox)}
+        value1._fields = {1: PtrOptValue(self.nodebox)}
 
         expected = """
         [p0]
@@ -536,12 +537,12 @@
         self.guards(info1, info2, value1, expected, [self.nodebox])
 
     def test_generate_guards_on_virtual_fields_matches_arraystruct(self):
-        innervalue1 = OptValue(self.nodebox)
+        innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
         innervalue1.make_constant_class(constclassbox, -1)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(OptValue(self.nodebox))
+        innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         innerinfo2.position = 1
 
         arraydescr = object()
@@ -554,7 +555,7 @@
         info2.fieldstate = [innerinfo2]
 
         value1 = VArrayStructValue(arraydescr, 1, self.nodebox)
-        value1._items[0][fielddescr] = OptValue(self.nodebox)
+        value1._items[0][fielddescr] = PtrOptValue(self.nodebox)
 
         expected = """
         [p0]
@@ -568,7 +569,7 @@
 
     def test_virtuals_with_equal_fields(self):
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -596,7 +597,7 @@
 
     def test_virtuals_with_nonmatching_fields(self):
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -605,7 +606,7 @@
         assert vstate1.generalization_of(vstate1)
 
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox2)
+        value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -618,7 +619,7 @@
 
     def test_virtuals_with_nonmatching_descrs(self):
         info1 = VirtualStateInfo(ConstInt(42), [10, 20])
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -627,7 +628,7 @@
         assert vstate1.generalization_of(vstate1)
 
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox2)
+        value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -640,7 +641,7 @@
         
     def test_virtuals_with_nonmatching_classes(self):
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
-        value = OptValue(self.nodebox)
+        value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
         value.make_constant_class(classbox, -1)
         knownclass_info = NotVirtualStateInfo(value)
@@ -649,7 +650,7 @@
         assert vstate1.generalization_of(vstate1)
 
         info2 = VirtualStateInfo(ConstInt(7), [1, 2])
-        value = OptValue(self.nodebox2)
+        value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
         value.make_constant_class(classbox, -1)

From noreply at buildbot.pypy.org  Mon Dec 22 17:23:36 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 17:23:36 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: Fix test
Message-ID: <20141222162336.BF8BB1C0399@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r75063:f2ded50e70cd
Date: 2014-12-22 17:23 +0100
http://bitbucket.org/pypy/pypy/changeset/f2ded50e70cd/

Log:	Fix test

diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -1015,8 +1015,9 @@
                                           c=3, e=5).items()), pairs)                # mixed input
 
         # make sure no positional args conflict with possible kwdargs
-        self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args,
-                         ['self'])
+        if '__init__' in OrderedDict.__dict__:   # absent in PyPy
+            self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args,
+                             ['self'])
 
         # Make sure that direct calls to __init__ do not clear previous contents
         d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)])

From noreply at buildbot.pypy.org  Mon Dec 22 17:32:56 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 17:32:56 +0100 (CET)
Subject: [pypy-commit] pypy all_ordered_dicts: fix test
Message-ID: <20141222163256.B0F411C243B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: all_ordered_dicts
Changeset: r75064:85069f5f1bb8
Date: 2014-12-22 17:32 +0100
http://bitbucket.org/pypy/pypy/changeset/85069f5f1bb8/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -151,15 +151,13 @@
         assert loop.match_by_id('loadattr1',
         '''
         guard_not_invalidated(descr=...)
-        i19 = call(ConstClass(ll_dict_lookup), _, _, _, descr=...)
+        i19 = call(ConstClass(ll_call_lookup_function), _, _, _, descr=...)
         guard_no_exception(descr=...)
-        i21 = int_and(i19, _)
-        i22 = int_is_true(i21)
+        i22 = int_lt(i19, 0)
         guard_true(i22, descr=...)
-        i26 = call(ConstClass(ll_dict_lookup), _, _, _, descr=...)
+        i26 = call(ConstClass(ll_call_lookup_function), _, _, _, descr=...)
         guard_no_exception(descr=...)
-        i28 = int_and(i26, _)
-        i29 = int_is_true(i28)
+        i29 = int_lt(i26, 0)
         guard_true(i29, descr=...)
         ''')
         assert loop.match_by_id('loadattr2', "")   # completely folded away

From noreply at buildbot.pypy.org  Mon Dec 22 19:49:40 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 22 Dec 2014 19:49:40 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: cleanup
Message-ID: <20141222184940.1BAFA1C046A@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75065:beca0e690eb8
Date: 2014-12-22 13:49 -0500
http://bitbucket.org/pypy/pypy/changeset/beca0e690eb8/

Log:	cleanup

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -10,16 +10,16 @@
 from pypy.module._socket import interp_socket
 
 
-## user defined constants
+# user defined constants
 X509_NAME_MAXLEN = 256
-## # these mirror ssl.h
+# these mirror ssl.h
 PY_SSL_ERROR_NONE, PY_SSL_ERROR_SSL = 0, 1
 PY_SSL_ERROR_WANT_READ, PY_SSL_ERROR_WANT_WRITE = 2, 3
 PY_SSL_ERROR_WANT_X509_LOOKUP = 4
-PY_SSL_ERROR_SYSCALL = 5 # look at error stack/return value/errno
+PY_SSL_ERROR_SYSCALL = 5  # look at error stack/return value/errno
 PY_SSL_ERROR_ZERO_RETURN, PY_SSL_ERROR_WANT_CONNECT = 6, 7
 # start of non ssl.h errorcodes
-PY_SSL_ERROR_EOF = 8 # special case of SSL_ERROR_SYSCALL
+PY_SSL_ERROR_EOF = 8  # special case of SSL_ERROR_SYSCALL
 PY_SSL_ERROR_INVALID_ERROR_CODE = 9
 
 PY_SSL_CERT_NONE, PY_SSL_CERT_OPTIONAL, PY_SSL_CERT_REQUIRED = 0, 1, 2
@@ -84,6 +84,7 @@
 constants["_OPENSSL_API_VERSION"] = version_info
 constants["OPENSSL_VERSION"] = SSLEAY_VERSION
 
+
 def ssl_error(space, msg, errno=0, exc='w_sslerror'):
     w_exception_class = get_exception_class(space, exc)
     if not errno:
@@ -143,8 +144,8 @@
         self.handshake_done = False
 
         sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
-        self.ssl = libssl_SSL_new(sslctx.ctx) # new ssl struct
-        libssl_SSL_set_fd(self.ssl, sock_fd) # set the socket for SSL
+        self.ssl = libssl_SSL_new(sslctx.ctx)  # new ssl struct
+        libssl_SSL_set_fd(self.ssl, sock_fd)  # set the socket for SSL
         # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address
         # of a str object may be changed by the garbage collector.
         libssl_SSL_set_mode(
@@ -268,7 +269,7 @@
                 elif err == SSL_ERROR_WANT_WRITE:
                     sockstate = checkwait(space, self.w_socket, True)
                 elif (err == SSL_ERROR_ZERO_RETURN and
-                   libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN):
+                        libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN):
                     return space.wrap("")
                 else:
                     sockstate = SOCKET_OPERATION_OK
@@ -450,6 +451,19 @@
             else:
                 return _decode_certificate(space, self.peer_cert)
 
+_SSLSocket.typedef = TypeDef(
+    "_ssl._SSLSocket",
+
+    do_handshake=interp2app(_SSLSocket.do_handshake),
+    write=interp2app(_SSLSocket.write),
+    read=interp2app(_SSLSocket.read),
+    pending=interp2app(_SSLSocket.pending),
+    peer_certificate=interp2app(_SSLSocket.peer_certificate),
+    cipher=interp2app(_SSLSocket.cipher),
+    shutdown=interp2app(_SSLSocket.shutdown),
+)
+
+
 def _decode_certificate(space, certificate, verbose=False):
     w_retval = space.newdict()
 
@@ -510,6 +524,7 @@
 
     return w_retval
 
+
 def _create_tuple_for_X509_NAME(space, xname):
     entry_count = libssl_X509_NAME_entry_count(xname)
     dn_w = []
@@ -538,6 +553,7 @@
         dn_w.append(space.newtuple(list(rdn_w)))
     return space.newtuple(list(dn_w))
 
+
 def _get_peer_alt_names(space, certificate):
     # this code follows the procedure outlined in
     # OpenSSL's crypto/x509v3/v3_prn.c:X509v3_EXT_print()
@@ -573,11 +589,11 @@
                 null = lltype.nullptr(rffi.VOIDP.TO)
                 if method[0].c_it:
                     names = rffi.cast(GENERAL_NAMES, libssl_ASN1_item_d2i(
-                            null, p_ptr, length,
-                            libssl_ASN1_ITEM_ptr(method[0].c_it)))
+                        null, p_ptr, length,
+                        libssl_ASN1_ITEM_ptr(method[0].c_it)))
                 else:
                     names = rffi.cast(GENERAL_NAMES, method[0].c_d2i(
-                            null, p_ptr, length))
+                        null, p_ptr, length))
 
             for j in range(libssl_sk_GENERAL_NAME_num(names)):
                 # Get a rendering of each name in the set of names
@@ -589,9 +605,9 @@
                     # attributes
                     dirname = libssl_pypy_GENERAL_NAME_dirn(name)
                     w_t = space.newtuple([
-                            space.wrap("DirName"),
-                            _create_tuple_for_X509_NAME(space, dirname)
-                            ])
+                        space.wrap("DirName"),
+                        _create_tuple_for_X509_NAME(space, dirname)
+                        ])
                 elif gntype in (GEN_EMAIL, GEN_DNS, GEN_URI):
                     # GENERAL_NAME_print() doesn't handle NULL bytes in
                     # ASN1_string correctly, CVE-2013-4238
@@ -607,8 +623,8 @@
                     as_ = rffi.cast(ASN1_STRING, as_)
                     buf = libssl_ASN1_STRING_data(as_)
                     length = libssl_ASN1_STRING_length(as_)
-                    w_t = space.newtuple([v,
-                        space.wrap(rffi.charpsize2str(buf, length))])
+                    w_t = space.newtuple([
+                        v, space.wrap(rffi.charpsize2str(buf, length))])
                 else:
                     # for everything else, we use the OpenSSL print form
                     if gntype not in (GEN_OTHERNAME, GEN_X400, GEN_EDIPARTY,
@@ -636,6 +652,7 @@
     else:
         return space.w_None
 
+
 def _create_tuple_for_attribute(space, name, value):
     with lltype.scoped_alloc(rffi.CCHARP.TO, X509_NAME_MAXLEN) as buf:
         length = libssl_OBJ_obj2txt(buf, X509_NAME_MAXLEN, name, 0)
@@ -652,16 +669,6 @@
 
     return space.newtuple([w_name, w_value])
 
-_SSLSocket.typedef = TypeDef("_SSLSocket",
-    do_handshake = interp2app(_SSLSocket.do_handshake),
-    write = interp2app(_SSLSocket.write),
-    read = interp2app(_SSLSocket.read),
-    pending = interp2app(_SSLSocket.pending),
-    peer_certificate = interp2app(_SSLSocket.peer_certificate),
-    cipher = interp2app(_SSLSocket.cipher),
-    shutdown = interp2app(_SSLSocket.shutdown),
-)
-
 
 def checkwait(space, w_sock, writing):
     """If the socket has a timeout, do a select()/poll() on the socket.
@@ -681,7 +688,6 @@
     if sock_fd < 0:
         return SOCKET_HAS_BEEN_CLOSED
 
-
     # see if the socket is ready
 
     # Prefer poll, if available, since you can poll() any fd
@@ -718,6 +724,7 @@
     else:
         return SOCKET_HAS_TIMED_OUT
 
+
 def _ssl_seterror(space, ss, ret):
     assert ret <= 0
 
@@ -793,6 +800,7 @@
         self.w_ssleoferror = space.new_exception_class(
             "_ssl.SSLEOFError", self.w_sslerror)
 
+
 def get_exception_class(space, name):
     return getattr(space.fromcache(Cache), name)
 
@@ -857,6 +865,10 @@
             libssl_ERR_clear_error()
             raise ssl_error(space, "No cipher can be selected.")
 
+    def descr_set_default_verify_paths(self, space):
+        if not libssl_SSL_CTX_set_default_verify_paths(self.ctx):
+            raise ssl_error(space, "")
+
     def descr_get_options(self, space):
         return space.newlong(libssl_SSL_CTX_get_options(self.ctx))
 
@@ -912,23 +924,19 @@
                         "CERT_OPTIONAL or CERT_REQUIRED")
         self.check_hostname = check_hostname
 
-    def descr_set_default_verify_paths(self, space):
-        if not libssl_SSL_CTX_set_default_verify_paths(self.ctx):
-            raise ssl_error(space, "")
+_SSLContext.typedef = TypeDef(
+    "_ssl._SSLContext",
+    __new__=interp2app(_SSLContext.descr_new),
+    _wrap_socket=interp2app(_SSLContext.descr_wrap_socket),
+    set_ciphers=interp2app(_SSLContext.descr_set_ciphers),
+    set_default_verify_paths=interp2app(_SSLContext.descr_set_default_verify_paths),
 
-
-_SSLContext.typedef = TypeDef("_SSLContext",
-    __module__ = "_ssl",
-    __new__ = interp2app(_SSLContext.descr_new),
-    _wrap_socket = interp2app(_SSLContext.descr_wrap_socket),
-    set_ciphers = interp2app(_SSLContext.descr_set_ciphers),
-    options = GetSetProperty(_SSLContext.descr_get_options,
-                             _SSLContext.descr_set_options),
-    verify_mode = GetSetProperty(_SSLContext.descr_get_verify_mode,
-                                 _SSLContext.descr_set_verify_mode),
-    check_hostname = GetSetProperty(_SSLContext.descr_get_check_hostname,
-                                    _SSLContext.descr_set_check_hostname),
-    set_default_verify_paths=interp2app(_SSLContext.descr_set_default_verify_paths),
+    options=GetSetProperty(_SSLContext.descr_get_options,
+                           _SSLContext.descr_set_options),
+    verify_mode=GetSetProperty(_SSLContext.descr_get_verify_mode,
+                               _SSLContext.descr_set_verify_mode),
+    check_hostname=GetSetProperty(_SSLContext.descr_get_check_hostname,
+                                  _SSLContext.descr_set_check_hostname),
 )
 
 
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -8,6 +8,8 @@
 
     def test_init_module(self):
         import _ssl
+        assert _ssl._SSLSocket.__module__ == '_ssl'
+        assert _ssl._SSLContext.__module__ == '_ssl'
 
     def test_sslerror(self):
         import _ssl, _socket

From noreply at buildbot.pypy.org  Mon Dec 22 20:55:44 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Mon, 22 Dec 2014 20:55:44 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Bind
	gcc_jit_object_get_debug_string
Message-ID: <20141222195544.855011C243B@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75066:8b2b47726e8a
Date: 2014-12-22 14:35 -0500
http://bitbucket.org/pypy/pypy/changeset/8b2b47726e8a/

Log:	Bind gcc_jit_object_get_debug_string

diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -64,6 +64,8 @@
                                                     compilation_info=eci))
         self.GCC_JIT_RESULT_P = lltype.Ptr(COpaque(name='gcc_jit_result',
                                                    compilation_info=eci))
+        self.GCC_JIT_OBJECT_P = lltype.Ptr(COpaque(name='gcc_jit_object',
+                                                   compilation_info=eci))
         self.GCC_JIT_TYPE_P = lltype.Ptr(COpaque(name='gcc_jit_type',
                                                  compilation_info=eci))
         self.GCC_JIT_FIELD_P = lltype.Ptr(COpaque(name='gcc_jit_field',
@@ -129,6 +131,9 @@
                 (lltype.Void,
                  'gcc_jit_result_release', [self.GCC_JIT_RESULT_P]),
 
+                (CCHARP,
+                 'gcc_jit_object_get_debug_string', [self.GCC_JIT_OBJECT_P]),
+
                 ############################################################
                 # Types
                 ############################################################
@@ -206,6 +211,10 @@
                                                   INT,
                                                   self.PARAM_P_P,
                                                   INT]),
+
+                (self.GCC_JIT_OBJECT_P,
+                 'gcc_jit_function_as_object', [self.GCC_JIT_FUNCTION_P]),
+
                 (self.GCC_JIT_LVALUE_P,
                  'gcc_jit_function_new_local', [self.GCC_JIT_FUNCTION_P,
                                                 self.GCC_JIT_LOCATION_P,

From noreply at buildbot.pypy.org  Mon Dec 22 20:55:45 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Mon, 22 Dec 2014 20:55:45 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Raise an exception when an
 error occurs on a gcc_jit_context
Message-ID: <20141222195545.CBD9C1C243B@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75067:6b48e7ef126a
Date: 2014-12-22 15:03 -0500
http://bitbucket.org/pypy/pypy/changeset/6b48e7ef126a/

Log:	Raise an exception when an error occurs on a gcc_jit_context

	This makes it much easier to track down issues.

	Requires not-yet-in-trunk API entrypoint:
	gcc_jit_context_get_last_error

diff --git a/rpython/jit/backend/libgccjit/notes.rst b/rpython/jit/backend/libgccjit/notes.rst
--- a/rpython/jit/backend/libgccjit/notes.rst
+++ b/rpython/jit/backend/libgccjit/notes.rst
@@ -9,6 +9,8 @@
 
      * :c:func:`gcc_jit_context_new_rvalue_from_long`
 
+     * :c:func:`gcc_jit_context_get_last_error`
+
    * a new value :c:macro:`GCC_JIT_UNARY_OP_ABS` within
      :c:type:`enum gcc_jit_unary_op`.
 
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -120,6 +120,9 @@
                                                   CCHARP,
                                                   INT]),
 
+                (CCHARP,
+                 'gcc_jit_context_get_last_error', [self.GCC_JIT_CONTEXT_P]),
+
                 (VOIDP,
                  'gcc_jit_result_get_code', [self.GCC_JIT_RESULT_P,
                                              CCHARP]),
@@ -485,11 +488,13 @@
 
     def get_type(self, r_enum):
         return Type(self.lib,
+                    self,
                     self.lib.gcc_jit_context_get_type(self.inner_ctxt,
                                                       r_enum))
 
     def get_int_type(self, num_bytes, is_signed):
         return Type(self.lib,
+                    self,
                     self.lib.gcc_jit_context_get_int_type(self.inner_ctxt,
                                                           num_bytes,
                                                           is_signed))
@@ -501,7 +506,7 @@
                                                    type_.inner_type,
                                                    name_charp)
         free_charp(name_charp)
-        return Field(self.lib, field)
+        return Field(self.lib, self, field)
 
     def new_struct_type(self, name, fields):
         name_charp = str2charp(name)
@@ -518,7 +523,7 @@
                                                      field_array))
         lltype.free(field_array, flavor='raw')
         free_charp(name_charp)
-        return Struct(self.lib, inner_struct)
+        return Struct(self.lib, self, inner_struct)
     
     def new_opaque_struct(self, name):
         name_charp = str2charp(name)
@@ -527,7 +532,7 @@
                                                        self.lib.null_location_ptr,
                                                        name_charp))
         free_charp(name_charp)
-        return Struct(self.lib, inner_struct)
+        return Struct(self.lib, self, inner_struct)
 
     def new_union_type(self, name, fields):
         name_charp = str2charp(name)
@@ -544,7 +549,7 @@
                                                     field_array))
         lltype.free(field_array, flavor='raw')
         free_charp(name_charp)
-        return Type(self.lib, inner_type)
+        return Type(self.lib, self, inner_type)
 
     def new_function_ptr_type(self, returntype, param_types, is_variadic):
         raw_type_array = lltype.malloc(self.lib.TYPE_P_P.TO,
@@ -561,34 +566,39 @@
                                                                is_variadic)
         lltype.free(raw_type_array, flavor='raw')
 
-        return Type(self.lib, type_)
+        return Type(self.lib, self, type_)
 
     def new_rvalue_from_int(self, type_, llvalue):
         return RValue(self.lib,
+                      self,
                       self.lib.gcc_jit_context_new_rvalue_from_int(self.inner_ctxt,
                                                                    type_.inner_type,
                                                                    llvalue))
 
     def new_rvalue_from_long(self, type_, llvalue):
         return RValue(self.lib,
+                      self,
                       self.lib.gcc_jit_context_new_rvalue_from_long(self.inner_ctxt,
                                                                     type_.inner_type,
                                                                     llvalue))
 
     def new_rvalue_from_double(self, type_, llvalue):
         return RValue(self.lib,
+                      self,
                       self.lib.gcc_jit_context_new_rvalue_from_double(self.inner_ctxt,
                                                                       type_.inner_type,
                                                                       llvalue))
 
     def new_rvalue_from_ptr(self, type_, llvalue):
         return RValue(self.lib,
+                      self,
                       self.lib.gcc_jit_context_new_rvalue_from_ptr(self.inner_ctxt,
                                                                    type_.inner_type,
                                                                    llvalue))
 
     def new_unary_op(self, op, type_, rvalue):
         return RValue(self.lib,
+                      self,
                       self.lib.gcc_jit_context_new_unary_op(self.inner_ctxt,
                                                             self.lib.null_location_ptr,
                                                             op,
@@ -597,6 +607,7 @@
 
     def new_binary_op(self, op, type_, a, b):
         return RValue(self.lib,
+                      self,
                       self.lib.gcc_jit_context_new_binary_op(self.inner_ctxt,
                                                              self.lib.null_location_ptr,
                                                              op,
@@ -605,6 +616,7 @@
 
     def new_comparison(self, op, a, b):
         return RValue(self.lib,
+                      self,
                       self.lib.gcc_jit_context_new_comparison(self.inner_ctxt,
                                                               self.lib.null_location_ptr,
                                                               op,
@@ -622,7 +634,7 @@
                                                    r_int(len(args)),
                                                    raw_arg_array)
         lltype.free(raw_arg_array, flavor='raw')
-        return RValue(self.lib, rvalue)
+        return RValue(self.lib, self, rvalue)
 
     def new_call_through_ptr(self, fn_ptr, args):
         raw_arg_array = lltype.malloc(self.lib.RVALUE_P_P.TO,
@@ -636,7 +648,7 @@
                                                                r_int(len(args)),
                                                                raw_arg_array)
         lltype.free(raw_arg_array, flavor='raw')
-        return RValue(self.lib, rvalue)
+        return RValue(self.lib, self, rvalue)
 
     def new_param(self, type_, name):
         name_charp = str2charp(name)
@@ -645,7 +657,7 @@
                                                    type_.inner_type,
                                                    name_charp)
         free_charp(name_charp)
-        return Param(self.lib, param)
+        return Param(self.lib, self, param)
 
     def new_function(self, kind, returntype, name, params, is_variadic):
         name_charp = str2charp(name)
@@ -666,7 +678,7 @@
         lltype.free(raw_param_array, flavor='raw')
         free_charp(name_charp)
 
-        return Function(self.lib, fn)
+        return Function(self.lib, self, fn)
 
     def new_global(self, kind, type_, name):
         name_charp = str2charp(name)
@@ -676,36 +688,54 @@
                                                      type_.inner_type,
                                                      name_charp)
         free_charp(name_charp)
-        return LValue(self.lib, lvalue)
+        return LValue(self.lib, self, lvalue)
 
     def new_cast(self, rvalue, type_):
         return RValue(self.lib,
+                      self, 
                       self.lib.gcc_jit_context_new_cast(self.inner_ctxt,
                                                         self.lib.null_location_ptr,
                                                         rvalue.inner_rvalue,
                                                         type_.inner_type))
 
-class Type(Wrapper):
-    def __init__(self, lib, inner_type):
+class LibgccjitError(Exception):
+    def __init__(self, ctxt):
+        self.msg = charp2str(ctxt.lib.gcc_jit_context_get_last_error (ctxt.inner_ctxt))
+        #print('self.msg: %r' % self.msg)
+
+    def __str__(self):
+        return self.msg
+
+class Object(Wrapper):
+    def __init__(self, lib, ctxt, inner_obj):
+        if not inner_obj:
+            raise LibgccjitError(ctxt)
         Wrapper.__init__(self, lib)
+        self.inner_obj = inner_obj
+
+class Type(Object):
+    def __init__(self, lib, ctxt, inner_type):
+        Object.__init__(self, lib, ctxt, inner_type)
         self.inner_type = inner_type
 
     def get_pointer(self):
         return Type(self.lib,
+                    self, 
                     self.lib.gcc_jit_type_get_pointer(self.inner_type))
 
-class Field(Wrapper):
-    def __init__(self, lib, inner_field):
-        Wrapper.__init__(self, lib)
+class Field(Object):
+    def __init__(self, lib, ctxt, inner_field):
+        Object.__init__(self, lib, ctxt, inner_field)
         self.inner_field = inner_field
 
-class Struct(Wrapper):
-    def __init__(self, lib, inner_struct):
-        Wrapper.__init__(self, lib)
+class Struct(Object):
+    def __init__(self, lib, ctxt, inner_struct):
+        Object.__init__(self, lib, ctxt, inner_struct)
         self.inner_struct = inner_struct
 
     def as_type(self):
         return Type(self.lib,
+                    self, 
                     self.lib.gcc_jit_struct_as_type(self.inner_struct))
 
 
@@ -721,44 +751,48 @@
                                            field_array)
         lltype.free(field_array, flavor='raw')
 
-class RValue(Wrapper):
-    def __init__(self, lib, inner_rvalue):
-        Wrapper.__init__(self, lib)
+class RValue(Object):
+    def __init__(self, lib, ctxt, inner_rvalue):
+        Object.__init__(self, lib, ctxt, inner_rvalue)
         self.inner_rvalue = inner_rvalue
 
     def dereference_field(self, field):
         return LValue(self.lib,
+                      self,
                       self.lib.gcc_jit_rvalue_dereference_field(self.inner_rvalue,
                                                                 self.lib.null_location_ptr,
                                                                 field.inner_field))
 
-class LValue(Wrapper):
-    def __init__(self, lib, inner_lvalue):
-        Wrapper.__init__(self, lib)
+class LValue(Object):
+    def __init__(self, lib, ctxt, inner_lvalue):
+        Object.__init__(self, lib, ctxt, inner_lvalue)
         self.inner_lvalue = inner_lvalue
 
     def as_rvalue(self):
         return RValue(self.lib,
+                      self,
                       self.lib.gcc_jit_lvalue_as_rvalue(self.inner_lvalue))
 
     def access_field(self, field):
         return LValue(self.lib,
+                      self,
                       self.lib.gcc_jit_lvalue_access_field (self.inner_lvalue,
                                                             self.lib.null_location_ptr,
                                                             field.inner_field))
 
-class Param(Wrapper):
-    def __init__(self, lib, inner_param):
-        Wrapper.__init__(self, lib)
+class Param(Object):
+    def __init__(self, lib, ctxt, inner_param):
+        Object.__init__(self, lib, ctxt, inner_param)
         self.inner_param = inner_param
 
     def as_rvalue(self):
         return RValue(self.lib,
+                      self,
                       self.lib.gcc_jit_param_as_rvalue(self.inner_param))
 
-class Function(Wrapper):
-    def __init__(self, lib, inner_function):
-        Wrapper.__init__(self, lib)
+class Function(Object):
+    def __init__(self, lib, ctxt, inner_function):
+        Object.__init__(self, lib, ctxt, inner_function)
         self.inner_function = inner_function
 
     def new_local(self, type_, name):
@@ -768,7 +802,7 @@
                                                     type_.inner_type,
                                                     name_charp)
         free_charp(name_charp)
-        return LValue(self.lib, local)
+        return LValue(self.lib, self, local)
 
     def new_block(self, name=None):
         if name is not None:
@@ -779,11 +813,11 @@
                                                     name_charp)
         if name_charp:
             free_charp(name_charp)
-        return Block(self.lib, block)
+        return Block(self.lib, self, block)
 
-class Block(Wrapper):
-    def __init__(self, lib, inner_block):
-        Wrapper.__init__(self, lib)
+class Block(Object):
+    def __init__(self, lib, ctxt, inner_block):
+        Object.__init__(self, lib, ctxt, inner_block)
         self.inner_block = inner_block
 
     def add_assignment(self, lvalue, rvalue):
@@ -816,7 +850,7 @@
                                                self.lib.null_location_ptr,
                                                rvalue.inner_rvalue)
 
-class Result(Wrapper):
+class Result(Object):
     def __init__(self, lib, inner_result):
         Wrapper.__init__(self, lib)
         self.inner_result = inner_result

From noreply at buildbot.pypy.org  Mon Dec 22 21:37:44 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 21:37:44 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix on Windows: now one of these
	functions called by
Message-ID: <20141222203744.5DE1A1D38A8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75068:246cc6f071ab
Date: 2014-12-22 21:37 +0100
http://bitbucket.org/pypy/pypy/changeset/246cc6f071ab/

Log:	Fix on Windows: now one of these functions called by
	after_external_call() modifies GetLastError too, so we need to save
	and restore it as well.

diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -7,7 +7,7 @@
 # all but one will be blocked.  The other threads get a chance to run
 # from time to time, using the periodic action GILReleaseAction.
 
-from rpython.rlib import rthread, rgil
+from rpython.rlib import rthread, rgil, rwin32
 from pypy.module.thread.error import wrap_thread_error
 from pypy.interpreter.executioncontext import PeriodicAsyncAction
 from pypy.module.thread.threadlocals import OSThreadLocals
@@ -76,9 +76,14 @@
 
 def after_external_call():
     e = get_errno()
+    e2 = 0
+    if rwin32.WIN32:
+        e2 = rwin32.GetLastError()
     rgil.gil_acquire()
     rthread.gc_thread_run()
     after_thread_switch()
+    if rwin32.WIN32:
+        rwin32.SetLastError(e2)
     set_errno(e)
 after_external_call._gctransformer_hint_cannot_collect_ = True
 after_external_call._dont_reach_me_in_del_ = True

From noreply at buildbot.pypy.org  Mon Dec 22 22:08:45 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 22 Dec 2014 22:08:45 +0100 (CET)
Subject: [pypy-commit] pypy default: Get/SetLastError should be nowrapper
 functions, like get/set_errno in
Message-ID: <20141222210845.3A7191C0399@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75069:40df46ca45b0
Date: 2014-12-22 22:08 +0100
http://bitbucket.org/pypy/pypy/changeset/40df46ca45b0/

Log:	Get/SetLastError should be nowrapper functions, like get/set_errno
	in rposix.py

diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -118,8 +118,10 @@
     INVALID_HANDLE_VALUE = rffi.cast(HANDLE, -1)
     PFILETIME = rffi.CArrayPtr(FILETIME)
 
-    _GetLastError = winexternal('GetLastError', [], DWORD, releasegil=False)
-    _SetLastError = winexternal('SetLastError', [DWORD], lltype.Void)
+    _GetLastError = winexternal('GetLastError', [], DWORD,
+                                _nowrapper=True, sandboxsafe=True)
+    _SetLastError = winexternal('SetLastError', [DWORD], lltype.Void,
+                                _nowrapper=True, sandboxsafe=True)
 
     def GetLastError():
         return rffi.cast(lltype.Signed, _GetLastError())

From noreply at buildbot.pypy.org  Mon Dec 22 22:28:43 2014
From: noreply at buildbot.pypy.org (bdkearns)
Date: Mon, 22 Dec 2014 22:28:43 +0100 (CET)
Subject: [pypy-commit] pypy py3k: fix _curses.setupterm when termstr is text
Message-ID: <20141222212843.73F001C0399@cobra.cs.uni-duesseldorf.de>

Author: Brian Kearns <bdkearns at gmail.com>
Branch: py3k
Changeset: r75070:d8475a6257bc
Date: 2014-12-22 16:28 -0500
http://bitbucket.org/pypy/pypy/changeset/d8475a6257bc/

Log:	fix _curses.setupterm when termstr is text

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -1127,6 +1127,8 @@
 
     if term is None:
         term = ffi.NULL
+    elif isinstance(term, str):
+        term = term.encode('utf-8')
     err = ffi.new("int *")
     if lib.setupterm(term, fd, err) == lib.ERR:
         err = err[0]

From noreply at buildbot.pypy.org  Mon Dec 22 22:35:09 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Mon, 22 Dec 2014 22:35:09 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_compile_bridge to
	work
Message-ID: <20141222213509.5D67B1C0399@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75071:c77c1f9355fd
Date: 2014-12-22 16:42 -0500
http://bitbucket.org/pypy/pypy/changeset/c77c1f9355fd/

Log:	Get test_compile_bridge to work

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -23,6 +23,21 @@
         self.paramtypes = [assembler.t_jit_frame_ptr,
                            assembler.t_void_ptr]
 
+class Function:
+    """
+    Wrapper around a rffi_bindings.Function, for a loop or bridge
+    """
+    def __init__(self, name, rffi_fn):
+        self.name = name
+        self.rffi_fn = rffi_fn
+        self.patchpoints = []
+
+    def new_block(self, name): 
+       return self.rffi_fn.new_block(name)
+
+    def new_local(self, type_, name): 
+       return self.rffi_fn.new_local(type_, name)
+
 class Patchpoint:
     """
     We need to be able to patch out the generated code that runs when a
@@ -91,7 +106,12 @@
         # this is a:
         #   struct JITFRAME * (**guard_failure_fn) (struct JITFRAME *, void *)
         # i.e. one extra level of indirection.
-        fn_ptr_ptr = result.get_global(self.fn_ptr_name)
+        self.fn_ptr_ptr = result.get_global(self.fn_ptr_name)
+
+        self.set_handler(handler)
+
+    def set_handler(self, handler):
+        print('set_handler(%r)' % handler)
 
         # We want to write the equivalent of:
         #    (*fn_ptr_ptr) = handler;
@@ -102,7 +122,7 @@
 
         # We can't directly express the function ptr ptr in lltype,
         # so instead pretend we have a (const char **) and a (const char *):
-        fn_ptr_ptr = rffi.cast(rffi.CCHARPP, fn_ptr_ptr)
+        fn_ptr_ptr = rffi.cast(rffi.CCHARPP, self.fn_ptr_ptr)
         handler = rffi.cast(rffi.CCHARP, handler)
 
         # ...and write through the ptr:
@@ -133,13 +153,18 @@
 
         self.num_anon_loops = 0
         self.num_guard_failure_fns = 0
+        self.num_bridges = 0
 
         self.sizeof_signed = rffi.sizeof(lltype.Signed)
-        self.patchpoints = []
+        self.label_for_descr = {}
+        self.block_for_label_descr = {}
+        self.function_for_label_descr = {}
+        self.patchpoint_for_descr = {}
+
+        eci = make_eci()
+        self.lib = Library(eci)
 
     def make_context(self):
-        eci = make_eci()
-        self.lib = Library(eci)
         self.ctxt = Context.acquire(self.lib)#self.lib.gcc_jit_context_acquire()
         if 0:
             self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_TREE,
@@ -162,7 +187,22 @@
         if 0:
             self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
                                       r_int(1))
-        
+
+        self.t_Signed = self.ctxt.get_int_type(r_int(self.sizeof_signed),
+                                               r_int(1))
+        self.t_UINT = self.ctxt.get_int_type(r_int(self.sizeof_signed),
+                                             r_int(0))
+        self.t_float = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_DOUBLE) # FIXME                                          
+        self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
+        self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
+        self.t_void = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID)
+
+        self.u_signed = self.ctxt.new_field(self.t_Signed, "u_signed")
+        self.u_float = self.ctxt.new_field(self.t_float, "u_float")
+        self.t_any = self.ctxt.new_union_type ("any",
+                                               [self.u_signed,
+                                                self.u_float])
+
     def setup(self, looptoken):
         allblocks = self.get_asmmemmgr_blocks(looptoken)
         self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr,
@@ -191,22 +231,69 @@
         clt.frame_info.clear() # for now
 
         self.make_context()
-        self.t_Signed = self.ctxt.get_int_type(r_int(self.sizeof_signed),
-                                               r_int(1))
-        self.t_UINT = self.ctxt.get_int_type(r_int(self.sizeof_signed),
-                                             r_int(0))
-        self.t_float = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_DOUBLE) # FIXME                                          
-        self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
-        self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
-        self.t_void = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID)
-
-        self.u_signed = self.ctxt.new_field(self.t_Signed, "u_signed")
-        self.u_float = self.ctxt.new_field(self.t_float, "u_float")
-        self.t_any = self.ctxt.new_union_type ("any",
-                                               [self.u_signed,
-                                                self.u_float])
+        max_args, initial_locs = self.make_JITFRAME_struct(inputargs,
+                                                           operations)
         self.lvalue_for_box = {}
 
+        if not loopname:
+            loopname = 'anonloop_%i' % self.num_anon_loops
+            self.num_anon_loops += 1
+        print("  loopname: %r" % loopname)
+        self.make_function(loopname, inputargs, operations)
+
+        # Ensure that the frame is large enough
+        baseofs = self.cpu.get_baseofs_of_frame_field()
+        clt.frame_info.update_frame_depth(baseofs,
+                                          max_args)
+
+        self.ctxt.dump_to_file("/tmp/%s.c" % loopname, r_int(1))
+
+        #raise foo
+
+        jit_result = self.ctxt.compile()
+        self.ctxt.release()
+
+        # Patch all patchpoints to their initial handlers:
+        for pp in self.fn.patchpoints:
+            pp.write_initial_handler(jit_result)
+
+        fn_ptr = jit_result.get_code(loopname)
+
+        looptoken._ll_function_addr = fn_ptr
+
+        looptoken.compiled_loop_token._ll_initial_locs = initial_locs
+
+        # FIXME: this leaks the gcc_jit_result
+
+    def assemble_bridge(self, logger, faildescr, inputargs, operations,
+                        original_loop_token, log):
+        print('assemble_bridge(%r)' % locals())
+        if not we_are_translated():
+            # Arguments should be unique
+            assert len(set(inputargs)) == len(inputargs)
+
+        self.make_context()
+        max_args, initial_locs = self.make_JITFRAME_struct(inputargs,
+                                                           operations)
+        self.lvalue_for_box = {}
+
+        name = "bridge_%i" % self.num_bridges
+        self.num_bridges += 1
+
+        self.make_function(name, inputargs, operations)
+        self.ctxt.dump_to_file("/tmp/%s.c" % name, r_int(1))
+        jit_result = self.ctxt.compile()
+        self.ctxt.release()
+
+        # Patch all patchpoints to their initial handlers:
+        for pp in self.fn.patchpoints:
+            pp.write_initial_handler(jit_result)
+
+        # Patch the patchpoint for "faildescr" to point to our new code
+        fn_ptr = jit_result.get_code(name)
+        self.patchpoint_for_descr[faildescr].set_handler(fn_ptr)
+
+    def make_JITFRAME_struct(self, inputargs, operations):
         #print(jitframe.JITFRAME)
         #print(dir(jitframe.JITFRAME))
         #print('jitframe.JITFRAME._flds: %r' % jitframe.JITFRAME._flds)
@@ -274,26 +361,24 @@
 
         struct_jit_frame.set_fields (fields)
 
+        return max_args, initial_locs
+
+    def make_function(self, name, inputargs, operations):
         # Make function:
         #print('  inputargs: %r' % (inputargs, ))
         #jitframe.JITFRAMEINFOPTR
         self.loop_params = Params(self)
 
-        if not loopname:
-            loopname = 'anonloop_%i' % self.num_anon_loops
-            self.num_anon_loops += 1
-        print("  loopname: %r" % loopname)
-        self.fn = self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_EXPORTED,
-                                         self.t_jit_frame_ptr,
-                                         loopname,
-                                         self.loop_params.paramlist,
-                                         r_int(0))
+        self.fn = Function(name,
+                           self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_EXPORTED,
+                                                  self.t_jit_frame_ptr,
+                                                  name,
+                                                  self.loop_params.paramlist,
+                                                  r_int(0)))
 
         self.b_current = self.fn.new_block("initial")
-        self.label_for_descr = {}
-        self.block_for_label_descr = {}
 
-        # Add an initial comment summarizing the loop
+        # Add an initial comment summarizing the loop or bridge:
         text = '\n\tinputargs: %s\n\n' % inputargs
         for op in operations:
             #print(op)
@@ -302,7 +387,7 @@
             #print(repr(op.getopname()))
             text += '\t%s\n' % op
         self.b_current.add_comment(str(text))
-    
+
         # Get initial values from input args:
         for idx, arg in enumerate(inputargs):
             self.b_current.add_comment("inputargs[%i]: %s" % (idx, arg))
@@ -334,39 +419,6 @@
             methname = 'emit_%s' % op.getopname()
             getattr(self, methname) (op)
 
-        # Ensure that the frame is large enough
-        baseofs = self.cpu.get_baseofs_of_frame_field()
-        clt.frame_info.update_frame_depth(baseofs,
-                                          max_args)
-        
-        self.ctxt.dump_to_file("/tmp/%s.c" % loopname, r_int(1))
-
-        #raise foo
-
-        jit_result = self.ctxt.compile()
-        self.ctxt.release()
-
-        # Patch all patchpoints to their initial handlers:
-        for pp in self.patchpoints:
-            pp.write_initial_handler(jit_result)
-
-        fn_ptr = jit_result.get_code(loopname)
-
-        looptoken._ll_function_addr = fn_ptr
-
-        looptoken.compiled_loop_token._ll_initial_locs = initial_locs
-
-        # FIXME: this leaks the gcc_jit_result
-
-    def assemble_bridge(self, logger, faildescr, inputargs, operations,
-                        original_loop_token, log):
-        print('assemble_bridge(%r)' % locals())
-        if not we_are_translated():
-            # Arguments should be unique
-            assert len(set(inputargs)) == len(inputargs)
-
-        raise foo
-
     def expr_to_rvalue(self, expr):
         """
         print('expr_to_rvalue')
@@ -466,8 +518,27 @@
             self.b_current.add_assignment(
                 self.get_box_as_lvalue(label._args[i]),
                 tmps[i].as_rvalue())
-            
-        self.b_current.end_with_jump(self.block_for_label_descr[jumpop.getdescr()])
+
+        print('jumpop.getdescr(): %r' % jumpop.getdescr())
+        dest_fn = self.function_for_label_descr[jumpop.getdescr()]
+        print('dest_fn: %r' % dest_fn)
+        if dest_fn == self.fn:
+            b_dest = self.block_for_label_descr[jumpop.getdescr()]
+            self.b_current.end_with_jump(b_dest)
+        else:
+            # Implement as a tail-call:
+            # Sadly, we need to "import" the fn by name, since it was
+            # created on a different gcc_jit_context.
+            p = Params(self)
+            other_fn = self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_IMPORTED,
+                                              self.t_jit_frame_ptr,
+                                              dest_fn.name,
+                                              p.paramlist,
+                                              r_int(0))
+            args = [param.as_rvalue()
+                    for param in self.loop_params.paramlist]
+            call = self.ctxt.new_call(other_fn, args)
+            self.b_current.end_with_return(call)
 
     def emit_finish(self, resop):
         self._impl_write_output_args(self.loop_params, resop._args)
@@ -483,6 +554,7 @@
         b_new = self.fn.new_block(str(resop))
         self.block_for_label_descr[resop.getdescr()] = b_new
         self.label_for_descr[resop.getdescr()] = resop
+        self.function_for_label_descr[resop.getdescr()] = self.fn
         self.b_current.end_with_jump(b_new)
         self.b_current = b_new
 
@@ -511,7 +583,8 @@
         # Implement it as a tail-call through a function ptr to
         # a handler function, allowing patchability
         pp = Patchpoint(self)
-        self.patchpoints.append(pp)
+        self.fn.patchpoints.append(pp)
+        self.patchpoint_for_descr[resop.getdescr()] = pp
         args = [param.as_rvalue()
                 for param in self.loop_params.paramlist]
         call = self.ctxt.new_call_through_ptr (pp.failure_fn_ptr.as_rvalue(),

From noreply at buildbot.pypy.org  Mon Dec 22 22:49:20 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Mon, 22 Dec 2014 22:49:20 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Fix test_finish
Message-ID: <20141222214920.932611C0399@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75072:946fdf6ef9d7
Date: 2014-12-22 16:57 -0500
http://bitbucket.org/pypy/pypy/changeset/946fdf6ef9d7/

Log:	Fix test_finish

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -399,7 +399,7 @@
             #src_ptr_rvalue = self.ctxt.new_cast(src_ptr_rvalue, self.t_float.get_pointer())
             #src_rvalue = self.ctxt.new_dereference(src_ptr_rvalue)
             # or do it as a union
-            field = self.get_union_field_for_box(arg)
+            field = self.get_union_field_for_expr(arg)
             src_rvalue = self.get_arg_as_lvalue(idx).access_field(field).as_rvalue()
             # FIXME: this may need a cast:
             #src_rvalue = self.ctxt.new_cast(src_rvalue, self.t_float)
@@ -476,13 +476,13 @@
         else:
             raise ValueError('unhandled box: %s %s' % (box, type(box)))
 
-    def get_union_field_for_box(self, box):
-        if isinstance(box, BoxInt):
+    def get_union_field_for_expr(self, expr):
+        if isinstance(expr, (BoxInt, ConstInt)):
             return self.u_signed;
-        elif isinstance(box, BoxFloat):
+        elif isinstance(expr, (BoxFloat, ConstFloat)):
             return self.u_float;
         else:
-            raise ValueError('unhandled box: %s %s' % (box, type(box)))        
+            raise ValueError('unhandled expr: %s %s' % (expr, type(expr)))        
 
     # Handling of specific ResOperation subclasses: each one is
     # a method named "emit_foo", where "foo" is the str() of the resop.
@@ -614,8 +614,8 @@
         # Write outputs back:
         for idx, arg in enumerate(args):
             if arg is not None:
-                src_rvalue = self.get_box_as_lvalue(arg).as_rvalue()
-                field = self.get_union_field_for_box(arg)
+                src_rvalue = self.expr_to_rvalue(arg)
+                field = self.get_union_field_for_expr(arg)
                 dst_lvalue = self.get_arg_as_lvalue(idx).access_field(field)
                 self.b_current.add_assignment(dst_lvalue, src_rvalue)
             else:

From noreply at buildbot.pypy.org  Tue Dec 23 09:35:05 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 23 Dec 2014 09:35:05 +0100 (CET)
Subject: [pypy-commit] pypy gc-counters: Implement simple counting of
 different type ids surviving minor collections
Message-ID: <20141223083505.0DE191C046A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-counters
Changeset: r75073:9cdaba812857
Date: 2014-12-23 10:34 +0200
http://bitbucket.org/pypy/pypy/changeset/9cdaba812857/

Log:	Implement simple counting of different type ids surviving minor
	collections

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -175,6 +175,8 @@
 
 # ____________________________________________________________
 
+MAX_TID = 65536
+
 class IncrementalMiniMarkGC(MovingGCBase):
     _alloc_flavor_ = "raw"
     inline_simple_malloc = True
@@ -302,6 +304,8 @@
         self.debug_tiny_nursery = -1
         self.debug_rotating_nurseries = lltype.nullptr(NURSARRAY)
         self.extra_threshold = 0
+        self.tid_counters = lltype.malloc(lltype.Array(lltype.Signed), MAX_TID,
+                                          zero=True, immortal=True)
         #
         # The ArenaCollection() handles the nonmovable objects allocation.
         if ArenaCollectionClass is None:
@@ -1856,6 +1860,10 @@
         # A bit of no-ops to convince llarena that we are changing
         # the layout, in non-translated versions.
         typeid = self.get_type_id(obj)
+
+        # XXX
+        self.tid_counters[typeid] += 1
+        
         obj = llarena.getfakearenaaddress(obj)
         llarena.arena_reset(obj - size_gc_header, totalsize, 0)
         llarena.arena_reserve(obj - size_gc_header,
@@ -2590,3 +2598,12 @@
                 (obj + offset).address[0] = llmemory.NULL
         self.old_objects_with_weakrefs.delete()
         self.old_objects_with_weakrefs = new_with_weakref
+
+    def reset_tid_counters(self):
+        i = 0
+        while i < MAX_TID:
+            self.tid_counters[i] = 0
+            i += 1
+
+    def get_tid_counters(self):
+        return self.tid_counters
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -474,6 +474,14 @@
                                         [s_gc, SomeAddress()],
                                         annmodel.SomeBool())
 
+        if hasattr(GCClass, 'get_tid_counters'):
+            self.get_tid_ptr = getfn(GCClass.get_tid_counters,
+                                     [s_gc],
+                                     SomePtr(lltype.Ptr(
+                                         lltype.Array(lltype.Signed))))
+            self.reset_tid_ptr = getfn(GCClass.reset_tid_counters,
+                                       [s_gc], annmodel.s_None)
+
         self.write_barrier_ptr = None
         self.write_barrier_from_array_ptr = None
         if GCClass.needs_write_barrier:
@@ -1258,6 +1266,17 @@
         else:
             GCTransformer.gct_setfield(self, hop)
 
+    def gct_gc_reset_tid_counters(self, hop):
+        if hasattr(self, 'reset_tid_ptr'):
+            hop.genop("direct_call", [self.reset_tid_ptr,
+                                      self.c_const_gc])
+
+    def gct_gc_get_tid_counters(self, hop):
+        if hasattr(self, 'get_tid_ptr'):
+            hop.genop("direct_call", [self.get_tid_ptr,
+                                      self.c_const_gc],
+                resultvar=hop.spaceop.result)
+
     def var_needs_set_transform(self, var):
         return var_needsgc(var)
 
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -676,3 +676,32 @@
         lambda_func = hop.args_s[1].const
         hop.exception_cannot_occur()
         hop.rtyper.custom_trace_funcs.append((TP, lambda_func()))
+
+def get_tid_counters():
+    return []
+
+def reset_tid_counters():
+    pass
+
+class GetTidCountersEntry(ExtRegistryEntry):
+    _about_ = get_tid_counters
+
+    def compute_result_annotation(self):
+        from rpython.rtyper.llannotation import SomePtr
+        from rpython.rtyper.lltypesystem import lltype
+        
+        return SomePtr(lltype.Ptr(lltype.Array(lltype.Signed)))
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        return hop.genop('gc_get_tid_counters', [], resulttype=hop.r_result)
+
+class ResetTidCountersEntry(ExtRegistryEntry):
+    _about_ = reset_tid_counters
+
+    def compute_result_annotation(self):
+        pass
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        hop.genop('gc_reset_tid_counters', [])
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -495,6 +495,8 @@
     'gc_typeids_z'        : LLOp(),
     'gc_gcflag_extra'     : LLOp(),
     'gc_add_memory_pressure': LLOp(),
+    'gc_reset_tid_counters': LLOp(),
+    'gc_get_tid_counters'  : LLOp(),
 
     # ------- JIT & GC interaction, only for some GCs ----------
 
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1548,6 +1548,32 @@
         res = self.run("random_pin")
         assert res == 28495
 
+    def define_get_tid_ptr(self):
+        class A(object):
+            pass
+
+        prebuilt = A()
+        prebuilt.a = []
+
+        def f():
+            rgc.reset_tid_counters()
+            for i in range(1000):
+                prebuilt.a.append(A())
+                prebuilt.a[-1] = A()
+                prebuilt.a[-1] = A()
+                prebuilt.a[-1] = A()
+                prebuilt.a[-1] = A()
+            llop.gc__collect(lltype.Void)
+            a = rgc.get_tid_counters()
+            sum = 0
+            for i in range(65536):
+                sum += a[i]
+            return sum
+        return f
+
+    def test_get_tid_ptr(self):
+        res = self.run("get_tid_ptr")
+        assert 1050 > res > 1000 # counting objects surviving minor collections
 
 # ____________________________________________________________________
 

From noreply at buildbot.pypy.org  Tue Dec 23 09:40:05 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 23 Dec 2014 09:40:05 +0100 (CET)
Subject: [pypy-commit] pypy gc-counters: export the first crude approximation
Message-ID: <20141223084005.8A7EF1C01E8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-counters
Changeset: r75074:c2056dcff6bd
Date: 2014-12-23 10:39 +0200
http://bitbucket.org/pypy/pypy/changeset/c2056dcff6bd/

Log:	export the first crude approximation

diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -10,6 +10,7 @@
         'enable_finalizers': 'interp_gc.enable_finalizers',
         'disable_finalizers': 'interp_gc.disable_finalizers',
         'garbage': 'space.newlist([])',
+        'get_tid_counters': 'interp_gc.get_tid_counters',
         #'dump_heap_stats': 'interp_gc.dump_heap_stats',
     }
     appleveldefs = {}
diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py
--- a/pypy/module/gc/interp_gc.py
+++ b/pypy/module/gc/interp_gc.py
@@ -61,3 +61,13 @@
         f.write("%d %d " % (tb[i].count, tb[i].size))
         f.write(",".join([str(tb[i].links[j]) for j in range(len(tb))]) + "\n")
     f.close()
+
+def get_tid_counters(space):
+    a = rgc.get_tid_counters()
+    l_w = [None] * 65536
+    i = 0
+    while i < 65536:
+        l_w[i] = space.wrap(a[i])
+        i += 1
+    rgc.reset_tid_counters()
+    return space.newlist(l_w)

From noreply at buildbot.pypy.org  Tue Dec 23 12:23:01 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Dec 2014 12:23:01 +0100 (CET)
Subject: [pypy-commit] pypy default: Add gc.get_typeids_list() to the Python
 'gc' module. Returns a list
Message-ID: <20141223112301.3FBB31C34AB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75075:ea9a1e9c799d
Date: 2014-12-23 11:36 +0100
http://bitbucket.org/pypy/pypy/changeset/ea9a1e9c799d/

Log:	Add gc.get_typeids_list() to the Python 'gc' module. Returns a list
	of integers that are typeids. The Nth item in the list correspond
	to the 'memberN' in the string returned by gc.get_typeids_z().

diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -30,6 +30,7 @@
                 'get_referrers': 'referents.get_referrers',
                 '_dump_rpy_heap': 'referents._dump_rpy_heap',
                 'get_typeids_z': 'referents.get_typeids_z',
+                'get_typeids_list': 'referents.get_typeids_list',
                 'GcRef': 'referents.W_GcRef',
                 })
         MixedModule.__init__(self, space, w_name)
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -228,3 +228,8 @@
     a = rgc.get_typeids_z()
     s = ''.join([a[i] for i in range(len(a))])
     return space.wrap(s)
+
+def get_typeids_list(space):
+    l = rgc.get_typeids_list()
+    list_w = [space.wrap(l[i] for i in range(len(l)))]
+    return space.newlist(list_w)
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -1,7 +1,7 @@
 """
 Utility RPython functions to inspect objects in the GC.
 """
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup
 from rpython.rlib.objectmodel import free_non_gc_object
 from rpython.rtyper.module.ll_os import UNDERSCORE_ON_WIN32
 from rpython.rlib import rposix, rgc
@@ -238,3 +238,8 @@
 def get_typeids_z(gc):
     srcaddress = gc.root_walker.gcdata.typeids_z
     return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(rgc.ARRAY_OF_CHAR))
+
+def get_typeids_list(gc):
+    srcaddress = gc.root_walker.gcdata.typeids_list
+    return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(ARRAY_OF_HALFWORDS))
+ARRAY_OF_HALFWORDS = lltype.Array(llgroup.HALFWORD)
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -170,6 +170,7 @@
         gcdata.static_root_end = a_random_address        # patched in finish()
         gcdata.max_type_id = 13                          # patched in finish()
         gcdata.typeids_z = a_random_address              # patched in finish()
+        gcdata.typeids_list = a_random_address           # patched in finish()
         self.gcdata = gcdata
         self.malloc_fnptr_cache = {}
 
@@ -205,6 +206,7 @@
         data_classdef.generalize_attr('static_root_end', SomeAddress())
         data_classdef.generalize_attr('max_type_id', annmodel.SomeInteger())
         data_classdef.generalize_attr('typeids_z', SomeAddress())
+        data_classdef.generalize_attr('typeids_list', SomeAddress())
 
         annhelper = annlowlevel.MixLevelHelperAnnotator(self.translator.rtyper)
 
@@ -455,6 +457,11 @@
                                        [s_gc],
                                        SomePtr(lltype.Ptr(rgc.ARRAY_OF_CHAR)),
                                        minimal_transform=False)
+        self.get_typeids_list_ptr = getfn(inspector.get_typeids_list,
+                                       [s_gc],
+                                       SomePtr(lltype.Ptr(
+                                           lltype.Array(llgroup.HALFWORD))),
+                                       minimal_transform=False)
 
         self.set_max_heap_size_ptr = getfn(GCClass.set_max_heap_size.im_func,
                                            [s_gc,
@@ -596,7 +603,8 @@
         newgcdependencies = []
         newgcdependencies.append(ll_static_roots_inside)
         ll_instance.inst_max_type_id = len(group.members)
-        typeids_z = self.write_typeid_list()
+        #
+        typeids_z, typeids_list = self.write_typeid_list()
         ll_typeids_z = lltype.malloc(rgc.ARRAY_OF_CHAR,
                                      len(typeids_z),
                                      immortal=True)
@@ -604,6 +612,15 @@
             ll_typeids_z[i] = typeids_z[i]
         ll_instance.inst_typeids_z = llmemory.cast_ptr_to_adr(ll_typeids_z)
         newgcdependencies.append(ll_typeids_z)
+        #
+        ll_typeids_list = lltype.malloc(lltype.Array(llgroup.HALFWORD),
+                                        len(typeids_list),
+                                        immortal=True)
+        for i in range(len(typeids_list)):
+            ll_typeids_list[i] = typeids_list[i]
+        ll_instance.inst_typeids_list= llmemory.cast_ptr_to_adr(ll_typeids_list)
+        newgcdependencies.append(ll_typeids_list)
+        #
         return newgcdependencies
 
     def get_finish_tables(self):
@@ -624,6 +641,13 @@
         # XXX argh argh, this only gives the member index but not the
         #     real typeid, which is a complete mess to obtain now...
         all_ids = self.layoutbuilder.id_of_type.items()
+        list_data = []
+        ZERO = rffi.cast(llgroup.HALFWORD, 0)
+        for _, typeinfo in all_ids:
+            while len(list_data) <= typeinfo.index:
+                list_data.append(ZERO)
+            list_data[typeinfo.index] = typeinfo
+        #
         all_ids = [(typeinfo.index, TYPE) for (TYPE, typeinfo) in all_ids]
         all_ids = dict(all_ids)
         f = udir.join("typeids.txt").open("w")
@@ -632,9 +656,10 @@
         f.close()
         try:
             import zlib
-            return zlib.compress(udir.join("typeids.txt").read(), 9)
+            z_data = zlib.compress(udir.join("typeids.txt").read(), 9)
         except ImportError:
-            return ''
+            z_data = ''
+        return z_data, list_data
 
     def transform_graph(self, graph):
         func = getattr(graph, 'func', None)
@@ -1175,6 +1200,13 @@
                   resultvar=hop.spaceop.result)
         self.pop_roots(hop, livevars)
 
+    def gct_gc_typeids_list(self, hop):
+        livevars = self.push_roots(hop)
+        hop.genop("direct_call",
+                  [self.get_typeids_list_ptr, self.c_const_gc],
+                  resultvar=hop.spaceop.result)
+        self.pop_roots(hop, livevars)
+
     def _set_into_gc_array_part(self, op):
         if op.opname == 'setarrayitem':
             return op.args[1]
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -445,6 +445,10 @@
     "NOT_RPYTHON"
     raise NotImplementedError
 
+def get_typeids_list():
+    "NOT_RPYTHON"
+    raise NotImplementedError
+
 def has_gcflag_extra():
     "NOT_RPYTHON"
     return True
@@ -644,6 +648,18 @@
         return hop.genop('gc_typeids_z', [], resulttype = hop.r_result)
 
 class Entry(ExtRegistryEntry):
+    _about_ = get_typeids_list
+
+    def compute_result_annotation(self):
+        from rpython.rtyper.llannotation import SomePtr
+        from rpython.rtyper.lltypesystem import llgroup
+        return SomePtr(lltype.Ptr(lltype.Array(llgroup.HALFWORD)))
+
+    def specialize_call(self, hop):
+        hop.exception_is_here()
+        return hop.genop('gc_typeids_list', [], resulttype = hop.r_result)
+
+class Entry(ExtRegistryEntry):
     _about_ = (has_gcflag_extra, get_gcflag_extra, toggle_gcflag_extra)
     def compute_result_annotation(self, s_arg=None):
         from rpython.annotator.model import s_Bool
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -911,6 +911,9 @@
     def op_gc_typeids_z(self):
         raise NotImplementedError("gc_typeids_z")
 
+    def op_gc_typeids_list(self):
+        raise NotImplementedError("gc_typeids_list")
+
     def op_gc_gcflag_extra(self, subopnum, *args):
         return self.heap.gcflag_extra(subopnum, *args)
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -493,6 +493,7 @@
     'gc_is_rpy_instance'  : LLOp(),
     'gc_dump_rpy_heap'    : LLOp(),
     'gc_typeids_z'        : LLOp(),
+    'gc_typeids_list'     : LLOp(),
     'gc_gcflag_extra'     : LLOp(),
     'gc_add_memory_pressure': LLOp(),
 
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1147,6 +1147,10 @@
             fd = os.open(filename, open_flags, 0666)
             os.write(fd, s)
             os.close(fd)
+            #
+            a = rgc.get_typeids_list()
+            assert len(a) > 1
+            assert 0 < rffi.cast(lltype.Signed, a[1]) < 10000
             return 0
 
         return fn

From noreply at buildbot.pypy.org  Tue Dec 23 12:23:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Dec 2014 12:23:02 +0100 (CET)
Subject: [pypy-commit] pypy default: argh,
	typo.  No generator expression intended.
Message-ID: <20141223112302.7D02C1C34AB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75076:6ff7cdf9e09b
Date: 2014-12-23 11:46 +0100
http://bitbucket.org/pypy/pypy/changeset/6ff7cdf9e09b/

Log:	argh, typo. No generator expression intended.

diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -231,5 +231,5 @@
 
 def get_typeids_list(space):
     l = rgc.get_typeids_list()
-    list_w = [space.wrap(l[i] for i in range(len(l)))]
+    list_w = [space.wrap(l[i]) for i in range(len(l))]
     return space.newlist(list_w)

From noreply at buildbot.pypy.org  Tue Dec 23 12:49:07 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Dec 2014 12:49:07 +0100 (CET)
Subject: [pypy-commit] pypy default: Dump the new list into a 'typeids.lst'
	file when calling
Message-ID: <20141223114907.E429B1D24B5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75077:310243e1ade2
Date: 2014-12-23 12:48 +0100
http://bitbucket.org/pypy/pypy/changeset/310243e1ade2/

Log:	Dump the new list into a 'typeids.lst' file when calling
	gc.dump_rpy_heap(filename).

diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -16,7 +16,8 @@
     [0][0][0][-1] inserted after all GC roots, before all non-roots.
 
     If the argument is a filename and the 'zlib' module is available,
-    we also write a 'typeids.txt' in the same directory, if none exists.
+    we also write 'typeids.txt' and 'typeids.lst' in the same directory,
+    if they don't already exist.
     """
     if isinstance(file, str):
         f = open(file, 'wb')
@@ -30,7 +31,13 @@
             filename2 = os.path.join(os.path.dirname(file), 'typeids.txt')
             if not os.path.exists(filename2):
                 data = zlib.decompress(gc.get_typeids_z())
-                f = open(filename2, 'wb')
+                f = open(filename2, 'w')
+                f.write(data)
+                f.close()
+            filename2 = os.path.join(os.path.dirname(file), 'typeids.lst')
+            if not os.path.exists(filename2):
+                data = ''.join(['%d\n' % n for n in gc.get_typeids_list()])
+                f = open(filename2, 'w')
                 f.write(data)
                 f.close()
     else:

From noreply at buildbot.pypy.org  Tue Dec 23 14:50:12 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 23 Dec 2014 14:50:12 +0100 (CET)
Subject: [pypy-commit] pypy gc-counters: merge default
Message-ID: <20141223135012.114FB1C01E8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-counters
Changeset: r75078:690400327627
Date: 2014-12-23 15:48 +0200
http://bitbucket.org/pypy/pypy/changeset/690400327627/

Log:	merge default

diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -397,7 +397,7 @@
 def test_invalid_indexing():
     p = new_primitive_type("int")
     x = cast(p, 42)
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises(TypeError, "x[0]")
 
 def test_default_str():
     BChar = new_primitive_type("char")
diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -31,6 +31,7 @@
                 'get_referrers': 'referents.get_referrers',
                 '_dump_rpy_heap': 'referents._dump_rpy_heap',
                 'get_typeids_z': 'referents.get_typeids_z',
+                'get_typeids_list': 'referents.get_typeids_list',
                 'GcRef': 'referents.W_GcRef',
                 })
         MixedModule.__init__(self, space, w_name)
diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -16,7 +16,8 @@
     [0][0][0][-1] inserted after all GC roots, before all non-roots.
 
     If the argument is a filename and the 'zlib' module is available,
-    we also write a 'typeids.txt' in the same directory, if none exists.
+    we also write 'typeids.txt' and 'typeids.lst' in the same directory,
+    if they don't already exist.
     """
     if isinstance(file, str):
         f = open(file, 'wb')
@@ -30,7 +31,13 @@
             filename2 = os.path.join(os.path.dirname(file), 'typeids.txt')
             if not os.path.exists(filename2):
                 data = zlib.decompress(gc.get_typeids_z())
-                f = open(filename2, 'wb')
+                f = open(filename2, 'w')
+                f.write(data)
+                f.close()
+            filename2 = os.path.join(os.path.dirname(file), 'typeids.lst')
+            if not os.path.exists(filename2):
+                data = ''.join(['%d\n' % n for n in gc.get_typeids_list()])
+                f = open(filename2, 'w')
                 f.write(data)
                 f.close()
     else:
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -228,3 +228,8 @@
     a = rgc.get_typeids_z()
     s = ''.join([a[i] for i in range(len(a))])
     return space.wrap(s)
+
+def get_typeids_list(space):
+    l = rgc.get_typeids_list()
+    list_w = [space.wrap(l[i]) for i in range(len(l))]
+    return space.newlist(list_w)
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -7,7 +7,7 @@
 # all but one will be blocked.  The other threads get a chance to run
 # from time to time, using the periodic action GILReleaseAction.
 
-from rpython.rlib import rthread, rgil
+from rpython.rlib import rthread, rgil, rwin32
 from pypy.module.thread.error import wrap_thread_error
 from pypy.interpreter.executioncontext import PeriodicAsyncAction
 from pypy.module.thread.threadlocals import OSThreadLocals
@@ -76,9 +76,14 @@
 
 def after_external_call():
     e = get_errno()
+    e2 = 0
+    if rwin32.WIN32:
+        e2 = rwin32.GetLastError()
     rgil.gil_acquire()
     rthread.gc_thread_run()
     after_thread_switch()
+    if rwin32.WIN32:
+        rwin32.SetLastError(e2)
     set_errno(e)
 after_external_call._gctransformer_hint_cannot_collect_ = True
 after_external_call._dont_reach_me_in_del_ = True
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -15,14 +15,15 @@
         assert NotEmpty.__dict__.get("b") is None
         raises(TypeError, 'NotEmpty.__dict__[15] = "y"')
         raises(KeyError, 'del NotEmpty.__dict__[15]')
+
+        key, value = NotEmpty.__dict__.popitem()
+        assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
+
         assert NotEmpty.__dict__.setdefault("string", 1) == 1
         assert NotEmpty.__dict__.setdefault("string", 2) == 1
         assert NotEmpty.string == 1
         raises(TypeError, 'NotEmpty.__dict__.setdefault(15, 1)')
 
-        key, value = NotEmpty.__dict__.popitem()
-        assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
-
     def test_dictproxy_getitem(self):
         class NotEmpty(object):
             a = 1
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -1,7 +1,7 @@
 """
 Utility RPython functions to inspect objects in the GC.
 """
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup
 from rpython.rlib.objectmodel import free_non_gc_object
 from rpython.rtyper.module.ll_os import UNDERSCORE_ON_WIN32
 from rpython.rlib import rposix, rgc
@@ -238,3 +238,8 @@
 def get_typeids_z(gc):
     srcaddress = gc.root_walker.gcdata.typeids_z
     return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(rgc.ARRAY_OF_CHAR))
+
+def get_typeids_list(gc):
+    srcaddress = gc.root_walker.gcdata.typeids_list
+    return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(ARRAY_OF_HALFWORDS))
+ARRAY_OF_HALFWORDS = lltype.Array(llgroup.HALFWORD)
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -170,6 +170,7 @@
         gcdata.static_root_end = a_random_address        # patched in finish()
         gcdata.max_type_id = 13                          # patched in finish()
         gcdata.typeids_z = a_random_address              # patched in finish()
+        gcdata.typeids_list = a_random_address           # patched in finish()
         self.gcdata = gcdata
         self.malloc_fnptr_cache = {}
 
@@ -205,6 +206,7 @@
         data_classdef.generalize_attr('static_root_end', SomeAddress())
         data_classdef.generalize_attr('max_type_id', annmodel.SomeInteger())
         data_classdef.generalize_attr('typeids_z', SomeAddress())
+        data_classdef.generalize_attr('typeids_list', SomeAddress())
 
         annhelper = annlowlevel.MixLevelHelperAnnotator(self.translator.rtyper)
 
@@ -455,6 +457,11 @@
                                        [s_gc],
                                        SomePtr(lltype.Ptr(rgc.ARRAY_OF_CHAR)),
                                        minimal_transform=False)
+        self.get_typeids_list_ptr = getfn(inspector.get_typeids_list,
+                                       [s_gc],
+                                       SomePtr(lltype.Ptr(
+                                           lltype.Array(llgroup.HALFWORD))),
+                                       minimal_transform=False)
 
         self.set_max_heap_size_ptr = getfn(GCClass.set_max_heap_size.im_func,
                                            [s_gc,
@@ -604,7 +611,8 @@
         newgcdependencies = []
         newgcdependencies.append(ll_static_roots_inside)
         ll_instance.inst_max_type_id = len(group.members)
-        typeids_z = self.write_typeid_list()
+        #
+        typeids_z, typeids_list = self.write_typeid_list()
         ll_typeids_z = lltype.malloc(rgc.ARRAY_OF_CHAR,
                                      len(typeids_z),
                                      immortal=True)
@@ -612,6 +620,15 @@
             ll_typeids_z[i] = typeids_z[i]
         ll_instance.inst_typeids_z = llmemory.cast_ptr_to_adr(ll_typeids_z)
         newgcdependencies.append(ll_typeids_z)
+        #
+        ll_typeids_list = lltype.malloc(lltype.Array(llgroup.HALFWORD),
+                                        len(typeids_list),
+                                        immortal=True)
+        for i in range(len(typeids_list)):
+            ll_typeids_list[i] = typeids_list[i]
+        ll_instance.inst_typeids_list= llmemory.cast_ptr_to_adr(ll_typeids_list)
+        newgcdependencies.append(ll_typeids_list)
+        #
         return newgcdependencies
 
     def get_finish_tables(self):
@@ -632,6 +649,13 @@
         # XXX argh argh, this only gives the member index but not the
         #     real typeid, which is a complete mess to obtain now...
         all_ids = self.layoutbuilder.id_of_type.items()
+        list_data = []
+        ZERO = rffi.cast(llgroup.HALFWORD, 0)
+        for _, typeinfo in all_ids:
+            while len(list_data) <= typeinfo.index:
+                list_data.append(ZERO)
+            list_data[typeinfo.index] = typeinfo
+        #
         all_ids = [(typeinfo.index, TYPE) for (TYPE, typeinfo) in all_ids]
         all_ids = dict(all_ids)
         f = udir.join("typeids.txt").open("w")
@@ -640,9 +664,10 @@
         f.close()
         try:
             import zlib
-            return zlib.compress(udir.join("typeids.txt").read(), 9)
+            z_data = zlib.compress(udir.join("typeids.txt").read(), 9)
         except ImportError:
-            return ''
+            z_data = ''
+        return z_data, list_data
 
     def transform_graph(self, graph):
         func = getattr(graph, 'func', None)
@@ -1183,6 +1208,13 @@
                   resultvar=hop.spaceop.result)
         self.pop_roots(hop, livevars)
 
+    def gct_gc_typeids_list(self, hop):
+        livevars = self.push_roots(hop)
+        hop.genop("direct_call",
+                  [self.get_typeids_list_ptr, self.c_const_gc],
+                  resultvar=hop.spaceop.result)
+        self.pop_roots(hop, livevars)
+
     def _set_into_gc_array_part(self, op):
         if op.opname == 'setarrayitem':
             return op.args[1]
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -445,6 +445,10 @@
     "NOT_RPYTHON"
     raise NotImplementedError
 
+def get_typeids_list():
+    "NOT_RPYTHON"
+    raise NotImplementedError
+
 def has_gcflag_extra():
     "NOT_RPYTHON"
     return True
@@ -644,6 +648,18 @@
         return hop.genop('gc_typeids_z', [], resulttype = hop.r_result)
 
 class Entry(ExtRegistryEntry):
+    _about_ = get_typeids_list
+
+    def compute_result_annotation(self):
+        from rpython.rtyper.llannotation import SomePtr
+        from rpython.rtyper.lltypesystem import llgroup
+        return SomePtr(lltype.Ptr(lltype.Array(llgroup.HALFWORD)))
+
+    def specialize_call(self, hop):
+        hop.exception_is_here()
+        return hop.genop('gc_typeids_list', [], resulttype = hop.r_result)
+
+class Entry(ExtRegistryEntry):
     _about_ = (has_gcflag_extra, get_gcflag_extra, toggle_gcflag_extra)
     def compute_result_annotation(self, s_arg=None):
         from rpython.annotator.model import s_Bool
diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -118,8 +118,10 @@
     INVALID_HANDLE_VALUE = rffi.cast(HANDLE, -1)
     PFILETIME = rffi.CArrayPtr(FILETIME)
 
-    _GetLastError = winexternal('GetLastError', [], DWORD, releasegil=False)
-    _SetLastError = winexternal('SetLastError', [DWORD], lltype.Void)
+    _GetLastError = winexternal('GetLastError', [], DWORD,
+                                _nowrapper=True, sandboxsafe=True)
+    _SetLastError = winexternal('SetLastError', [DWORD], lltype.Void,
+                                _nowrapper=True, sandboxsafe=True)
 
     def GetLastError():
         return rffi.cast(lltype.Signed, _GetLastError())
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -911,6 +911,9 @@
     def op_gc_typeids_z(self):
         raise NotImplementedError("gc_typeids_z")
 
+    def op_gc_typeids_list(self):
+        raise NotImplementedError("gc_typeids_list")
+
     def op_gc_gcflag_extra(self, subopnum, *args):
         return self.heap.gcflag_extra(subopnum, *args)
 
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -493,6 +493,7 @@
     'gc_is_rpy_instance'  : LLOp(),
     'gc_dump_rpy_heap'    : LLOp(),
     'gc_typeids_z'        : LLOp(),
+    'gc_typeids_list'     : LLOp(),
     'gc_gcflag_extra'     : LLOp(),
     'gc_add_memory_pressure': LLOp(),
     'gc_reset_tid_counters': LLOp(),
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1147,6 +1147,10 @@
             fd = os.open(filename, open_flags, 0666)
             os.write(fd, s)
             os.close(fd)
+            #
+            a = rgc.get_typeids_list()
+            assert len(a) > 1
+            assert 0 < rffi.cast(lltype.Signed, a[1]) < 10000
             return 0
 
         return fn

From noreply at buildbot.pypy.org  Tue Dec 23 14:50:13 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 23 Dec 2014 14:50:13 +0100 (CET)
Subject: [pypy-commit] pypy gc-counters: count the total size
Message-ID: <20141223135013.4A7371C01E8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-counters
Changeset: r75079:db1879e7c5f7
Date: 2014-12-23 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/db1879e7c5f7/

Log:	count the total size

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1862,7 +1862,7 @@
         typeid = self.get_type_id(obj)
 
         # XXX
-        self.tid_counters[typeid] += 1
+        self.tid_counters[typeid] += totalsize
         
         obj = llarena.getfakearenaaddress(obj)
         llarena.arena_reset(obj - size_gc_header, totalsize, 0)

From noreply at buildbot.pypy.org  Tue Dec 23 14:58:53 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 23 Dec 2014 14:58:53 +0100 (CET)
Subject: [pypy-commit] pypy gc-counters: we don't quite care about the
 performance here on this branch so just call
Message-ID: <20141223135853.562861C10F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-counters
Changeset: r75080:cf2ed69b520d
Date: 2014-12-23 15:58 +0200
http://bitbucket.org/pypy/pypy/changeset/cf2ed69b520d/

Log:	we don't quite care about the performance here on this branch so
	just call get_member_index and get rid of a problem of size and
	fetching type ids from somewhere else

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1862,7 +1862,7 @@
         typeid = self.get_type_id(obj)
 
         # XXX
-        self.tid_counters[typeid] += totalsize
+        self.tid_counters[self.get_member_index(typeid)] += totalsize
         
         obj = llarena.getfakearenaaddress(obj)
         llarena.arena_reset(obj - size_gc_header, totalsize, 0)

From noreply at buildbot.pypy.org  Tue Dec 23 15:13:56 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 15:13:56 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_field_basic to pass
Message-ID: <20141223141356.4581F1C1347@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75081:7842cdbb5db6
Date: 2014-12-23 09:21 -0500
http://bitbucket.org/pypy/pypy/changeset/7842cdbb5db6/

Log:	Get test_field_basic to pass

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -4,7 +4,7 @@
 from rpython.jit.backend.llsupport.regalloc import FrameManager
 from rpython.jit.backend.model import CompiledLoopToken
 from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library, make_param_array, make_field_array, Context, Type
-from rpython.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, ConstFloat
+from rpython.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, ConstFloat, BoxPtr, ConstPtr
 from rpython.jit.metainterp.resoperation import *
 from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref, cast_object_to_ptr
 from rpython.rtyper.lltypesystem.rffi import *
@@ -196,12 +196,15 @@
         self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
         self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
         self.t_void = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID)
+        self.t_char_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_CHAR).get_pointer()
 
         self.u_signed = self.ctxt.new_field(self.t_Signed, "u_signed")
         self.u_float = self.ctxt.new_field(self.t_float, "u_float")
+        self.u_ptr = self.ctxt.new_field(self.t_void_ptr, "u_ptr")
         self.t_any = self.ctxt.new_union_type ("any",
                                                [self.u_signed,
-                                                self.u_float])
+                                                self.u_float,
+                                                self.u_ptr])
 
     def setup(self, looptoken):
         allblocks = self.get_asmmemmgr_blocks(looptoken)
@@ -429,7 +432,7 @@
         print(' %s' % expr.__dict__)
         """
 
-        if isinstance(expr, (BoxInt, BoxFloat)):
+        if isinstance(expr, (BoxInt, BoxFloat, BoxPtr)):
             return self.get_box_as_lvalue(expr).as_rvalue()
         elif isinstance(expr, ConstInt):
             #print('value: %r' % expr.value)
@@ -441,6 +444,11 @@
             #print('type(value): %r' % type(expr.value))
             return self.ctxt.new_rvalue_from_double(self.t_float,
                                                     expr.value)#r_double(expr.value))
+        elif isinstance(expr, ConstPtr):
+            #print('value: %r' % expr.value)
+            #print('type(value): %r' % type(expr.value))
+            return self.ctxt.new_rvalue_from_ptr(self.t_void_ptr,
+                                                 expr.value)
         raise ValueError('unhandled expr: %s %s' % (expr, type(expr)))
 
     def get_box_as_lvalue(self, box):
@@ -463,7 +471,7 @@
         print(' %s' % dir(expr))
         print(' %s' % expr.__dict__)
         """
-        if isinstance(expr, (BoxInt, BoxFloat)):
+        if isinstance(expr, (BoxInt, BoxFloat, BoxPtr)):
             return self.get_box_as_lvalue(expr)
             
         raise ValueError('unhandled expr: %s' % expr)
@@ -473,6 +481,8 @@
             return self.t_Signed;
         elif isinstance(box, BoxFloat):
             return self.t_float;
+        elif isinstance(box, BoxPtr):
+            return self.t_void_ptr;
         else:
             raise ValueError('unhandled box: %s %s' % (box, type(box)))
 
@@ -481,6 +491,8 @@
             return self.u_signed;
         elif isinstance(expr, (BoxFloat, ConstFloat)):
             return self.u_float;
+        elif isinstance(expr, (BoxPtr, ConstPtr)):
+            return self.u_ptr;
         else:
             raise ValueError('unhandled expr: %s %s' % (expr, type(expr)))        
 
@@ -788,3 +800,72 @@
         self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_GT)
     def emit_float_ge(self, resop):
         self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
+
+    def impl_get_lvalue_for_field(self, ptr_expr, fielddescr):
+        assert isinstance(ptr_expr, (BoxPtr, ConstPtr))
+        #print(fielddescr)
+        #print(dir(fielddescr))
+        #print('fielddescr.field_size: %r' % fielddescr.field_size)
+
+        ptr = self.expr_to_rvalue(ptr_expr)
+
+        # Cast to (char *) so we can use offset:
+        # ((char *)ARG0)
+        ptr = self.ctxt.new_cast(ptr, self.t_char_ptr)
+
+        # ((char *)ARG0)[offset]
+        ptr = self.ctxt.new_array_access(
+            ptr,
+            self.ctxt.new_rvalue_from_int(self.t_Signed,
+                                          r_int(fielddescr.offset)))
+
+        # (char **)(((char *)ARG0)[offset])
+        field_ptr_address = ptr.get_address ()
+
+        # ...and cast back to the correct type:
+        if fielddescr.is_pointer_field():
+            t_field = self.t_void_ptr
+        elif fielddescr.is_float_field():
+            # FIXME: do we need to handle C float vs C double?
+            t_field = self.t_float
+        else:
+            t_field = self.ctxt.get_int_type(r_int(fielddescr.field_size),
+                                             r_int(fielddescr.is_field_signed()))
+
+        # (T *)(char **)(((char *)ARG0)[offset])
+        field_ptr = self.ctxt.new_cast(field_ptr_address,
+                                       t_field.get_pointer())
+
+        # and dereference:
+        # *(T *)(char **)(((char *)ARG0)[offset])
+        field_lvalue = field_ptr.dereference()
+
+        return field_lvalue, t_field
+
+    def emit_getfield_gc(self, resop):
+        #print(repr(resop))
+        assert isinstance(resop._arg0, (BoxPtr, ConstPtr))
+        lvalres = self.expr_to_lvalue(resop.result)
+        field_lvalue, t_field = self.impl_get_lvalue_for_field(
+            resop._arg0,
+            resop.getdescr())
+        self.b_current.add_assignment(
+            lvalres,
+            self.ctxt.new_cast(field_lvalue.as_rvalue(),
+                               self.get_type_for_box(resop.result)))
+
+    def emit_setfield_gc(self, resop):
+        #print(repr(resop))
+        assert isinstance(resop._arg0, (BoxPtr, ConstPtr))
+        #print(resop._arg1)
+
+        field_lvalue, t_field = self.impl_get_lvalue_for_field(
+            resop._arg0,
+            resop.getdescr())
+
+        self.b_current.add_assignment(
+            field_lvalue,
+            
+            # ... = ARG1,
+            self.ctxt.new_cast(self.expr_to_rvalue(resop._arg1),
+                               t_field))
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -308,6 +308,12 @@
                                               self.GCC_JIT_TYPE_P]),
 
                 (self.GCC_JIT_LVALUE_P,
+                 'gcc_jit_context_new_array_access', [self.GCC_JIT_CONTEXT_P,
+                                                      self.GCC_JIT_LOCATION_P,
+                                                      self.GCC_JIT_RVALUE_P,
+                                                      self.GCC_JIT_RVALUE_P]),
+
+                (self.GCC_JIT_LVALUE_P,
                  'gcc_jit_lvalue_access_field', [self.GCC_JIT_LVALUE_P,
                                                  self.GCC_JIT_LOCATION_P,
                                                  self.GCC_JIT_FIELD_P]),
@@ -317,6 +323,14 @@
                                                       self.GCC_JIT_LOCATION_P,
                                                       self.GCC_JIT_FIELD_P]),
 
+                (self.GCC_JIT_LVALUE_P,
+                 'gcc_jit_rvalue_dereference', [self.GCC_JIT_RVALUE_P,
+                                                self.GCC_JIT_LOCATION_P]),
+
+                (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_lvalue_get_address', [self.GCC_JIT_LVALUE_P,
+                                                self.GCC_JIT_LOCATION_P]),
+
                 ############################################################
                 # Statement-creation.
                 ############################################################
@@ -698,6 +712,15 @@
                                                         rvalue.inner_rvalue,
                                                         type_.inner_type))
 
+    def new_array_access(self, ptr, index):
+        return LValue(self.lib,
+                      self,
+                      self.lib.gcc_jit_context_new_array_access(
+                          self.inner_ctxt,
+                          self.lib.null_location_ptr,
+                          ptr.inner_rvalue,
+                          index.inner_rvalue))
+
 class LibgccjitError(Exception):
     def __init__(self, ctxt):
         self.msg = charp2str(ctxt.lib.gcc_jit_context_get_last_error (ctxt.inner_ctxt))
@@ -763,6 +786,13 @@
                                                                 self.lib.null_location_ptr,
                                                                 field.inner_field))
 
+    def dereference(self):
+        return LValue(self.lib,
+                      self,
+                      self.lib.gcc_jit_rvalue_dereference(
+                          self.inner_rvalue,
+                          self.lib.null_location_ptr))
+
 class LValue(Object):
     def __init__(self, lib, ctxt, inner_lvalue):
         Object.__init__(self, lib, ctxt, inner_lvalue)
@@ -780,6 +810,13 @@
                                                             self.lib.null_location_ptr,
                                                             field.inner_field))
 
+    def get_address(self):
+        return RValue(self.lib,
+                      self,
+                      self.lib.gcc_jit_lvalue_get_address(
+                          self.inner_lvalue,
+                          self.lib.null_location_ptr))
+
 class Param(Object):
     def __init__(self, lib, ctxt, inner_param):
         Object.__init__(self, lib, ctxt, inner_param)

From noreply at buildbot.pypy.org  Tue Dec 23 15:41:52 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 15:41:52 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_passing_guards to
	pass
Message-ID: <20141223144152.B94B31D22DE@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75082:cf9335135432
Date: 2014-12-23 09:49 -0500
http://bitbucket.org/pypy/pypy/changeset/cf9335135432/

Log:	Get test_passing_guards to pass

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -3,7 +3,9 @@
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.regalloc import FrameManager
 from rpython.jit.backend.model import CompiledLoopToken
-from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library, make_param_array, make_field_array, Context, Type
+from rpython.jit.backend.libgccjit.rffi_bindings import (
+    make_eci, Library, make_param_array, make_field_array, Context, Type,
+    RValue)
 from rpython.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, ConstFloat, BoxPtr, ConstPtr
 from rpython.jit.metainterp.resoperation import *
 from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref, cast_object_to_ptr
@@ -572,13 +574,10 @@
 
     # GUARD_*
 
-    def _impl_guard(self, resop, istrue):
-        print(resop)
-        print(resop.__dict__)
+    def _impl_guard(self, resop, istrue, boolval):
+        assert isinstance(boolval, RValue)
         b_true = self.fn.new_block("on_true_at_%s" % resop)
         b_false = self.fn.new_block("on_false_at_%s" % resop)
-        boolval = self.ctxt.new_cast(self.expr_to_rvalue(resop._arg0),
-                                     self.t_bool)
         self.b_current.end_with_conditional(boolval,
                                             b_true, b_false)
 
@@ -616,11 +615,39 @@
         # Further operations go into the guard success block in the original fn:
         self.b_current = b_guard_success
 
+    def _impl_bool_guard(self, resop, istrue):
+        boolval = self.ctxt.new_cast(self.expr_to_rvalue(resop._arg0),
+                                     self.t_bool)
+        self._impl_guard(resop, istrue, boolval)
+
     def emit_guard_true(self, resop):
-        self._impl_guard(resop, r_int(1))
+        self._impl_bool_guard(resop, r_int(1))
         
     def emit_guard_false(self, resop):
-        self._impl_guard(resop, r_int(0))
+        self._impl_bool_guard(resop, r_int(0))
+
+    def emit_guard_value(self, resop):
+        boolval = self.ctxt.new_comparison(
+            self.lib.GCC_JIT_COMPARISON_EQ,
+            self.expr_to_rvalue(resop._arg0),
+            self.expr_to_rvalue(resop._arg1))
+        self._impl_guard(resop, r_int(1), boolval)
+
+    def emit_guard_nonnull(self, resop):
+        ptr_rvalue = self.expr_to_rvalue(resop._arg0)
+        boolval = self.ctxt.new_comparison(
+            self.lib.GCC_JIT_COMPARISON_NE,
+            ptr_rvalue,
+            self.ctxt.null(ptr_rvalue.get_type()))
+        self._impl_guard(resop, r_int(1), boolval)
+
+    def emit_guard_isnull(self, resop):
+        ptr_rvalue = self.expr_to_rvalue(resop._arg0)
+        boolval = self.ctxt.new_comparison(
+            self.lib.GCC_JIT_COMPARISON_EQ,
+            ptr_rvalue,
+            self.ctxt.null(ptr_rvalue.get_type()))
+        self._impl_guard(resop, r_int(1), boolval)
 
     def _impl_write_output_args(self, params, args):
         # Write outputs back:
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -241,6 +241,9 @@
                 (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_lvalue_as_rvalue', [self.GCC_JIT_LVALUE_P]),
 
+                (self.GCC_JIT_TYPE_P,
+                 'gcc_jit_rvalue_get_type', [self.GCC_JIT_RVALUE_P]),
+
                 # Integer constants.
                 (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_context_new_rvalue_from_int', [self.GCC_JIT_CONTEXT_P,
@@ -266,6 +269,10 @@
                                                          VOIDP]),
 
                 (self.GCC_JIT_RVALUE_P,
+                 'gcc_jit_context_null', [self.GCC_JIT_CONTEXT_P,
+                                          self.GCC_JIT_TYPE_P]),
+
+                (self.GCC_JIT_RVALUE_P,
                  'gcc_jit_context_new_unary_op', [self.GCC_JIT_CONTEXT_P,
                                                   self.GCC_JIT_LOCATION_P,
                                                   INT, # enum gcc_jit_unary_op op,
@@ -610,6 +617,13 @@
                                                                    type_.inner_type,
                                                                    llvalue))
 
+
+    def null(self, pointer_type):
+        return RValue(self.lib,
+                      self,
+                      self.lib.gcc_jit_context_null(self.inner_ctxt,
+                                                    pointer_type.inner_type))
+
     def new_unary_op(self, op, type_, rvalue):
         return RValue(self.lib,
                       self,
@@ -779,6 +793,11 @@
         Object.__init__(self, lib, ctxt, inner_rvalue)
         self.inner_rvalue = inner_rvalue
 
+    def get_type(self):
+        return Type(self.lib,
+                    self,
+                    self.lib.gcc_jit_rvalue_get_type(self.inner_rvalue))
+
     def dereference_field(self, field):
         return LValue(self.lib,
                       self,

From noreply at buildbot.pypy.org  Tue Dec 23 16:32:54 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 23 Dec 2014 16:32:54 +0100 (CET)
Subject: [pypy-commit] pypy default: Ignore exception message
Message-ID: <20141223153254.2ADAE1D2549@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75083:cb215931449b
Date: 2014-12-23 16:32 +0100
http://bitbucket.org/pypy/pypy/changeset/cb215931449b/

Log:	Ignore exception message

diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -225,9 +225,9 @@
     >>> element.remove(subelement)
     >>> serialize(element) # 5
     '<tag key="value" />'
-    >>> element.remove(subelement)
+    >>> element.remove(subelement)    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ValueError: list.remove(x): x not in list
+    ValueError: list.remove(x): ...
     >>> serialize(element) # 6
     '<tag key="value" />'
     >>> element[0:0] = [subelement, subelement, subelement]

From noreply at buildbot.pypy.org  Tue Dec 23 17:30:36 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 17:30:36 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Work towards implementing
 GUARD_CLASS and GUARD_NONNULL_CLASS
Message-ID: <20141223163036.6C20F1D22DE@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75084:45e9e4b77db5
Date: 2014-12-23 10:51 -0500
http://bitbucket.org/pypy/pypy/changeset/45e9e4b77db5/

Log:	Work towards implementing GUARD_CLASS and GUARD_NONNULL_CLASS

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -422,6 +422,8 @@
 
             # Compile the operation itself...
             methname = 'emit_%s' % op.getopname()
+            if not hasattr(self, methname):
+                raise NotImplementedError('resop not yet implemented: %s' % op)
             getattr(self, methname) (op)
 
     def expr_to_rvalue(self, expr):
@@ -439,6 +441,11 @@
         elif isinstance(expr, ConstInt):
             #print('value: %r' % expr.value)
             #print('type(value): %r' % type(expr.value))
+            if isinstance(expr.value, llmemory.AddressAsInt):
+                assert isinstance(expr.value.adr, llmemory.fakeaddress)
+                assert isinstance(expr.value.adr.ptr, lltype._ptr)
+                return self.ctxt.new_rvalue_from_ptr(self.t_void_ptr,
+                                                     expr.value.adr.ptr)
             return self.ctxt.new_rvalue_from_long(self.t_Signed,
                                                   r_long(expr.value))
         elif isinstance(expr, ConstFloat):
@@ -451,7 +458,7 @@
             #print('type(value): %r' % type(expr.value))
             return self.ctxt.new_rvalue_from_ptr(self.t_void_ptr,
                                                  expr.value)
-        raise ValueError('unhandled expr: %s %s' % (expr, type(expr)))
+        raise NotImplementedError('unhandled expr: %s %s' % (expr, type(expr)))
 
     def get_box_as_lvalue(self, box):
         if box not in self.lvalue_for_box:
@@ -476,7 +483,7 @@
         if isinstance(expr, (BoxInt, BoxFloat, BoxPtr)):
             return self.get_box_as_lvalue(expr)
             
-        raise ValueError('unhandled expr: %s' % expr)
+        raise NotImplementedError('unhandled expr: %s' % expr)
 
     def get_type_for_box(self, box):
         if isinstance(box, BoxInt):
@@ -486,7 +493,7 @@
         elif isinstance(box, BoxPtr):
             return self.t_void_ptr;
         else:
-            raise ValueError('unhandled box: %s %s' % (box, type(box)))
+            raise NotImplementedError('unhandled box: %s %s' % (box, type(box)))
 
     def get_union_field_for_expr(self, expr):
         if isinstance(expr, (BoxInt, ConstInt)):
@@ -496,7 +503,7 @@
         elif isinstance(expr, (BoxPtr, ConstPtr)):
             return self.u_ptr;
         else:
-            raise ValueError('unhandled expr: %s %s' % (expr, type(expr)))        
+            raise NotImplementedError('unhandled expr: %s %s' % (expr, type(expr)))
 
     # Handling of specific ResOperation subclasses: each one is
     # a method named "emit_foo", where "foo" is the str() of the resop.
@@ -633,6 +640,21 @@
             self.expr_to_rvalue(resop._arg1))
         self._impl_guard(resop, r_int(1), boolval)
 
+    def emit_guard_class(self, resop):
+        print(resop)
+        print(dir(resop))
+        vtable_offset = self.cpu.vtable_offset
+        print('vtable_offset: %r' % vtable_offset)
+        assert vtable_offset is not None
+        lvalue_obj_vtable = self.impl_get_lvalue_at_offset_from_ptr(
+            resop._arg0, r_int(vtable_offset), self.t_void_ptr)
+        print('resop._arg1: %r' % resop._arg1)
+        boolval = self.ctxt.new_comparison(
+            self.lib.GCC_JIT_COMPARISON_EQ,
+            lvalue_obj_vtable.as_rvalue(),
+            self.expr_to_rvalue(resop._arg1))
+        self._impl_guard(resop, r_int(1), boolval)
+
     def emit_guard_nonnull(self, resop):
         ptr_rvalue = self.expr_to_rvalue(resop._arg0)
         boolval = self.ctxt.new_comparison(
@@ -828,12 +850,7 @@
     def emit_float_ge(self, resop):
         self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
 
-    def impl_get_lvalue_for_field(self, ptr_expr, fielddescr):
-        assert isinstance(ptr_expr, (BoxPtr, ConstPtr))
-        #print(fielddescr)
-        #print(dir(fielddescr))
-        #print('fielddescr.field_size: %r' % fielddescr.field_size)
-
+    def impl_get_lvalue_at_offset_from_ptr(self, ptr_expr, ll_offset, t_field):
         ptr = self.expr_to_rvalue(ptr_expr)
 
         # Cast to (char *) so we can use offset:
@@ -843,22 +860,11 @@
         # ((char *)ARG0)[offset]
         ptr = self.ctxt.new_array_access(
             ptr,
-            self.ctxt.new_rvalue_from_int(self.t_Signed,
-                                          r_int(fielddescr.offset)))
+            self.ctxt.new_rvalue_from_int(self.t_Signed, ll_offset))
 
         # (char **)(((char *)ARG0)[offset])
         field_ptr_address = ptr.get_address ()
 
-        # ...and cast back to the correct type:
-        if fielddescr.is_pointer_field():
-            t_field = self.t_void_ptr
-        elif fielddescr.is_float_field():
-            # FIXME: do we need to handle C float vs C double?
-            t_field = self.t_float
-        else:
-            t_field = self.ctxt.get_int_type(r_int(fielddescr.field_size),
-                                             r_int(fielddescr.is_field_signed()))
-
         # (T *)(char **)(((char *)ARG0)[offset])
         field_ptr = self.ctxt.new_cast(field_ptr_address,
                                        t_field.get_pointer())
@@ -867,13 +873,36 @@
         # *(T *)(char **)(((char *)ARG0)[offset])
         field_lvalue = field_ptr.dereference()
 
-        return field_lvalue, t_field
+        return field_lvalue
+
+    def impl_get_type_for_field(self, fielddescr):
+        # ...and cast back to the correct type:
+        if fielddescr.is_pointer_field():
+            return self.t_void_ptr
+        elif fielddescr.is_float_field():
+            # FIXME: do we need to handle C float vs C double?
+            return self.t_float
+        else:
+            return self.ctxt.get_int_type(r_int(fielddescr.field_size),
+                                          r_int(fielddescr.is_field_signed()))
+
+    def impl_get_lvalue_for_field(self, ptr_expr, fielddescr):
+        assert isinstance(ptr_expr, (BoxPtr, ConstPtr))
+        #print(fielddescr)
+        #print(dir(fielddescr))
+        #print('fielddescr.field_size: %r' % fielddescr.field_size)
+
+        offset = fielddescr.offset
+        t_field = self.impl_get_type_for_field(fielddescr)
+
+        return self.impl_get_lvalue_at_offset_from_ptr(
+            ptr_expr, r_int(offset), t_field)
 
     def emit_getfield_gc(self, resop):
         #print(repr(resop))
         assert isinstance(resop._arg0, (BoxPtr, ConstPtr))
         lvalres = self.expr_to_lvalue(resop.result)
-        field_lvalue, t_field = self.impl_get_lvalue_for_field(
+        field_lvalue = self.impl_get_lvalue_for_field(
             resop._arg0,
             resop.getdescr())
         self.b_current.add_assignment(
@@ -886,9 +915,10 @@
         assert isinstance(resop._arg0, (BoxPtr, ConstPtr))
         #print(resop._arg1)
 
-        field_lvalue, t_field = self.impl_get_lvalue_for_field(
+        field_lvalue = self.impl_get_lvalue_for_field(
             resop._arg0,
             resop.getdescr())
+        t_field = self.impl_get_type_for_field(fielddescr)
 
         self.b_current.add_assignment(
             field_lvalue,

From noreply at buildbot.pypy.org  Tue Dec 23 17:30:37 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 17:30:37 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Whitespace cleanups
Message-ID: <20141223163037.A675D1D22DE@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75085:a2a235719797
Date: 2014-12-23 11:07 -0500
http://bitbucket.org/pypy/pypy/changeset/a2a235719797/

Log:	Whitespace cleanups

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -6,9 +6,11 @@
 from rpython.jit.backend.libgccjit.rffi_bindings import (
     make_eci, Library, make_param_array, make_field_array, Context, Type,
     RValue)
-from rpython.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, ConstFloat, BoxPtr, ConstPtr
+from rpython.jit.metainterp.history import (
+    BoxInt, ConstInt, BoxFloat, ConstFloat, BoxPtr, ConstPtr)
 from rpython.jit.metainterp.resoperation import *
-from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref, cast_object_to_ptr
+from rpython.rtyper.annlowlevel import (
+    llhelper, cast_instance_to_gcref, cast_object_to_ptr)
 from rpython.rtyper.lltypesystem.rffi import *
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
 
@@ -34,10 +36,10 @@
         self.rffi_fn = rffi_fn
         self.patchpoints = []
 
-    def new_block(self, name): 
+    def new_block(self, name):
        return self.rffi_fn.new_block(name)
 
-    def new_local(self, type_, name): 
+    def new_local(self, type_, name):
        return self.rffi_fn.new_local(type_, name)
 
 class Patchpoint:
@@ -69,7 +71,7 @@
 
       0x00007fffeb7086d0 <+16>:  jle    0x7fffeb7086c8 <anonloop_0+8>
       0x00007fffeb7086d2 <+18>:  mov    %rax,0x48(%rdi)
-      0x00007fffeb7086d6 <+22>:  mov    0x2008fb(%rip),%rax        # 0x7fffeb908fd8
+      0x00007fffeb7086d6 <+22>:  mov    0x2008fb(%rip),%rax   # 0x7fffeb908fd8
       0x00007fffeb7086dd <+29>:  mov    (%rax),%rax
       0x00007fffeb7086e0 <+32>:  jmpq   *%rax
     """
@@ -77,10 +79,10 @@
         self.failure_params = Params(assembler)
         self.serial = assembler.num_guard_failure_fns
         assembler.num_guard_failure_fns += 1
-        self.t_fn_ptr_type = (
-            assembler.ctxt.new_function_ptr_type (assembler.t_jit_frame_ptr,
-                                                  self.failure_params.paramtypes,
-                                                  r_int(0)))
+        self.t_fn_ptr_type = assembler.ctxt.new_function_ptr_type (
+            assembler.t_jit_frame_ptr,
+            self.failure_params.paramtypes,
+            r_int(0))
         # Create the function ptr
         # Globals are zero-initialized, so we'll need to
         # write in the ptr to the initial handler before the loop is called,
@@ -169,36 +171,44 @@
     def make_context(self):
         self.ctxt = Context.acquire(self.lib)#self.lib.gcc_jit_context_acquire()
         if 0:
-            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_TREE,
-                                      r_int(1))
+            self.ctxt.set_bool_option(
+                self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_TREE,
+                r_int(1))
         if 0:
-            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
-                                      r_int(1))
+            self.ctxt.set_bool_option(
+                self.lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
+                r_int(1))
         if 1:
-            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DEBUGINFO,
-                                      r_int(1))
+            self.ctxt.set_bool_option(
+                self.lib.GCC_JIT_BOOL_OPTION_DEBUGINFO,
+                r_int(1))
         if 1:
-            self.ctxt.set_int_option(self.lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
-                                     r_int(2))
+            self.ctxt.set_int_option(
+                self.lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
+                r_int(2))
         if 1:
-            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
-                                      r_int(1))
+            self.ctxt.set_bool_option(
+                self.lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
+                r_int(1))
         if 1:
-            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
-                                      r_int(1))
+            self.ctxt.set_bool_option(
+                self.lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
+                r_int(1))
         if 0:
-            self.ctxt.set_bool_option(self.lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
-                                      r_int(1))
+            self.ctxt.set_bool_option(
+                self.lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
+                r_int(1))
 
         self.t_Signed = self.ctxt.get_int_type(r_int(self.sizeof_signed),
                                                r_int(1))
         self.t_UINT = self.ctxt.get_int_type(r_int(self.sizeof_signed),
                                              r_int(0))
-        self.t_float = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_DOUBLE) # FIXME                                          
+        self.t_float = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_DOUBLE) # FIXME
         self.t_bool = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_BOOL)
         self.t_void_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID_PTR)
         self.t_void = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_VOID)
-        self.t_char_ptr = self.ctxt.get_type(self.lib.GCC_JIT_TYPE_CHAR).get_pointer()
+        self.t_char_ptr = self.ctxt.get_type(
+            self.lib.GCC_JIT_TYPE_CHAR).get_pointer()
 
         self.u_signed = self.ctxt.new_field(self.t_Signed, "u_signed")
         self.u_float = self.ctxt.new_field(self.t_float, "u_float")
@@ -374,12 +384,13 @@
         #jitframe.JITFRAMEINFOPTR
         self.loop_params = Params(self)
 
-        self.fn = Function(name,
-                           self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_EXPORTED,
-                                                  self.t_jit_frame_ptr,
-                                                  name,
-                                                  self.loop_params.paramlist,
-                                                  r_int(0)))
+        self.fn = Function(
+            name,
+            self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_EXPORTED,
+                                   self.t_jit_frame_ptr,
+                                   name,
+                                   self.loop_params.paramlist,
+                                   r_int(0)))
 
         self.b_current = self.fn.new_block("initial")
 
@@ -401,11 +412,13 @@
             # (gdb) p *(double*)&jitframe->arg1
             # $8 = -2.25
             #src_ptr_rvalue = self.get_arg_as_lvalue(idx).get_address()
-            #src_ptr_rvalue = self.ctxt.new_cast(src_ptr_rvalue, self.t_float.get_pointer())
+            #src_ptr_rvalue = self.ctxt.new_cast(src_ptr_rvalue,
+            #                                    self.t_float.get_pointer())
             #src_rvalue = self.ctxt.new_dereference(src_ptr_rvalue)
             # or do it as a union
             field = self.get_union_field_for_expr(arg)
-            src_rvalue = self.get_arg_as_lvalue(idx).access_field(field).as_rvalue()
+            src_rvalue = self.get_arg_as_lvalue(
+                idx).access_field(field).as_rvalue()
             # FIXME: this may need a cast:
             #src_rvalue = self.ctxt.new_cast(src_rvalue, self.t_float)
             self.b_current.add_assignment(
@@ -451,8 +464,9 @@
         elif isinstance(expr, ConstFloat):
             #print('value: %r' % expr.value)
             #print('type(value): %r' % type(expr.value))
-            return self.ctxt.new_rvalue_from_double(self.t_float,
-                                                    expr.value)#r_double(expr.value))
+            return self.ctxt.new_rvalue_from_double(
+                self.t_float,
+                expr.value)#r_double(expr.value))
         elif isinstance(expr, ConstPtr):
             #print('value: %r' % expr.value)
             #print('type(value): %r' % type(expr.value))
@@ -482,7 +496,7 @@
         """
         if isinstance(expr, (BoxInt, BoxFloat, BoxPtr)):
             return self.get_box_as_lvalue(expr)
-            
+
         raise NotImplementedError('unhandled expr: %s' % expr)
 
     def get_type_for_box(self, box):
@@ -503,7 +517,8 @@
         elif isinstance(expr, (BoxPtr, ConstPtr)):
             return self.u_ptr;
         else:
-            raise NotImplementedError('unhandled expr: %s %s' % (expr, type(expr)))
+            raise NotImplementedError('unhandled expr: %s %s'
+                                      % (expr, type(expr)))
 
     # Handling of specific ResOperation subclasses: each one is
     # a method named "emit_foo", where "foo" is the str() of the resop.
@@ -524,8 +539,8 @@
 
         # We need to write to the boxes listed in the label's args
         # with the values from those listed in the jump's args.
-        # However, there are potential cases like JUMP(i0, i1) going to LABEL(i1, i0)
-        # where all such assignments happen "simultaneously".
+        # However, there are potential cases like JUMP(i0, i1) going
+        # to LABEL(i1, i0) where all such assignments happen "simultaneously".
         # Hence we need to set up temporaries.
         # First pass: capture the value of relevant boxes at the JUMP:
         tmps = []
@@ -551,11 +566,12 @@
             # Sadly, we need to "import" the fn by name, since it was
             # created on a different gcc_jit_context.
             p = Params(self)
-            other_fn = self.ctxt.new_function(self.lib.GCC_JIT_FUNCTION_IMPORTED,
-                                              self.t_jit_frame_ptr,
-                                              dest_fn.name,
-                                              p.paramlist,
-                                              r_int(0))
+            other_fn = self.ctxt.new_function(
+                self.lib.GCC_JIT_FUNCTION_IMPORTED,
+                self.t_jit_frame_ptr,
+                dest_fn.name,
+                p.paramlist,
+                r_int(0))
             args = [param.as_rvalue()
                     for param in self.loop_params.paramlist]
             call = self.ctxt.new_call(other_fn, args)
@@ -564,7 +580,8 @@
     def emit_finish(self, resop):
         self._impl_write_output_args(self.loop_params, resop._args)
         self._impl_write_jf_descr(self.loop_params, resop)
-        self.b_current.end_with_return(self.loop_params.param_frame.as_rvalue ())
+        self.b_current.end_with_return(
+            self.loop_params.param_frame.as_rvalue ())
 
     def emit_label(self, resop):
         print(resop)
@@ -613,7 +630,8 @@
         b_within_failure_fn = pp.failure_fn.new_block("initial")
         self.b_current = b_within_failure_fn
         self._impl_write_jf_descr(pp.failure_params, resop)
-        self.b_current.end_with_return(pp.failure_params.param_frame.as_rvalue ())
+        self.b_current.end_with_return(
+            pp.failure_params.param_frame.as_rvalue ())
         rd_locs = []
         for idx, arg in enumerate(resop._fail_args):
             rd_locs.append(idx * self.sizeof_signed)
@@ -629,7 +647,7 @@
 
     def emit_guard_true(self, resop):
         self._impl_bool_guard(resop, r_int(1))
-        
+
     def emit_guard_false(self, resop):
         self._impl_bool_guard(resop, r_int(0))
 
@@ -922,7 +940,6 @@
 
         self.b_current.add_assignment(
             field_lvalue,
-            
             # ... = ARG1,
             self.ctxt.new_cast(self.expr_to_rvalue(resop._arg1),
                                t_field))
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -96,274 +96,284 @@
 
         # Entrypoints:
         for returntype, name, paramtypes in [
-                (self.GCC_JIT_CONTEXT_P,
-                 'gcc_jit_context_acquire', []),
+            (self.GCC_JIT_CONTEXT_P,
+             'gcc_jit_context_acquire', []),
 
-                (lltype.Void,
-                 'gcc_jit_context_release', [self.GCC_JIT_CONTEXT_P]),
+            (lltype.Void,
+             'gcc_jit_context_release', [self.GCC_JIT_CONTEXT_P]),
 
-                (lltype.Void,
-                 'gcc_jit_context_set_int_option', [self.GCC_JIT_CONTEXT_P,
-                                                    INT, # FIXME: enum gcc_jit_int_option opt,
-                                                    INT]),
-                (lltype.Void,
-                 'gcc_jit_context_set_bool_option', [self.GCC_JIT_CONTEXT_P,
-                                                     INT, # FIXME: enum gcc_jit_bool_option opt,
-                                                     INT]),
+            (lltype.Void,
+             'gcc_jit_context_set_int_option', [self.GCC_JIT_CONTEXT_P,
+                                                # FIXME:
+                                                #   enum gcc_jit_int_option:
+                                                INT,
+                                                INT]),
+            (lltype.Void,
+             'gcc_jit_context_set_bool_option', [self.GCC_JIT_CONTEXT_P,
+                                                 # FIXME:
+                                                 # enum gcc_jit_bool_option:
+                                                 INT,
+                                                 INT]),
 
-                (self.GCC_JIT_RESULT_P,
-                 'gcc_jit_context_compile', [self.GCC_JIT_CONTEXT_P]),
+            (self.GCC_JIT_RESULT_P,
+             'gcc_jit_context_compile', [self.GCC_JIT_CONTEXT_P]),
 
 
-                (lltype.Void,
-                 'gcc_jit_context_dump_to_file', [self.GCC_JIT_CONTEXT_P,
-                                                  CCHARP,
-                                                  INT]),
-
-                (CCHARP,
-                 'gcc_jit_context_get_last_error', [self.GCC_JIT_CONTEXT_P]),
-
-                (VOIDP,
-                 'gcc_jit_result_get_code', [self.GCC_JIT_RESULT_P,
-                                             CCHARP]),
-
-                (VOIDP,
-                 'gcc_jit_result_get_global', [self.GCC_JIT_RESULT_P,
-                                               CCHARP]),
-
-                (lltype.Void,
-                 'gcc_jit_result_release', [self.GCC_JIT_RESULT_P]),
-
-                (CCHARP,
-                 'gcc_jit_object_get_debug_string', [self.GCC_JIT_OBJECT_P]),
-
-                ############################################################
-                # Types
-                ############################################################
-                (self.GCC_JIT_TYPE_P,
-                 'gcc_jit_context_get_type', [self.GCC_JIT_CONTEXT_P,
+            (lltype.Void,
+             'gcc_jit_context_dump_to_file', [self.GCC_JIT_CONTEXT_P,
+                                              CCHARP,
                                               INT]),
 
-                (self.GCC_JIT_TYPE_P,
-                 'gcc_jit_context_get_int_type', [self.GCC_JIT_CONTEXT_P,
-                                                  INT,
-                                                  INT]),
+            (CCHARP,
+             'gcc_jit_context_get_last_error', [self.GCC_JIT_CONTEXT_P]),
 
-                (self.GCC_JIT_TYPE_P,
-                 'gcc_jit_type_get_pointer', [self.GCC_JIT_TYPE_P]),
+            (VOIDP,
+             'gcc_jit_result_get_code', [self.GCC_JIT_RESULT_P,
+                                         CCHARP]),
 
-                (self.GCC_JIT_FIELD_P,
-                 'gcc_jit_context_new_field', [self.GCC_JIT_CONTEXT_P,
+            (VOIDP,
+             'gcc_jit_result_get_global', [self.GCC_JIT_RESULT_P,
+                                           CCHARP]),
+
+            (lltype.Void,
+             'gcc_jit_result_release', [self.GCC_JIT_RESULT_P]),
+
+            (CCHARP,
+             'gcc_jit_object_get_debug_string', [self.GCC_JIT_OBJECT_P]),
+
+            ############################################################
+            # Types
+            ############################################################
+            (self.GCC_JIT_TYPE_P,
+             'gcc_jit_context_get_type', [self.GCC_JIT_CONTEXT_P,
+                                              INT]),
+
+            (self.GCC_JIT_TYPE_P,
+             'gcc_jit_context_get_int_type', [self.GCC_JIT_CONTEXT_P,
+                                              INT,
+                                              INT]),
+
+            (self.GCC_JIT_TYPE_P,
+             'gcc_jit_type_get_pointer', [self.GCC_JIT_TYPE_P]),
+
+            (self.GCC_JIT_FIELD_P,
+             'gcc_jit_context_new_field', [self.GCC_JIT_CONTEXT_P,
+                                           self.GCC_JIT_LOCATION_P,
+                                           self.GCC_JIT_TYPE_P,
+                                           CCHARP]),
+            (self.GCC_JIT_STRUCT_P,
+             'gcc_jit_context_new_struct_type', [self.GCC_JIT_CONTEXT_P,
+                                                 self.GCC_JIT_LOCATION_P,
+                                                 CCHARP,
+                                                 INT,
+                                                 self.FIELD_P_P]),
+
+            (self.GCC_JIT_STRUCT_P,
+             'gcc_jit_context_new_opaque_struct', [self.GCC_JIT_CONTEXT_P,
+                                                   self.GCC_JIT_LOCATION_P,
+                                                   CCHARP]),
+            (self.GCC_JIT_TYPE_P,
+             'gcc_jit_struct_as_type', [self.GCC_JIT_STRUCT_P]),
+
+            (lltype.Void,
+             'gcc_jit_struct_set_fields', [self.GCC_JIT_STRUCT_P,
+                                           self.GCC_JIT_LOCATION_P,
+                                           INT,
+                                           self.FIELD_P_P]),
+
+            (self.GCC_JIT_TYPE_P,
+             'gcc_jit_context_new_union_type', [self.GCC_JIT_CONTEXT_P,
+                                                self.GCC_JIT_LOCATION_P,
+                                                CCHARP,
+                                                INT,
+                                                self.FIELD_P_P]),
+
+            (self.GCC_JIT_TYPE_P,
+             'gcc_jit_context_new_function_ptr_type', [self.GCC_JIT_CONTEXT_P,
+                                                       self.GCC_JIT_LOCATION_P,
+                                                       self.GCC_JIT_TYPE_P,
+                                                       INT,
+                                                       self.TYPE_P_P,
+                                                       INT]),
+
+            ############################################################
+            # Constructing functions.
+            ############################################################
+            (self.GCC_JIT_PARAM_P,
+             'gcc_jit_context_new_param', [self.GCC_JIT_CONTEXT_P,
+                                           self.GCC_JIT_LOCATION_P,
+                                           self.GCC_JIT_TYPE_P,
+                                           CCHARP]),
+            (self.GCC_JIT_LVALUE_P,
+             'gcc_jit_param_as_lvalue', [self.GCC_JIT_PARAM_P]),
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_param_as_rvalue', [self.GCC_JIT_PARAM_P]),
+
+            (self.GCC_JIT_FUNCTION_P,
+             'gcc_jit_context_new_function', [self.GCC_JIT_CONTEXT_P,
+                                              self.GCC_JIT_LOCATION_P,
+                                              # FIXME:
+                                              #   enum gcc_jit_function_kind:
+                                              INT,
+                                              self.GCC_JIT_TYPE_P,
+                                              CCHARP,
+                                              INT,
+                                              self.PARAM_P_P,
+                                              INT]),
+
+            (self.GCC_JIT_OBJECT_P,
+             'gcc_jit_function_as_object', [self.GCC_JIT_FUNCTION_P]),
+
+            (self.GCC_JIT_LVALUE_P,
+             'gcc_jit_function_new_local', [self.GCC_JIT_FUNCTION_P,
+                                            self.GCC_JIT_LOCATION_P,
+                                            self.GCC_JIT_TYPE_P,
+                                            CCHARP]),
+
+            (self.GCC_JIT_BLOCK_P,
+             'gcc_jit_function_new_block', [self.GCC_JIT_FUNCTION_P,
+                                            CCHARP]),
+
+            ############################################################
+            # lvalues, rvalues and expressions.
+            ############################################################
+            (self.GCC_JIT_LVALUE_P,
+             'gcc_jit_context_new_global', [self.GCC_JIT_CONTEXT_P,
+                                            self.GCC_JIT_LOCATION_P,
+                                            # FIXME enum gcc_jit_global_kind:
+                                            INT,
+                                            self.GCC_JIT_TYPE_P,
+                                            CCHARP]),
+
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_lvalue_as_rvalue', [self.GCC_JIT_LVALUE_P]),
+
+            (self.GCC_JIT_TYPE_P,
+             'gcc_jit_rvalue_get_type', [self.GCC_JIT_RVALUE_P]),
+
+            # Integer constants.
+            (self.GCC_JIT_RVALUE_P,
+            'gcc_jit_context_new_rvalue_from_int', [self.GCC_JIT_CONTEXT_P,
+                                                    self.GCC_JIT_TYPE_P,
+                                                    INT]),
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_new_rvalue_from_long', [self.GCC_JIT_CONTEXT_P,
+                                                      self.GCC_JIT_TYPE_P,
+                                                      LONG]),
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_zero', [self.GCC_JIT_CONTEXT_P,
+                                      self.GCC_JIT_TYPE_P]),
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_one', [self.GCC_JIT_CONTEXT_P,
+                                     self.GCC_JIT_TYPE_P]),
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_new_rvalue_from_double', [self.GCC_JIT_CONTEXT_P,
+                                                        self.GCC_JIT_TYPE_P,
+                                                        DOUBLE]),
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_new_rvalue_from_ptr', [self.GCC_JIT_CONTEXT_P,
+                                                     self.GCC_JIT_TYPE_P,
+                                                     VOIDP]),
+
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_null', [self.GCC_JIT_CONTEXT_P,
+                                      self.GCC_JIT_TYPE_P]),
+
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_new_unary_op', [self.GCC_JIT_CONTEXT_P,
+                                              self.GCC_JIT_LOCATION_P,
+                                              # FIXME enum gcc_jit_unary_op:
+                                              INT,
+                                              self.GCC_JIT_TYPE_P,
+                                              self.GCC_JIT_RVALUE_P]),
+
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_new_binary_op', [self.GCC_JIT_CONTEXT_P,
                                                self.GCC_JIT_LOCATION_P,
+                                               # FIXME enum gcc_jit_binary_op:
+                                               INT,
                                                self.GCC_JIT_TYPE_P,
-                                               CCHARP]),
-                (self.GCC_JIT_STRUCT_P,
-                 'gcc_jit_context_new_struct_type', [self.GCC_JIT_CONTEXT_P,
+                                               self.GCC_JIT_RVALUE_P,
+                                               self.GCC_JIT_RVALUE_P]),
+
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_new_comparison', [self.GCC_JIT_CONTEXT_P,
+                                                self.GCC_JIT_LOCATION_P,
+                                                # FIXME enum gcc_jit_comparison:
+                                                INT,
+                                                self.GCC_JIT_RVALUE_P,
+                                                self.GCC_JIT_RVALUE_P]),
+
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_new_call', [self.GCC_JIT_CONTEXT_P,
+                                          self.GCC_JIT_LOCATION_P,
+                                          self.GCC_JIT_FUNCTION_P,
+                                          INT,
+                                          self.RVALUE_P_P]),
+
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_new_call_through_ptr',[self.GCC_JIT_CONTEXT_P,
                                                      self.GCC_JIT_LOCATION_P,
-                                                     CCHARP,
+                                                     self.GCC_JIT_RVALUE_P,
                                                      INT,
-                                                     self.FIELD_P_P]),
+                                                     self.RVALUE_P_P]),
 
-                (self.GCC_JIT_STRUCT_P,
-                 'gcc_jit_context_new_opaque_struct', [self.GCC_JIT_CONTEXT_P,
-                                                       self.GCC_JIT_LOCATION_P,
-                                                       CCHARP]),
-                (self.GCC_JIT_TYPE_P,
-                 'gcc_jit_struct_as_type', [self.GCC_JIT_STRUCT_P]),
-
-                (lltype.Void,
-                 'gcc_jit_struct_set_fields', [self.GCC_JIT_STRUCT_P,
-                                               self.GCC_JIT_LOCATION_P,
-                                               INT,
-                                               self.FIELD_P_P]),
-
-                (self.GCC_JIT_TYPE_P,
-                 'gcc_jit_context_new_union_type', [self.GCC_JIT_CONTEXT_P,
-                                                    self.GCC_JIT_LOCATION_P,
-                                                    CCHARP,
-                                                    INT,
-                                                    self.FIELD_P_P]),
-
-                (self.GCC_JIT_TYPE_P,
-                 'gcc_jit_context_new_function_ptr_type', [self.GCC_JIT_CONTEXT_P,
-                                                           self.GCC_JIT_LOCATION_P,
-                                                           self.GCC_JIT_TYPE_P,
-                                                           INT,
-                                                           self.TYPE_P_P,
-                                                           INT]),
-
-                ############################################################
-                # Constructing functions.
-                ############################################################
-                (self.GCC_JIT_PARAM_P,
-                 'gcc_jit_context_new_param', [self.GCC_JIT_CONTEXT_P,
-                                               self.GCC_JIT_LOCATION_P,
-                                               self.GCC_JIT_TYPE_P,
-                                               CCHARP]),
-                (self.GCC_JIT_LVALUE_P,
-                 'gcc_jit_param_as_lvalue', [self.GCC_JIT_PARAM_P]),
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_param_as_rvalue', [self.GCC_JIT_PARAM_P]),
-
-                (self.GCC_JIT_FUNCTION_P,
-                 'gcc_jit_context_new_function', [self.GCC_JIT_CONTEXT_P,
-                                                  self.GCC_JIT_LOCATION_P,
-                                                  INT, # enum gcc_jit_function_kind kind,
-                                                  self.GCC_JIT_TYPE_P,
-                                                  CCHARP,
-                                                  INT,
-                                                  self.PARAM_P_P,
-                                                  INT]),
-
-                (self.GCC_JIT_OBJECT_P,
-                 'gcc_jit_function_as_object', [self.GCC_JIT_FUNCTION_P]),
-
-                (self.GCC_JIT_LVALUE_P,
-                 'gcc_jit_function_new_local', [self.GCC_JIT_FUNCTION_P,
-                                                self.GCC_JIT_LOCATION_P,
-                                                self.GCC_JIT_TYPE_P,
-                                                CCHARP]),
-
-                (self.GCC_JIT_BLOCK_P,
-                 'gcc_jit_function_new_block', [self.GCC_JIT_FUNCTION_P,
-                                                CCHARP]),
-
-                ############################################################
-                # lvalues, rvalues and expressions.
-                ############################################################
-                (self.GCC_JIT_LVALUE_P,
-                 'gcc_jit_context_new_global', [self.GCC_JIT_CONTEXT_P,
-                                                self.GCC_JIT_LOCATION_P,
-                                                INT, # enum gcc_jit_global_kind
-                                                self.GCC_JIT_TYPE_P,
-                                                CCHARP]),
-
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_lvalue_as_rvalue', [self.GCC_JIT_LVALUE_P]),
-
-                (self.GCC_JIT_TYPE_P,
-                 'gcc_jit_rvalue_get_type', [self.GCC_JIT_RVALUE_P]),
-
-                # Integer constants.
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_new_rvalue_from_int', [self.GCC_JIT_CONTEXT_P,
-                                                         self.GCC_JIT_TYPE_P,
-                                                         INT]),
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_new_rvalue_from_long', [self.GCC_JIT_CONTEXT_P,
-                                                          self.GCC_JIT_TYPE_P,
-                                                          LONG]),
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_zero', [self.GCC_JIT_CONTEXT_P,
-                                          self.GCC_JIT_TYPE_P]),
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_one', [self.GCC_JIT_CONTEXT_P,
-                                         self.GCC_JIT_TYPE_P]),
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_new_rvalue_from_double', [self.GCC_JIT_CONTEXT_P,
-                                                            self.GCC_JIT_TYPE_P,
-                                                            DOUBLE]),
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_new_rvalue_from_ptr', [self.GCC_JIT_CONTEXT_P,
-                                                         self.GCC_JIT_TYPE_P,
-                                                         VOIDP]),
-
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_null', [self.GCC_JIT_CONTEXT_P,
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_context_new_cast', [self.GCC_JIT_CONTEXT_P,
+                                          self.GCC_JIT_LOCATION_P,
+                                          self.GCC_JIT_RVALUE_P,
                                           self.GCC_JIT_TYPE_P]),
 
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_new_unary_op', [self.GCC_JIT_CONTEXT_P,
+            (self.GCC_JIT_LVALUE_P,
+             'gcc_jit_context_new_array_access', [self.GCC_JIT_CONTEXT_P,
                                                   self.GCC_JIT_LOCATION_P,
-                                                  INT, # enum gcc_jit_unary_op op,
-                                                  self.GCC_JIT_TYPE_P,
+                                                  self.GCC_JIT_RVALUE_P,
                                                   self.GCC_JIT_RVALUE_P]),
 
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_new_binary_op', [self.GCC_JIT_CONTEXT_P,
-                                                   self.GCC_JIT_LOCATION_P,
-                                                   INT, # enum gcc_jit_binary_op op,
-                                                   self.GCC_JIT_TYPE_P,
-                                                   self.GCC_JIT_RVALUE_P,
-                                                   self.GCC_JIT_RVALUE_P]),
+            (self.GCC_JIT_LVALUE_P,
+             'gcc_jit_lvalue_access_field', [self.GCC_JIT_LVALUE_P,
+                                             self.GCC_JIT_LOCATION_P,
+                                             self.GCC_JIT_FIELD_P]),
 
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_new_comparison', [self.GCC_JIT_CONTEXT_P,
+            (self.GCC_JIT_LVALUE_P,
+             'gcc_jit_rvalue_dereference_field', [self.GCC_JIT_RVALUE_P,
+                                                  self.GCC_JIT_LOCATION_P,
+                                                  self.GCC_JIT_FIELD_P]),
+
+            (self.GCC_JIT_LVALUE_P,
+             'gcc_jit_rvalue_dereference', [self.GCC_JIT_RVALUE_P,
+                                            self.GCC_JIT_LOCATION_P]),
+
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_lvalue_get_address', [self.GCC_JIT_LVALUE_P,
+                                            self.GCC_JIT_LOCATION_P]),
+
+            ############################################################
+            # Statement-creation.
+            ############################################################
+            (lltype.Void,
+             'gcc_jit_block_add_assignment', [self.GCC_JIT_BLOCK_P,
+                                              self.GCC_JIT_LOCATION_P,
+                                              self.GCC_JIT_LVALUE_P,
+                                              self.GCC_JIT_RVALUE_P]),
+            (lltype.Void,
+             'gcc_jit_block_add_comment', [self.GCC_JIT_BLOCK_P,
+                                           self.GCC_JIT_LOCATION_P,
+                                           CCHARP]),
+            (lltype.Void,
+             'gcc_jit_block_end_with_conditional', [self.GCC_JIT_BLOCK_P,
                                                     self.GCC_JIT_LOCATION_P,
-                                                    INT, # enum gcc_jit_comparison op,
                                                     self.GCC_JIT_RVALUE_P,
-                                                    self.GCC_JIT_RVALUE_P]),
-
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_new_call', [self.GCC_JIT_CONTEXT_P,
-                                              self.GCC_JIT_LOCATION_P,
-                                              self.GCC_JIT_FUNCTION_P,
-                                              INT,
-                                              self.RVALUE_P_P]),
-
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_new_call_through_ptr',[self.GCC_JIT_CONTEXT_P,
-                                                         self.GCC_JIT_LOCATION_P,
-                                                         self.GCC_JIT_RVALUE_P,
-                                                         INT,
-                                                         self.RVALUE_P_P]),
-
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_context_new_cast', [self.GCC_JIT_CONTEXT_P,
-                                              self.GCC_JIT_LOCATION_P,
-                                              self.GCC_JIT_RVALUE_P,
-                                              self.GCC_JIT_TYPE_P]),
-
-                (self.GCC_JIT_LVALUE_P,
-                 'gcc_jit_context_new_array_access', [self.GCC_JIT_CONTEXT_P,
-                                                      self.GCC_JIT_LOCATION_P,
-                                                      self.GCC_JIT_RVALUE_P,
-                                                      self.GCC_JIT_RVALUE_P]),
-
-                (self.GCC_JIT_LVALUE_P,
-                 'gcc_jit_lvalue_access_field', [self.GCC_JIT_LVALUE_P,
-                                                 self.GCC_JIT_LOCATION_P,
-                                                 self.GCC_JIT_FIELD_P]),
-
-                (self.GCC_JIT_LVALUE_P,
-                 'gcc_jit_rvalue_dereference_field', [self.GCC_JIT_RVALUE_P,
-                                                      self.GCC_JIT_LOCATION_P,
-                                                      self.GCC_JIT_FIELD_P]),
-
-                (self.GCC_JIT_LVALUE_P,
-                 'gcc_jit_rvalue_dereference', [self.GCC_JIT_RVALUE_P,
-                                                self.GCC_JIT_LOCATION_P]),
-
-                (self.GCC_JIT_RVALUE_P,
-                 'gcc_jit_lvalue_get_address', [self.GCC_JIT_LVALUE_P,
-                                                self.GCC_JIT_LOCATION_P]),
-
-                ############################################################
-                # Statement-creation.
-                ############################################################
-                (lltype.Void,
-                 'gcc_jit_block_add_assignment', [self.GCC_JIT_BLOCK_P,
-                                                  self.GCC_JIT_LOCATION_P,
-                                                  self.GCC_JIT_LVALUE_P,
-                                                  self.GCC_JIT_RVALUE_P]),
-                (lltype.Void,
-                 'gcc_jit_block_add_comment', [self.GCC_JIT_BLOCK_P,
+                                                    self.GCC_JIT_BLOCK_P,
+                                                    self.GCC_JIT_BLOCK_P]),
+            (lltype.Void,
+             'gcc_jit_block_end_with_jump', [self.GCC_JIT_BLOCK_P,
+                                             self.GCC_JIT_LOCATION_P,
+                                             self.GCC_JIT_BLOCK_P]),
+            (lltype.Void,
+             'gcc_jit_block_end_with_return', [self.GCC_JIT_BLOCK_P,
                                                self.GCC_JIT_LOCATION_P,
-                                               CCHARP]),
-                (lltype.Void,
-                 'gcc_jit_block_end_with_conditional', [self.GCC_JIT_BLOCK_P,
-                                                        self.GCC_JIT_LOCATION_P,
-                                                        self.GCC_JIT_RVALUE_P,
-                                                        self.GCC_JIT_BLOCK_P,
-                                                        self.GCC_JIT_BLOCK_P]),
-                (lltype.Void,
-                 'gcc_jit_block_end_with_jump', [self.GCC_JIT_BLOCK_P,
-                                                 self.GCC_JIT_LOCATION_P,
-                                                 self.GCC_JIT_BLOCK_P]),
-                (lltype.Void,
-                 'gcc_jit_block_end_with_return', [self.GCC_JIT_BLOCK_P,
-                                                   self.GCC_JIT_LOCATION_P,
-                                                   self.GCC_JIT_RVALUE_P]),
+                                               self.GCC_JIT_RVALUE_P]),
         ]:
             self.add_entrypoint(returntype, name, paramtypes)
 
@@ -545,13 +555,13 @@
         lltype.free(field_array, flavor='raw')
         free_charp(name_charp)
         return Struct(self.lib, self, inner_struct)
-    
+
     def new_opaque_struct(self, name):
         name_charp = str2charp(name)
-        inner_struct = (
-            self.lib.gcc_jit_context_new_opaque_struct(self.inner_ctxt,
-                                                       self.lib.null_location_ptr,
-                                                       name_charp))
+        inner_struct = self.lib.gcc_jit_context_new_opaque_struct(
+            self.inner_ctxt,
+            self.lib.null_location_ptr,
+            name_charp)
         free_charp(name_charp)
         return Struct(self.lib, self, inner_struct)
 
@@ -579,12 +589,13 @@
         for i in range(len(param_types)):
             raw_type_array[i] = param_types[i].inner_type
 
-        type_ = self.lib.gcc_jit_context_new_function_ptr_type(self.inner_ctxt,
-                                                               self.lib.null_location_ptr,
-                                                               returntype.inner_type,
-                                                               r_int(len(param_types)),
-                                                               raw_type_array,
-                                                               is_variadic)
+        type_ = self.lib.gcc_jit_context_new_function_ptr_type(
+            self.inner_ctxt,
+            self.lib.null_location_ptr,
+            returntype.inner_type,
+            r_int(len(param_types)),
+            raw_type_array,
+            is_variadic)
         lltype.free(raw_type_array, flavor='raw')
 
         return Type(self.lib, self, type_)
@@ -592,30 +603,34 @@
     def new_rvalue_from_int(self, type_, llvalue):
         return RValue(self.lib,
                       self,
-                      self.lib.gcc_jit_context_new_rvalue_from_int(self.inner_ctxt,
-                                                                   type_.inner_type,
-                                                                   llvalue))
+                      self.lib.gcc_jit_context_new_rvalue_from_int(
+                          self.inner_ctxt,
+                          type_.inner_type,
+                          llvalue))
 
     def new_rvalue_from_long(self, type_, llvalue):
         return RValue(self.lib,
                       self,
-                      self.lib.gcc_jit_context_new_rvalue_from_long(self.inner_ctxt,
-                                                                    type_.inner_type,
-                                                                    llvalue))
+                      self.lib.gcc_jit_context_new_rvalue_from_long(
+                          self.inner_ctxt,
+                          type_.inner_type,
+                          llvalue))
 
     def new_rvalue_from_double(self, type_, llvalue):
         return RValue(self.lib,
                       self,
-                      self.lib.gcc_jit_context_new_rvalue_from_double(self.inner_ctxt,
-                                                                      type_.inner_type,
-                                                                      llvalue))
+                      self.lib.gcc_jit_context_new_rvalue_from_double(
+                          self.inner_ctxt,
+                          type_.inner_type,
+                          llvalue))
 
     def new_rvalue_from_ptr(self, type_, llvalue):
         return RValue(self.lib,
                       self,
-                      self.lib.gcc_jit_context_new_rvalue_from_ptr(self.inner_ctxt,
-                                                                   type_.inner_type,
-                                                                   llvalue))
+                      self.lib.gcc_jit_context_new_rvalue_from_ptr(
+                          self.inner_ctxt,
+                          type_.inner_type,
+                          llvalue))
 
 
     def null(self, pointer_type):
@@ -627,28 +642,31 @@
     def new_unary_op(self, op, type_, rvalue):
         return RValue(self.lib,
                       self,
-                      self.lib.gcc_jit_context_new_unary_op(self.inner_ctxt,
-                                                            self.lib.null_location_ptr,
-                                                            op,
-                                                            type_.inner_type,
-                                                             rvalue.inner_rvalue))
+                      self.lib.gcc_jit_context_new_unary_op(
+                          self.inner_ctxt,
+                          self.lib.null_location_ptr,
+                          op,
+                          type_.inner_type,
+                          rvalue.inner_rvalue))
 
     def new_binary_op(self, op, type_, a, b):
         return RValue(self.lib,
                       self,
-                      self.lib.gcc_jit_context_new_binary_op(self.inner_ctxt,
-                                                             self.lib.null_location_ptr,
-                                                             op,
-                                                             type_.inner_type,
-                                                             a.inner_rvalue, b.inner_rvalue))
+                      self.lib.gcc_jit_context_new_binary_op(
+                          self.inner_ctxt,
+                          self.lib.null_location_ptr,
+                          op,
+                          type_.inner_type,
+                          a.inner_rvalue, b.inner_rvalue))
 
     def new_comparison(self, op, a, b):
         return RValue(self.lib,
                       self,
-                      self.lib.gcc_jit_context_new_comparison(self.inner_ctxt,
-                                                              self.lib.null_location_ptr,
-                                                              op,
-                                                              a.inner_rvalue, b.inner_rvalue))
+                      self.lib.gcc_jit_context_new_comparison(
+                          self.inner_ctxt,
+                          self.lib.null_location_ptr,
+                          op,
+                          a.inner_rvalue, b.inner_rvalue))
 
     def new_call(self, fn, args):
         raw_arg_array = lltype.malloc(self.lib.RVALUE_P_P.TO,
@@ -670,11 +688,12 @@
                                       flavor='raw') # of maybe gc?
         for i in range(len(args)):
             raw_arg_array[i] = args[i].inner_rvalue
-        rvalue = self.lib.gcc_jit_context_new_call_through_ptr(self.inner_ctxt,
-                                                               self.lib.null_location_ptr,
-                                                               fn_ptr.inner_rvalue,
-                                                               r_int(len(args)),
-                                                               raw_arg_array)
+        rvalue = self.lib.gcc_jit_context_new_call_through_ptr(
+            self.inner_ctxt,
+            self.lib.null_location_ptr,
+            fn_ptr.inner_rvalue,
+            r_int(len(args)),
+            raw_arg_array)
         lltype.free(raw_arg_array, flavor='raw')
         return RValue(self.lib, self, rvalue)
 
@@ -720,11 +739,12 @@
 
     def new_cast(self, rvalue, type_):
         return RValue(self.lib,
-                      self, 
-                      self.lib.gcc_jit_context_new_cast(self.inner_ctxt,
-                                                        self.lib.null_location_ptr,
-                                                        rvalue.inner_rvalue,
-                                                        type_.inner_type))
+                      self,
+                      self.lib.gcc_jit_context_new_cast(
+                          self.inner_ctxt,
+                          self.lib.null_location_ptr,
+                          rvalue.inner_rvalue,
+                          type_.inner_type))
 
     def new_array_access(self, ptr, index):
         return LValue(self.lib,
@@ -737,7 +757,8 @@
 
 class LibgccjitError(Exception):
     def __init__(self, ctxt):
-        self.msg = charp2str(ctxt.lib.gcc_jit_context_get_last_error (ctxt.inner_ctxt))
+        self.msg = charp2str(ctxt.lib.gcc_jit_context_get_last_error (
+            ctxt.inner_ctxt))
         #print('self.msg: %r' % self.msg)
 
     def __str__(self):
@@ -757,7 +778,7 @@
 
     def get_pointer(self):
         return Type(self.lib,
-                    self, 
+                    self,
                     self.lib.gcc_jit_type_get_pointer(self.inner_type))
 
 class Field(Object):
@@ -772,7 +793,7 @@
 
     def as_type(self):
         return Type(self.lib,
-                    self, 
+                    self,
                     self.lib.gcc_jit_struct_as_type(self.inner_struct))
 
 
@@ -801,9 +822,10 @@
     def dereference_field(self, field):
         return LValue(self.lib,
                       self,
-                      self.lib.gcc_jit_rvalue_dereference_field(self.inner_rvalue,
-                                                                self.lib.null_location_ptr,
-                                                                field.inner_field))
+                      self.lib.gcc_jit_rvalue_dereference_field(
+                          self.inner_rvalue,
+                          self.lib.null_location_ptr,
+                          field.inner_field))
 
     def dereference(self):
         return LValue(self.lib,
@@ -825,9 +847,10 @@
     def access_field(self, field):
         return LValue(self.lib,
                       self,
-                      self.lib.gcc_jit_lvalue_access_field (self.inner_lvalue,
-                                                            self.lib.null_location_ptr,
-                                                            field.inner_field))
+                      self.lib.gcc_jit_lvalue_access_field (
+                          self.inner_lvalue,
+                          self.lib.null_location_ptr,
+                          field.inner_field))
 
     def get_address(self):
         return RValue(self.lib,
diff --git a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
@@ -34,7 +34,8 @@
     assert xf() == 3
 """
 
-from rpython.jit.backend.libgccjit.rffi_bindings import make_eci, Library, make_param_array, Context
+from rpython.jit.backend.libgccjit.rffi_bindings import (
+    make_eci, Library, make_param_array, Context)
 
 def test_compile_empty_context():
     eci = make_eci()
@@ -46,7 +47,7 @@
         result = lib.gcc_jit_context_compile(ctxt)
         lib.gcc_jit_context_release(ctxt)
         lib.gcc_jit_result_release(result)
-        
+
     f1 = compile_c(f, [], backendopt=False)
     f1 ()
     #assert False # to see stderr
@@ -62,21 +63,26 @@
     def f():
         ctxt = lib.gcc_jit_context_acquire()
 
-        lib.gcc_jit_context_set_bool_option(ctxt,
-                                            lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
-                                            r_int(1))
-        lib.gcc_jit_context_set_int_option(ctxt,
-                                           lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
-                                           r_int(3))
-        lib.gcc_jit_context_set_bool_option(ctxt,
-                                            lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
-                                            r_int(1))
-        lib.gcc_jit_context_set_bool_option(ctxt,
-                                            lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
-                                            r_int(1))
-        lib.gcc_jit_context_set_bool_option(ctxt,
-                                            lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
-                                            r_int(1))
+        lib.gcc_jit_context_set_bool_option(
+            ctxt,
+            lib.GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE,
+            r_int(1))
+        lib.gcc_jit_context_set_int_option(
+            ctxt,
+            lib.GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
+            r_int(3))
+        lib.gcc_jit_context_set_bool_option(
+            ctxt,
+            lib.GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
+            r_int(1))
+        lib.gcc_jit_context_set_bool_option(
+            ctxt,
+            lib.GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
+            r_int(1))
+        lib.gcc_jit_context_set_bool_option(
+            ctxt,
+            lib.GCC_JIT_BOOL_OPTION_DUMP_GENERATED_CODE,
+            r_int(1))
         t_int = lib.gcc_jit_context_get_type(ctxt, lib.GCC_JIT_TYPE_INT)
         param_name = str2charp("input")
         param = lib.gcc_jit_context_new_param(ctxt,
@@ -87,7 +93,7 @@
 
         # FIXME: how to build an array of params at this level?
         # see liststr2charpp in rffi.py
-        
+
         fn_name = str2charp("add_one_to")
         param_array = make_param_array(lib, [param])
         fn = lib.gcc_jit_context_new_function(ctxt,
@@ -109,12 +115,13 @@
         b_initial = lib.gcc_jit_function_new_block(fn, block_name)
         free_charp(block_name)
         c_one = lib.gcc_jit_context_new_rvalue_from_int(ctxt, t_int, r_int(1))
-        op_add = lib.gcc_jit_context_new_binary_op(ctxt,
-                                                   lib.null_location_ptr,
-                                                   lib.GCC_JIT_BINARY_OP_PLUS,
-                                                   t_int,
-                                                   lib.gcc_jit_param_as_rvalue(param),
-                                                   c_one)
+        op_add = lib.gcc_jit_context_new_binary_op(
+            ctxt,
+            lib.null_location_ptr,
+            lib.GCC_JIT_BINARY_OP_PLUS,
+            t_int,
+            lib.gcc_jit_param_as_rvalue(param),
+            c_one)
         lib.gcc_jit_block_add_assignment(b_initial, lib.null_location_ptr,
                                          v_res,
                                          op_add)
@@ -153,7 +160,7 @@
         lib.gcc_jit_result_release(jit_result)
 
         return int(fn_result)
-        
+
     f1 = compile_c(f, [], backendopt=False)
     assert f1() == 42
     assert False # to see stderr
@@ -222,10 +229,10 @@
         jit_result.release()
 
         return int(fn_result)
-        
+
     f1 = compile_c(f, [], backendopt=False)
     assert f1() == 42
     assert False # to see stderr
-    
+
 # TODO: test of an error
 # should turn it into an exception, and capture the error

From noreply at buildbot.pypy.org  Tue Dec 23 17:30:38 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 17:30:38 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Fix test_rffi_bindings.py
Message-ID: <20141223163038.D74041D22DE@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75086:988365232237
Date: 2014-12-23 11:38 -0500
http://bitbucket.org/pypy/pypy/changeset/988365232237/

Log:	Fix test_rffi_bindings.py

diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -31,7 +31,7 @@
 from rpython.rtyper.lltypesystem import lltype
 
 def make_eci():
-    eci = ExternalCompilationInfo(includes=['libgccjit.h'],
+    eci = ExternalCompilationInfo(includes=['stdio.h', 'libgccjit.h'],
                                   include_dirs=[INCLUDE_DIR],
                                   libraries=['gccjit'],
                                   library_dirs=[LIB_DIR])
@@ -144,6 +144,9 @@
             ############################################################
             # Types
             ############################################################
+            (self.GCC_JIT_OBJECT_P,
+             'gcc_jit_type_as_object', [self.GCC_JIT_TYPE_P]),
+
             (self.GCC_JIT_TYPE_P,
              'gcc_jit_context_get_type', [self.GCC_JIT_CONTEXT_P,
                                               INT]),
@@ -161,7 +164,11 @@
                                            self.GCC_JIT_LOCATION_P,
                                            self.GCC_JIT_TYPE_P,
                                            CCHARP]),
-            (self.GCC_JIT_STRUCT_P,
+
+            (self.GCC_JIT_OBJECT_P,
+             'gcc_jit_field_as_object', [self.GCC_JIT_FIELD_P]),
+
+           (self.GCC_JIT_STRUCT_P,
              'gcc_jit_context_new_struct_type', [self.GCC_JIT_CONTEXT_P,
                                                  self.GCC_JIT_LOCATION_P,
                                                  CCHARP,
@@ -204,6 +211,8 @@
                                            self.GCC_JIT_LOCATION_P,
                                            self.GCC_JIT_TYPE_P,
                                            CCHARP]),
+            (self.GCC_JIT_OBJECT_P,
+             'gcc_jit_param_as_object', [self.GCC_JIT_PARAM_P]),
             (self.GCC_JIT_LVALUE_P,
              'gcc_jit_param_as_lvalue', [self.GCC_JIT_PARAM_P]),
             (self.GCC_JIT_RVALUE_P,
@@ -233,6 +242,8 @@
             (self.GCC_JIT_BLOCK_P,
              'gcc_jit_function_new_block', [self.GCC_JIT_FUNCTION_P,
                                             CCHARP]),
+            (self.GCC_JIT_OBJECT_P,
+             'gcc_jit_block_as_object', [self.GCC_JIT_BLOCK_P]),
 
             ############################################################
             # lvalues, rvalues and expressions.
@@ -245,9 +256,15 @@
                                             self.GCC_JIT_TYPE_P,
                                             CCHARP]),
 
+            (self.GCC_JIT_OBJECT_P,
+             'gcc_jit_lvalue_as_object', [self.GCC_JIT_LVALUE_P]),
+
             (self.GCC_JIT_RVALUE_P,
              'gcc_jit_lvalue_as_rvalue', [self.GCC_JIT_LVALUE_P]),
 
+            (self.GCC_JIT_OBJECT_P,
+             'gcc_jit_rvalue_as_object', [self.GCC_JIT_RVALUE_P]),
+
             (self.GCC_JIT_TYPE_P,
              'gcc_jit_rvalue_get_type', [self.GCC_JIT_RVALUE_P]),
 
@@ -773,7 +790,8 @@
 
 class Type(Object):
     def __init__(self, lib, ctxt, inner_type):
-        Object.__init__(self, lib, ctxt, inner_type)
+        Object.__init__(self, lib, ctxt,
+                        lib.gcc_jit_type_as_object(inner_type))
         self.inner_type = inner_type
 
     def get_pointer(self):
@@ -783,12 +801,15 @@
 
 class Field(Object):
     def __init__(self, lib, ctxt, inner_field):
-        Object.__init__(self, lib, ctxt, inner_field)
+        Object.__init__(self, lib, ctxt,
+                        lib.gcc_jit_field_as_object(inner_field))
         self.inner_field = inner_field
 
 class Struct(Object):
     def __init__(self, lib, ctxt, inner_struct):
-        Object.__init__(self, lib, ctxt, inner_struct)
+        Object.__init__(self, lib, ctxt,
+                        lib.gcc_jit_type_as_object(
+                            lib.gcc_jit_struct_as_type(inner_struct)))
         self.inner_struct = inner_struct
 
     def as_type(self):
@@ -811,7 +832,8 @@
 
 class RValue(Object):
     def __init__(self, lib, ctxt, inner_rvalue):
-        Object.__init__(self, lib, ctxt, inner_rvalue)
+        Object.__init__(self, lib, ctxt,
+                        lib.gcc_jit_rvalue_as_object(inner_rvalue))
         self.inner_rvalue = inner_rvalue
 
     def get_type(self):
@@ -836,7 +858,8 @@
 
 class LValue(Object):
     def __init__(self, lib, ctxt, inner_lvalue):
-        Object.__init__(self, lib, ctxt, inner_lvalue)
+        Object.__init__(self, lib, ctxt,
+                        lib.gcc_jit_lvalue_as_object(inner_lvalue))
         self.inner_lvalue = inner_lvalue
 
     def as_rvalue(self):
@@ -861,7 +884,8 @@
 
 class Param(Object):
     def __init__(self, lib, ctxt, inner_param):
-        Object.__init__(self, lib, ctxt, inner_param)
+        Object.__init__(self, lib, ctxt,
+                        lib.gcc_jit_param_as_object(inner_param))
         self.inner_param = inner_param
 
     def as_rvalue(self):
@@ -871,7 +895,9 @@
 
 class Function(Object):
     def __init__(self, lib, ctxt, inner_function):
-        Object.__init__(self, lib, ctxt, inner_function)
+        Object.__init__(self, lib, ctxt,
+                        lib.gcc_jit_function_as_object(inner_function))
+
         self.inner_function = inner_function
 
     def new_local(self, type_, name):
@@ -896,7 +922,9 @@
 
 class Block(Object):
     def __init__(self, lib, ctxt, inner_block):
-        Object.__init__(self, lib, ctxt, inner_block)
+        Object.__init__(self, lib, ctxt,
+                        lib.gcc_jit_block_as_object(inner_block))
+
         self.inner_block = inner_block
 
     def add_assignment(self, lvalue, rvalue):
diff --git a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/test/test_rffi_bindings.py
@@ -50,7 +50,6 @@
 
     f1 = compile_c(f, [], backendopt=False)
     f1 ()
-    #assert False # to see stderr
 
 def test_compile_add_one_to():
     eci = make_eci()
@@ -163,7 +162,6 @@
 
     f1 = compile_c(f, [], backendopt=False)
     assert f1() == 42
-    assert False # to see stderr
 
 def test_oo_compile_add_one_to():
     eci = make_eci()
@@ -232,7 +230,6 @@
 
     f1 = compile_c(f, [], backendopt=False)
     assert f1() == 42
-    assert False # to see stderr
 
 # TODO: test of an error
 # should turn it into an exception, and capture the error

From noreply at buildbot.pypy.org  Tue Dec 23 17:47:02 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 17:47:02 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_passing_guard_class
 and test_failing_guard_class to pass
Message-ID: <20141223164702.7E86F1C1347@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75087:8b245e7f33c1
Date: 2014-12-23 11:54 -0500
http://bitbucket.org/pypy/pypy/changeset/8b245e7f33c1/

Log:	Get test_passing_guard_class and test_failing_guard_class to pass

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -659,14 +659,10 @@
         self._impl_guard(resop, r_int(1), boolval)
 
     def emit_guard_class(self, resop):
-        print(resop)
-        print(dir(resop))
         vtable_offset = self.cpu.vtable_offset
-        print('vtable_offset: %r' % vtable_offset)
         assert vtable_offset is not None
         lvalue_obj_vtable = self.impl_get_lvalue_at_offset_from_ptr(
             resop._arg0, r_int(vtable_offset), self.t_void_ptr)
-        print('resop._arg1: %r' % resop._arg1)
         boolval = self.ctxt.new_comparison(
             self.lib.GCC_JIT_COMPARISON_EQ,
             lvalue_obj_vtable.as_rvalue(),
@@ -689,6 +685,39 @@
             self.ctxt.null(ptr_rvalue.get_type()))
         self._impl_guard(resop, r_int(1), boolval)
 
+    def emit_guard_nonnull_class(self, resop):
+        # GUARD_NONNULL_CLASS means:
+        #   "check that _arg0 (instance ptr) is non-NULL
+        #    and then (if non-NULL)
+        #       check the ptr's vtable for equality against _arg1
+        #    if both true then guard is true,
+        #    if either fails then guard has failed"
+
+        # Hence we need two conditionals; we can do this
+        # using a logical AND of the two conditionals;
+        # libgccjit should shortcircuit things appropriately.
+
+        ptr_rvalue = self.expr_to_rvalue(resop._arg0)
+        boolval_is_nonnull = self.ctxt.new_comparison(
+            self.lib.GCC_JIT_COMPARISON_NE,
+            ptr_rvalue,
+            self.ctxt.null(ptr_rvalue.get_type()))
+
+        vtable_offset = self.cpu.vtable_offset
+        assert vtable_offset is not None
+        lvalue_obj_vtable = self.impl_get_lvalue_at_offset_from_ptr(
+            resop._arg0, r_int(vtable_offset), self.t_void_ptr)
+        boolval_vtable_equality = self.ctxt.new_comparison(
+            self.lib.GCC_JIT_COMPARISON_EQ,
+            lvalue_obj_vtable.as_rvalue(),
+            self.expr_to_rvalue(resop._arg1))
+
+        self._impl_guard(resop, r_int(1),
+                         self.ctxt.new_binary_op(
+                             self.lib.GCC_JIT_BINARY_OP_LOGICAL_AND,
+                             self.t_bool,
+                             boolval_is_nonnull, boolval_vtable_equality))
+
     def _impl_write_output_args(self, params, args):
         # Write outputs back:
         for idx, arg in enumerate(args):

From noreply at buildbot.pypy.org  Tue Dec 23 18:56:50 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 18:56:50 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Fix a MemoryError issue seen
 running test_int_operations
Message-ID: <20141223175650.575921D24B5@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75088:70555a2aeb22
Date: 2014-12-23 12:37 -0500
http://bitbucket.org/pypy/pypy/changeset/70555a2aeb22/

Log:	Fix a MemoryError issue seen running test_int_operations

	test_int_operations was failing after compiling about 200 loops with
	a MemoryError, unable to fulfil an allocation of 129GB of RAM.

	Fix it by cleaning up the datablockwrapper after each compile.

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -261,6 +261,9 @@
         clt.frame_info.update_frame_depth(baseofs,
                                           max_args)
 
+        self.datablockwrapper.done()      # finish using cpu.asmmemmgr
+        self.datablockwrapper = None
+
         self.ctxt.dump_to_file("/tmp/%s.c" % loopname, r_int(1))
 
         #raise foo
@@ -296,6 +299,10 @@
         self.num_bridges += 1
 
         self.make_function(name, inputargs, operations)
+
+        self.datablockwrapper.done()      # finish using cpu.asmmemmgr
+        self.datablockwrapper = None
+
         self.ctxt.dump_to_file("/tmp/%s.c" % name, r_int(1))
         jit_result = self.ctxt.compile()
         self.ctxt.release()

From noreply at buildbot.pypy.org  Tue Dec 23 18:56:51 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 18:56:51 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_int_operations to
	pass
Message-ID: <20141223175651.7F7FD1D24B5@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75089:d0c24c40d739
Date: 2014-12-23 13:04 -0500
http://bitbucket.org/pypy/pypy/changeset/d0c24c40d739/

Log:	Get test_int_operations to pass

	Implement remaining missing operations needed by test_int_operations
	(UINT_ comparisons, unary INT_ ops).

	FWIW, creates 778 loops.

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -875,7 +875,29 @@
     def emit_int_ge(self, resop):
         self.impl_int_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
 
-    #   "UINT" comparisons:  TODO
+    #   "UINT" comparisons:
+    def impl_uint_cmp(self, resop, gcc_jit_comparison):
+        rval0 = self.expr_to_rvalue(resop._arg0)
+        rval1 = self.expr_to_rvalue(resop._arg1)
+        rval0 = self.ctxt.new_cast(rval0, self.t_UINT)
+        rval1 = self.ctxt.new_cast(rval1, self.t_UINT)
+        lvalres = self.expr_to_lvalue(resop.result)
+        resop_cmp = (
+            self.ctxt.new_cast(
+                self.ctxt.new_comparison(gcc_jit_comparison,
+                                         rval0, rval1),
+                self.t_Signed)
+            )
+        self.b_current.add_assignment(lvalres,
+                                      resop_cmp)
+    def emit_uint_lt(self, resop):
+        self.impl_uint_cmp(resop, self.lib.GCC_JIT_COMPARISON_LT)
+    def emit_uint_le(self, resop):
+        self.impl_uint_cmp(resop, self.lib.GCC_JIT_COMPARISON_LE)
+    def emit_uint_gt(self, resop):
+        self.impl_uint_cmp(resop, self.lib.GCC_JIT_COMPARISON_GT)
+    def emit_uint_ge(self, resop):
+        self.impl_uint_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
 
     #   "FLOAT" comparisons:
     def impl_float_cmp(self, resop, gcc_jit_comparison):
@@ -904,6 +926,36 @@
     def emit_float_ge(self, resop):
         self.impl_float_cmp(resop, self.lib.GCC_JIT_COMPARISON_GE)
 
+    # Unary "INT" operations:
+    def _impl_int_unaryop(self, resop, gcc_jit_unary_op):
+        rvalue = self.expr_to_rvalue(resop._arg0)
+        lvalres = self.expr_to_lvalue(resop.result)
+        unaryop_expr = self.ctxt.new_unary_op(gcc_jit_unary_op,
+                                              self.t_Signed,
+                                              rvalue)
+        self.b_current.add_assignment(lvalres, unaryop_expr)
+
+    def emit_int_is_zero(self, resop):
+        self._impl_int_unaryop(resop, self.lib.GCC_JIT_UNARY_OP_LOGICAL_NEGATE)
+    def emit_int_is_true(self, resop):
+        rvalarg = self.expr_to_rvalue(resop._arg0)
+        lvalres = self.expr_to_lvalue(resop.result)
+        resop_cmp = (
+            self.ctxt.new_cast(
+                self.ctxt.new_comparison(self.lib.GCC_JIT_COMPARISON_NE,
+                                         rvalarg,
+                                         self.ctxt.zero(self.t_Signed)),
+                self.t_Signed)
+            )
+        self.b_current.add_assignment(lvalres,
+                                      resop_cmp)
+    def emit_int_neg(self, resop):
+        self._impl_int_unaryop(resop, self.lib.GCC_JIT_UNARY_OP_MINUS)
+    def emit_int_invert(self, resop):
+        self._impl_int_unaryop(resop, self.lib.GCC_JIT_UNARY_OP_BITWISE_NEGATE)
+
+    #
+
     def impl_get_lvalue_at_offset_from_ptr(self, ptr_expr, ll_offset, t_field):
         ptr = self.expr_to_rvalue(ptr_expr)
 
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -633,6 +633,13 @@
                           type_.inner_type,
                           llvalue))
 
+    def zero(self, numeric_type):
+        return RValue(self.lib,
+                      self,
+                      self.lib.gcc_jit_context_zero(
+                          self.inner_ctxt,
+                          numeric_type.inner_type))
+
     def new_rvalue_from_double(self, type_, llvalue):
         return RValue(self.lib,
                       self,

From noreply at buildbot.pypy.org  Tue Dec 23 18:58:07 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 23 Dec 2014 18:58:07 +0100 (CET)
Subject: [pypy-commit] pypy default: small level as a tag. that way we can
 store a few bits for future use (not
Message-ID: <20141223175807.768501D24B5@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75090:c9019489d422
Date: 2014-12-23 19:56 +0200
http://bitbucket.org/pypy/pypy/changeset/c9019489d422/

Log:	small level as a tag. that way we can store a few bits for future
	use (not used yet)

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -4,7 +4,8 @@
 from rpython.jit.metainterp.optimizeopt.util import args_dict
 from rpython.jit.metainterp.history import Const
 from rpython.jit.metainterp.jitexc import JitException
-from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED, LEVEL_CONSTANT
+from rpython.jit.metainterp.optimizeopt.optimizer import Optimization,\
+     MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation
@@ -137,7 +138,7 @@
                 continue
             value = optimizer.getvalue(op.getarg(0))
             if value in optimizer.opaque_pointers:
-                if value.level < LEVEL_KNOWNCLASS:
+                if value.getlevel() < LEVEL_KNOWNCLASS:
                     continue
                 if op.getopnum() != rop.SETFIELD_GC and op.getopnum() != rop.GETFIELD_GC:
                     continue
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -13,11 +13,16 @@
 from rpython.rlib.debug import debug_print
 from rpython.rlib.objectmodel import specialize
 
+""" The tag field on OptValue has a following meaning:
 
-LEVEL_UNKNOWN    = '\x00'
-LEVEL_NONNULL    = '\x01'
-LEVEL_KNOWNCLASS = '\x02'     # might also mean KNOWNARRAYDESCR, for arrays
-LEVEL_CONSTANT   = '\x03'
+lower two bits are LEVEL
+next 16 bits is the position in the original list, 0 if unknown or a constant
+"""
+
+LEVEL_UNKNOWN    = 0
+LEVEL_NONNULL    = 1
+LEVEL_KNOWNCLASS = 2     # might also mean KNOWNARRAYDESCR, for arrays
+LEVEL_CONSTANT   = 3
 
 MODE_ARRAY   = '\x00'
 MODE_STR     = '\x01'
@@ -41,35 +46,41 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'level')
+    _attrs_ = ('box', '_tag')
 
-    level = LEVEL_UNKNOWN
+    _tag = 0
 
     def __init__(self, box, level=None, known_class=None, intbound=None):
         self.box = box
         if level is not None:
-            self.level = level
+            self._tag = level
 
         if isinstance(box, Const):
             self.make_constant(box)
         # invariant: box is a Const if and only if level == LEVEL_CONSTANT
 
+    def getlevel(self):
+        return self._tag & 0x3
+
+    def setlevel(self, level):
+        self._tag = (self._tag & (~0x3)) | level
+
     def import_from(self, other, optimizer):
-        if self.level == LEVEL_CONSTANT:
-            assert other.level == LEVEL_CONSTANT
+        if self.getlevel() == LEVEL_CONSTANT:
+            assert other.getlevel() == LEVEL_CONSTANT
             assert other.box.same_constant(self.box)
             return
-        assert self.level <= LEVEL_NONNULL
-        if other.level == LEVEL_CONSTANT:
+        assert self.getlevel() <= LEVEL_NONNULL
+        if other.getlevel() == LEVEL_CONSTANT:
             self.make_constant(other.get_key_box())
-        elif other.level == LEVEL_KNOWNCLASS:
+        elif other.getlevel() == LEVEL_KNOWNCLASS:
             self.make_constant_class(other.get_known_class(), None)
         else:
-            if other.level == LEVEL_NONNULL:
+            if other.getlevel() == LEVEL_NONNULL:
                 self.ensure_nonnull()
 
     def make_guards(self, box):
-        if self.level == LEVEL_CONSTANT:
+        if self.getlevel() == LEVEL_CONSTANT:
             op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
             return [op]
         return []
@@ -77,7 +88,7 @@
     def copy_from(self, other_value):
         assert isinstance(other_value, OptValue)
         self.box = other_value.box
-        self.level = other_value.level
+        self._tag = other_value._tag
 
     def force_box(self, optforce):
         return self.box
@@ -105,7 +116,7 @@
         assert 0, "unreachable"
 
     def is_constant(self):
-        return self.level == LEVEL_CONSTANT
+        return self.getlevel() == LEVEL_CONSTANT
 
     def is_null(self):
         if self.is_constant():
@@ -122,7 +133,7 @@
         return self is other
 
     def is_nonnull(self):
-        level = self.level
+        level = self.getlevel()
         if level == LEVEL_NONNULL or level == LEVEL_KNOWNCLASS:
             return True
         elif level == LEVEL_CONSTANT:
@@ -133,8 +144,8 @@
             return False
 
     def ensure_nonnull(self):
-        if self.level < LEVEL_NONNULL:
-            self.level = LEVEL_NONNULL
+        if self.getlevel() < LEVEL_NONNULL:
+            self.setlevel(LEVEL_NONNULL)
 
     def is_virtual(self):
         # Don't check this with 'isinstance(_, VirtualValue)'!
@@ -179,7 +190,7 @@
         """Replace 'self.box' with a Const box."""
         assert isinstance(constbox, Const)
         self.box = constbox
-        self.level = LEVEL_CONSTANT
+        self.setlevel(LEVEL_CONSTANT)
 
     def get_last_guard(self):
         return None
@@ -212,7 +223,7 @@
         assert isinstance(other_value, PtrOptValue)
         self.box = other_value.box
         self.known_class = other_value.known_class
-        self.level = other_value.level
+        self._tag = other_value._tag
         self.last_guard = other_value.last_guard
         self.lenbound = other_value.lenbound
 
@@ -225,19 +236,19 @@
             self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))
 
     def make_nonnull(self, guardop):
-        assert self.level < LEVEL_NONNULL
-        self.level = LEVEL_NONNULL
+        assert self.getlevel() < LEVEL_NONNULL
+        self.setlevel(LEVEL_NONNULL)
         self.last_guard = guardop
 
     def make_constant_class(self, classbox, guardop):
-        assert self.level < LEVEL_KNOWNCLASS
+        assert self.getlevel() < LEVEL_KNOWNCLASS
         self.known_class = classbox
-        self.level = LEVEL_KNOWNCLASS
+        self.setlevel(LEVEL_KNOWNCLASS)
         self.last_guard = guardop
 
     def import_from(self, other, optimizer):
         OptValue.import_from(self, other, optimizer)
-        if self.level != LEVEL_CONSTANT:
+        if self.getlevel() != LEVEL_CONSTANT:
             if other.getlenbound():
                 if self.lenbound:
                     assert other.getlenbound().mode == self.lenbound.mode
@@ -248,16 +259,17 @@
 
     def make_guards(self, box):
         guards = []
-        if self.level == LEVEL_CONSTANT:
+        level = self.getlevel()
+        if level == LEVEL_CONSTANT:
             op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
             guards.append(op)
-        elif self.level == LEVEL_KNOWNCLASS:
+        elif level == LEVEL_KNOWNCLASS:
             op = ResOperation(rop.GUARD_NONNULL, [box], None)
             guards.append(op)
             op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
             guards.append(op)
         else:
-            if self.level == LEVEL_NONNULL:
+            if level == LEVEL_NONNULL:
                 op = ResOperation(rop.GUARD_NONNULL, [box], None)
                 guards.append(op)
             if self.lenbound:
@@ -276,7 +288,7 @@
         return guards
 
     def get_constant_class(self, cpu):
-        level = self.level
+        level = self.getlevel()
         if level == LEVEL_KNOWNCLASS:
             return self.known_class
         elif level == LEVEL_CONSTANT:
@@ -314,13 +326,13 @@
         assert isinstance(other_value, IntOptValue)
         self.box = other_value.box
         self.intbound = other_value.intbound
-        self.level = other_value.level
+        self._tag = other_value._tag
 
     def make_constant(self, constbox):
         """Replace 'self.box' with a Const box."""
         assert isinstance(constbox, ConstInt)
         self.box = constbox
-        self.level = LEVEL_CONSTANT
+        self.setlevel(LEVEL_CONSTANT)
         val = constbox.getint()
         self.intbound = IntBound(val, val)
 
@@ -334,25 +346,26 @@
         return False
 
     def make_nonnull(self, guardop):
-        assert self.level < LEVEL_NONNULL
-        self.level = LEVEL_NONNULL
+        assert self.getlevel() < LEVEL_NONNULL
+        self.setlevel(LEVEL_NONNULL)
 
     def import_from(self, other, optimizer):
         OptValue.import_from(self, other, optimizer)
-        if self.level != LEVEL_CONSTANT:
+        if self.getlevel() != LEVEL_CONSTANT:
             if other.getintbound() is not None: # VRawBufferValue
                 self.intbound.intersect(other.getintbound())
 
     def make_guards(self, box):
         guards = []
-        if self.level == LEVEL_CONSTANT:
+        level = self.getlevel()
+        if level == LEVEL_CONSTANT:
             op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
             guards.append(op)
-        elif self.level == LEVEL_KNOWNCLASS:
+        elif level == LEVEL_KNOWNCLASS:
             op = ResOperation(rop.GUARD_NONNULL, [box], None)
             guards.append(op)
         else:
-            if self.level == LEVEL_NONNULL:
+            if level == LEVEL_NONNULL:
                 op = ResOperation(rop.GUARD_NONNULL, [box], None)
                 guards.append(op)
             self.intbound.make_guards(box, guards)
@@ -508,6 +521,8 @@
         self.optpure = None
         self.optheap = None
         self.optearlyforce = None
+        # the following two fields is the data kept for unrolling,
+        # those are the operations that can go to the short_preamble
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -609,8 +624,8 @@
             except KeyError:
                 pass
             else:
-                assert value.level != LEVEL_CONSTANT
-                assert cur_value.level != LEVEL_CONSTANT
+                assert value.getlevel() != LEVEL_CONSTANT
+                assert cur_value.getlevel() != LEVEL_CONSTANT
                 # replacing with a different box
                 cur_value.copy_from(value)
                 return
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -16,7 +16,7 @@
 class AbstractVirtualValue(optimizer.PtrOptValue):
     _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
-    level = optimizer.LEVEL_NONNULL
+    _tag = optimizer.LEVEL_NONNULL
     is_about_raw = False
     _cached_vinfo = None
 
@@ -198,7 +198,7 @@
             fieldvalue.visitor_walk_recursive(visitor)
 
 class VirtualValue(AbstractVirtualStructValue):
-    level = optimizer.LEVEL_KNOWNCLASS
+    _tag = optimizer.LEVEL_KNOWNCLASS
 
     def __init__(self, cpu, known_class, keybox, source_op=None):
         AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -281,7 +281,7 @@
     def __init__(self, value, is_opaque=False):
         self.is_opaque = is_opaque
         self.known_class = value.get_known_class()
-        self.level = value.level
+        self.level = value.getlevel()
         if value.getintbound() is None:
             self.intbound = IntUnbounded()
         else:

From noreply at buildbot.pypy.org  Tue Dec 23 18:58:08 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 23 Dec 2014 18:58:08 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20141223175808.933651D24B5@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75091:4762c51578d6
Date: 2014-12-23 19:57 +0200
http://bitbucket.org/pypy/pypy/changeset/4762c51578d6/

Log:	merge

diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -225,9 +225,9 @@
     >>> element.remove(subelement)
     >>> serialize(element) # 5
     '<tag key="value" />'
-    >>> element.remove(subelement)
+    >>> element.remove(subelement)    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ValueError: list.remove(x): x not in list
+    ValueError: list.remove(x): ...
     >>> serialize(element) # 6
     '<tag key="value" />'
     >>> element[0:0] = [subelement, subelement, subelement]

From noreply at buildbot.pypy.org  Tue Dec 23 21:33:50 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 21:33:50 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_ovf_operations and
 test_ovf_operations_reversed to pass
Message-ID: <20141223203350.B88781C34AB@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75092:dfab06318973
Date: 2014-12-23 15:41 -0500
http://bitbucket.org/pypy/pypy/changeset/dfab06318973/

Log:	Get test_ovf_operations and test_ovf_operations_reversed to pass

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -14,6 +14,11 @@
 from rpython.rtyper.lltypesystem.rffi import *
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
 
+# Global state
+num_anon_loops = 0
+num_guard_failure_fns = 0
+num_bridges = 0
+
 class Params:
     def __init__(self, assembler):
         self.paramlist = []
@@ -77,8 +82,9 @@
     """
     def __init__(self, assembler):
         self.failure_params = Params(assembler)
-        self.serial = assembler.num_guard_failure_fns
-        assembler.num_guard_failure_fns += 1
+        global num_guard_failure_fns
+        self.serial = num_guard_failure_fns
+        num_guard_failure_fns += 1
         self.t_fn_ptr_type = assembler.ctxt.new_function_ptr_type (
             assembler.t_jit_frame_ptr,
             self.failure_params.paramtypes,
@@ -155,10 +161,6 @@
         self.propagate_exception_path = 0
         #self.teardown()
 
-        self.num_anon_loops = 0
-        self.num_guard_failure_fns = 0
-        self.num_bridges = 0
-
         self.sizeof_signed = rffi.sizeof(lltype.Signed)
         self.label_for_descr = {}
         self.block_for_label_descr = {}
@@ -251,8 +253,9 @@
         self.lvalue_for_box = {}
 
         if not loopname:
-            loopname = 'anonloop_%i' % self.num_anon_loops
-            self.num_anon_loops += 1
+            global num_anon_loops
+            loopname = 'anonloop_%i' % num_anon_loops
+            num_anon_loops += 1
         print("  loopname: %r" % loopname)
         self.make_function(loopname, inputargs, operations)
 
@@ -295,8 +298,9 @@
                                                            operations)
         self.lvalue_for_box = {}
 
-        name = "bridge_%i" % self.num_bridges
-        self.num_bridges += 1
+        global num_bridges
+        name = "bridge_%i" % num_bridges
+        num_bridges += 1
 
         self.make_function(name, inputargs, operations)
 
@@ -399,6 +403,7 @@
                                    self.loop_params.paramlist,
                                    r_int(0)))
 
+        self.lvalue_ovf = self.fn.new_local(self.t_bool, "ovf")
         self.b_current = self.fn.new_block("initial")
 
         # Add an initial comment summarizing the loop or bridge:
@@ -725,6 +730,14 @@
                              self.t_bool,
                              boolval_is_nonnull, boolval_vtable_equality))
 
+    def emit_guard_no_overflow(self, resop):
+        self._impl_guard(resop, r_int(0),
+                         self.lvalue_ovf.as_rvalue())
+
+    def emit_guard_overflow(self, resop):
+        self._impl_guard(resop, r_int(1),
+                         self.lvalue_ovf.as_rvalue())
+
     def _impl_write_output_args(self, params, args):
         # Write outputs back:
         for idx, arg in enumerate(args):
@@ -1031,3 +1044,29 @@
             # ... = ARG1,
             self.ctxt.new_cast(self.expr_to_rvalue(resop._arg1),
                                t_field))
+
+    # "INT_*_OVF" operations:
+    def _impl_int_ovf(self, resop, builtin_name):
+        """
+        We implement ovf int binops using GCC builtins,
+        generating a call of the form:
+          ovf = __builtin_FOO_overflow(arg0, arg1, &result);
+        This is then optimized by gcc's builtins.c
+        fold_builtin_arith_overflow.
+        """
+        rval0 = self.expr_to_rvalue(resop._arg0)
+        rval1 = self.expr_to_rvalue(resop._arg1)
+        lvalres = self.expr_to_lvalue(resop.result)
+        builtin_fn = self.ctxt.get_builtin_function(builtin_name)
+        # "ovf = __builtin_FOO_overflow(arg0, arg1, &result);"
+        call = self.ctxt.new_call(builtin_fn,
+                                  [rval0,
+                                   rval1,
+                                   lvalres.get_address()])
+        self.b_current.add_assignment(self.lvalue_ovf, call)
+    def emit_int_add_ovf(self, resop):
+        self._impl_int_ovf(resop, '__builtin_add_overflow')
+    def emit_int_sub_ovf(self, resop):
+        self._impl_int_ovf(resop, '__builtin_sub_overflow')
+    def emit_int_mul_ovf(self, resop):
+        self._impl_int_ovf(resop, '__builtin_mul_overflow')
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -230,6 +230,10 @@
                                               self.PARAM_P_P,
                                               INT]),
 
+            (self.GCC_JIT_FUNCTION_P,
+             'gcc_jit_context_get_builtin_function', [self.GCC_JIT_CONTEXT_P,
+                                                      CCHARP]),
+
             (self.GCC_JIT_OBJECT_P,
              'gcc_jit_function_as_object', [self.GCC_JIT_FUNCTION_P]),
 
@@ -751,6 +755,13 @@
 
         return Function(self.lib, self, fn)
 
+    def get_builtin_function(self, name):
+        name_charp = str2charp(name)
+        fn = self.lib.gcc_jit_context_get_builtin_function(self.inner_ctxt,
+                                                           name_charp)
+        free_charp(name_charp)
+        return Function(self.lib, self, fn)
+
     def new_global(self, kind, type_, name):
         name_charp = str2charp(name)
         lvalue = self.lib.gcc_jit_context_new_global(self.inner_ctxt,

From noreply at buildbot.pypy.org  Tue Dec 23 22:27:39 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 22:27:39 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_cast_int_to_ptr and
 test_cast_ptr_to_int to pass
Message-ID: <20141223212739.907061C1347@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75093:4c394d19f7b9
Date: 2014-12-23 16:25 -0500
http://bitbucket.org/pypy/pypy/changeset/4c394d19f7b9/

Log:	Get test_cast_int_to_ptr and test_cast_ptr_to_int to pass

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -849,7 +849,7 @@
     def emit_float_abs(self, resop):
         self._impl_float_unaryop(resop, self.lib.GCC_JIT_UNARY_OP_ABS)
 
-    # "CAST_" operations:
+    # "CAST_" operations for "FLOAT":
     def emit_cast_float_to_int(self, resop):
         rvalue = self.expr_to_rvalue(resop._arg0)
         lvalres = self.expr_to_lvalue(resop.result)
@@ -967,6 +967,28 @@
     def emit_int_invert(self, resop):
         self._impl_int_unaryop(resop, self.lib.GCC_JIT_UNARY_OP_BITWISE_NEGATE)
 
+    # "CAST_" operations for "INT" vs "PTR":
+    def emit_cast_ptr_to_int(self, resop):
+        rvalue_in = self.expr_to_rvalue(resop._arg0)
+        lvalue_tmp = self.fn.new_local(self.t_any, "tmp")
+        lvalue_result = self.expr_to_lvalue(resop.result)
+        self.b_current.add_assignment(
+            lvalue_tmp.access_field(self.u_ptr),
+            rvalue_in)
+        self.b_current.add_assignment(
+            lvalue_result,
+            lvalue_tmp.as_rvalue().access_field(self.u_signed))
+    def emit_cast_int_to_ptr(self, resop):
+        rvalue_in = self.expr_to_rvalue(resop._arg0)
+        lvalue_tmp = self.fn.new_local(self.t_any, "tmp")
+        lvalue_result = self.expr_to_lvalue(resop.result)
+        self.b_current.add_assignment(
+            lvalue_tmp.access_field(self.u_signed),
+            rvalue_in)
+        self.b_current.add_assignment(
+            lvalue_result,
+            lvalue_tmp.as_rvalue().access_field(self.u_ptr))
+
     #
 
     def impl_get_lvalue_at_offset_from_ptr(self, ptr_expr, ll_offset, t_field):
diff --git a/rpython/jit/backend/libgccjit/rffi_bindings.py b/rpython/jit/backend/libgccjit/rffi_bindings.py
--- a/rpython/jit/backend/libgccjit/rffi_bindings.py
+++ b/rpython/jit/backend/libgccjit/rffi_bindings.py
@@ -356,6 +356,11 @@
                                              self.GCC_JIT_LOCATION_P,
                                              self.GCC_JIT_FIELD_P]),
 
+            (self.GCC_JIT_RVALUE_P,
+             'gcc_jit_rvalue_access_field', [self.GCC_JIT_RVALUE_P,
+                                             self.GCC_JIT_LOCATION_P,
+                                             self.GCC_JIT_FIELD_P]),
+
             (self.GCC_JIT_LVALUE_P,
              'gcc_jit_rvalue_dereference_field', [self.GCC_JIT_RVALUE_P,
                                                   self.GCC_JIT_LOCATION_P,
@@ -859,6 +864,14 @@
                     self,
                     self.lib.gcc_jit_rvalue_get_type(self.inner_rvalue))
 
+    def access_field(self, field):
+        return RValue(self.lib,
+                      self,
+                      self.lib.gcc_jit_rvalue_access_field(
+                          self.inner_rvalue,
+                          self.lib.null_location_ptr,
+                          field.inner_field))
+
     def dereference_field(self, field):
         return LValue(self.lib,
                       self,

From noreply at buildbot.pypy.org  Tue Dec 23 22:27:40 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 22:27:40 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_noops to pass
Message-ID: <20141223212740.DC2F81C1347@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75094:5e659ec2cfd9
Date: 2014-12-23 16:35 -0500
http://bitbucket.org/pypy/pypy/changeset/5e659ec2cfd9/

Log:	Get test_noops to pass

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -1067,6 +1067,11 @@
             self.ctxt.new_cast(self.expr_to_rvalue(resop._arg1),
                                t_field))
 
+    def emit_debug_merge_point(self, resop):
+        pass # noop
+    def emit_jit_debug(self, resop):
+        pass # noop
+
     # "INT_*_OVF" operations:
     def _impl_int_ovf(self, resop, builtin_name):
         """

From noreply at buildbot.pypy.org  Tue Dec 23 23:11:48 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 23:11:48 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Fix test_compile_bridge
Message-ID: <20141223221148.A18C01C01E8@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75095:8fd7e004583c
Date: 2014-12-23 16:45 -0500
http://bitbucket.org/pypy/pypy/changeset/8fd7e004583c/

Log:	Fix test_compile_bridge

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -304,9 +304,6 @@
 
         self.make_function(name, inputargs, operations)
 
-        self.datablockwrapper.done()      # finish using cpu.asmmemmgr
-        self.datablockwrapper = None
-
         self.ctxt.dump_to_file("/tmp/%s.c" % name, r_int(1))
         jit_result = self.ctxt.compile()
         self.ctxt.release()

From noreply at buildbot.pypy.org  Tue Dec 23 23:11:49 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Tue, 23 Dec 2014 23:11:49 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Initial implementation of
	rop.CALL
Message-ID: <20141223221150.002241C01E8@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75096:a466f0d20644
Date: 2014-12-23 17:13 -0500
http://bitbucket.org/pypy/pypy/changeset/a466f0d20644/

Log:	Initial implementation of rop.CALL

	Working test_runner.py "test_call*" testcases with this commit:
	test_call test_call_many_arguments test_call_stack_alignment
	test_call_with_const_floats

	Failing: test_call_box_func test_call_to_c_function
	test_call_to_c_function_with_callback
	test_call_release_gil_return_types
	test_call_release_gil_variable_function_and_arguments

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -1,6 +1,7 @@
 from rpython.jit.backend.llsupport import jitframe
 from rpython.jit.backend.llsupport.assembler import BaseAssembler
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
+from rpython.jit.backend.llsupport.descr import CallDescr
 from rpython.jit.backend.llsupport.regalloc import FrameManager
 from rpython.jit.backend.model import CompiledLoopToken
 from rpython.jit.backend.libgccjit.rffi_bindings import (
@@ -486,7 +487,7 @@
     def get_box_as_lvalue(self, box):
         if box not in self.lvalue_for_box:
             self.lvalue_for_box[box] = (
-                self.fn.new_local(self.get_type_for_box(box),
+                self.fn.new_local(self.get_type_for_expr(box),
                                   str(box)))
         return self.lvalue_for_box[box]
 
@@ -508,15 +509,16 @@
 
         raise NotImplementedError('unhandled expr: %s' % expr)
 
-    def get_type_for_box(self, box):
-        if isinstance(box, BoxInt):
+    def get_type_for_expr(self, expr):
+        if isinstance(expr, (BoxInt, ConstInt)):
             return self.t_Signed;
-        elif isinstance(box, BoxFloat):
+        elif isinstance(expr, (BoxFloat, ConstFloat)):
             return self.t_float;
-        elif isinstance(box, BoxPtr):
+        elif isinstance(expr, (BoxPtr, ConstPtr)):
             return self.t_void_ptr;
         else:
-            raise NotImplementedError('unhandled box: %s %s' % (box, type(box)))
+            raise NotImplementedError('unhandled expr: %s %s'
+                                      % (expr, type(expr)))
 
     def get_union_field_for_expr(self, expr):
         if isinstance(expr, (BoxInt, ConstInt)):
@@ -1046,7 +1048,7 @@
         self.b_current.add_assignment(
             lvalres,
             self.ctxt.new_cast(field_lvalue.as_rvalue(),
-                               self.get_type_for_box(resop.result)))
+                               self.get_type_for_expr(resop.result)))
 
     def emit_setfield_gc(self, resop):
         #print(repr(resop))
@@ -1069,6 +1071,53 @@
     def emit_jit_debug(self, resop):
         pass # noop
 
+    # '_CANRAISE_FIRST', # ----- start of can_raise operations -----
+    # '_CALL_FIRST',
+
+    def emit_call(self, resop):
+        print(resop)
+        print(dir(resop))
+        calldescr = resop.getdescr()
+        assert isinstance(calldescr, CallDescr)
+        #size = calldescr.get_result_size()
+        #sign = calldescr.is_result_signed()
+        print('resop.numargs(): %r' % resop.numargs())
+        print('[resop.getarg(i) for i in range(resop.numargs())]: %r'
+              % [resop.getarg(i) for i in range(resop.numargs())])
+        #print('resop.result: %r' % resop.result)
+
+        print('resop.getarg(0): %r' % resop.getarg(0))
+        print('dir(resop.getarg(0)): %r' % dir(resop.getarg(0)))
+        print('resop.getarg(0).type: %r' % resop.getarg(0).type)
+
+        #arg0 is a fnptr
+        #arg1..N are the args to the call
+
+        # FIXME: is the returntype of the fn always that of the result
+        # location?
+        returntype = self.get_type_for_expr(resop.result)
+
+        # FIXME: are the params types always those of the input args?
+        paramtypes = [self.get_type_for_expr(resop.getarg(i))
+                       for i in range(1, resop.numargs())]
+
+        fn_ptr_type = self.ctxt.new_function_ptr_type(
+            returntype,
+            paramtypes,
+            r_int(0)) # is_variadic
+        fn_ptr = self.ctxt.new_cast(self.expr_to_rvalue(resop.getarg(0)),
+                                    fn_ptr_type)
+        arg_rvalues = [self.expr_to_rvalue(resop.getarg(i))
+                       for i in range(1, resop.numargs())]
+        call = self.ctxt.new_call_through_ptr(fn_ptr,
+                                              arg_rvalues)
+        self.b_current.add_assignment(self.expr_to_lvalue(resop.result),
+                                      call)
+
+    #'_CALL_LAST',
+    #'_CANRAISE_LAST', # ----- end of can_raise operations -----
+
+    # '_OVF_FIRST', # ----- start of is_ovf operations -----
     # "INT_*_OVF" operations:
     def _impl_int_ovf(self, resop, builtin_name):
         """
@@ -1094,3 +1143,6 @@
         self._impl_int_ovf(resop, '__builtin_sub_overflow')
     def emit_int_mul_ovf(self, resop):
         self._impl_int_ovf(resop, '__builtin_mul_overflow')
+
+    # '_OVF_LAST', # ----- end of is_ovf operations -----
+    # '_LAST',     # for the backend to add more internal operations

From noreply at buildbot.pypy.org  Wed Dec 24 01:25:17 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 24 Dec 2014 01:25:17 +0100 (CET)
Subject: [pypy-commit] pypy py3k: adapt to py3
Message-ID: <20141224002517.10E741C01E8@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r75097:89fe3a968399
Date: 2014-12-23 15:36 -0800
http://bitbucket.org/pypy/pypy/changeset/89fe3a968399/

Log:	adapt to py3

diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -1097,8 +1097,9 @@
 
     def test_interned_strings(self):
         source = """x = ('foo_bar42', 5); y = 'foo_bar42'; z = x[0]"""
-        exec source
-        assert y is z
+        ns = {}
+        exec(source, ns)
+        assert ns['y'] is ns['z']
 
 
 class AppTestExceptions:

From noreply at buildbot.pypy.org  Wed Dec 24 02:50:59 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 24 Dec 2014 02:50:59 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Fix test_field_basic
Message-ID: <20141224015059.C0E271C34AB@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75098:389b2c4ea525
Date: 2014-12-23 17:33 -0500
http://bitbucket.org/pypy/pypy/changeset/389b2c4ea525/

Log:	Fix test_field_basic

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -1054,10 +1054,10 @@
         #print(repr(resop))
         assert isinstance(resop._arg0, (BoxPtr, ConstPtr))
         #print(resop._arg1)
-
+        fielddescr = resop.getdescr()
         field_lvalue = self.impl_get_lvalue_for_field(
             resop._arg0,
-            resop.getdescr())
+            fielddescr)
         t_field = self.impl_get_type_for_field(fielddescr)
 
         self.b_current.add_assignment(

From noreply at buildbot.pypy.org  Wed Dec 24 02:51:01 2014
From: noreply at buildbot.pypy.org (dmalcolm)
Date: Wed, 24 Dec 2014 02:51:01 +0100 (CET)
Subject: [pypy-commit] pypy libgccjit-backend: Get test_math_sqrt to pass
Message-ID: <20141224015101.102121C34AB@cobra.cs.uni-duesseldorf.de>

Author: David Malcolm <dmalcolm at redhat.com>
Branch: libgccjit-backend
Changeset: r75099:01d5262da26e
Date: 2014-12-23 19:57 -0500
http://bitbucket.org/pypy/pypy/changeset/01d5262da26e/

Log:	Get test_math_sqrt to pass

diff --git a/rpython/jit/backend/libgccjit/assembler.py b/rpython/jit/backend/libgccjit/assembler.py
--- a/rpython/jit/backend/libgccjit/assembler.py
+++ b/rpython/jit/backend/libgccjit/assembler.py
@@ -7,6 +7,7 @@
 from rpython.jit.backend.libgccjit.rffi_bindings import (
     make_eci, Library, make_param_array, make_field_array, Context, Type,
     RValue)
+from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import (
     BoxInt, ConstInt, BoxFloat, ConstFloat, BoxPtr, ConstPtr)
 from rpython.jit.metainterp.resoperation import *
@@ -1075,24 +1076,47 @@
     # '_CALL_FIRST',
 
     def emit_call(self, resop):
-        print(resop)
-        print(dir(resop))
+        #print(resop)
+        #print(dir(resop))
         calldescr = resop.getdescr()
-        assert isinstance(calldescr, CallDescr)
-        #size = calldescr.get_result_size()
-        #sign = calldescr.is_result_signed()
-        print('resop.numargs(): %r' % resop.numargs())
-        print('[resop.getarg(i) for i in range(resop.numargs())]: %r'
-              % [resop.getarg(i) for i in range(resop.numargs())])
-        #print('resop.result: %r' % resop.result)
-
-        print('resop.getarg(0): %r' % resop.getarg(0))
-        print('dir(resop.getarg(0)): %r' % dir(resop.getarg(0)))
-        print('resop.getarg(0).type: %r' % resop.getarg(0).type)
+        #assert isinstance(calldescr, CallDescr)
+        #print('dir(calldescr): %r' % dir(calldescr))
+        #print('calldescr.__dict__: %r' % calldescr.__dict__)
+        #print('calldescr.extrainfo: %r' % calldescr.extrainfo)
 
         #arg0 is a fnptr
         #arg1..N are the args to the call
 
+        arg_rvalues = [self.expr_to_rvalue(resop.getarg(i))
+                       for i in range(1, resop.numargs())]
+
+        #print('type(calldescr.extrainfo): %r' % type(calldescr.extrainfo))
+        assert isinstance(calldescr.extrainfo, EffectInfo)
+        if calldescr.extrainfo.oopspecindex:
+            #print('  calldescr.extrainfo.oopspecindex: %r'
+            #      % calldescr.extrainfo.oopspecindex)
+            if calldescr.extrainfo.oopspecindex == EffectInfo.OS_MATH_SQRT:
+                # Convert into a call to builtin: "double sqrt(double);"
+                fn = self.ctxt.get_builtin_function('sqrt')
+                call = self.ctxt.new_call(fn,
+                                          arg_rvalues)
+                self.b_current.add_assignment(
+                    self.expr_to_lvalue(resop.result),
+                    call)
+                return
+
+        #size = calldescr.get_result_size()
+        #sign = calldescr.is_result_signed()
+        #print('resop.numargs(): %r' % resop.numargs())
+        #print('[resop.getarg(i) for i in range(resop.numargs())]: %r'
+        #      % [resop.getarg(i) for i in range(resop.numargs())])
+        #print('resop.result: %r' % resop.result)
+
+        #print('resop.getarg(0): %r' % resop.getarg(0))
+        #print('dir(resop.getarg(0)): %r' % dir(resop.getarg(0)))
+        #print('resop.getarg(0).type: %r' % resop.getarg(0).type)
+
+
         # FIXME: is the returntype of the fn always that of the result
         # location?
         returntype = self.get_type_for_expr(resop.result)
@@ -1107,8 +1131,6 @@
             r_int(0)) # is_variadic
         fn_ptr = self.ctxt.new_cast(self.expr_to_rvalue(resop.getarg(0)),
                                     fn_ptr_type)
-        arg_rvalues = [self.expr_to_rvalue(resop.getarg(i))
-                       for i in range(1, resop.numargs())]
         call = self.ctxt.new_call_through_ptr(fn_ptr,
                                               arg_rvalues)
         self.b_current.add_assignment(self.expr_to_lvalue(resop.result),

From noreply at buildbot.pypy.org  Wed Dec 24 11:48:27 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Dec 2014 11:48:27 +0100 (CET)
Subject: [pypy-commit] pypy.org extradoc: Document more package.py.
Message-ID: <20141224104827.789181C10F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r557:137a96636f8c
Date: 2014-12-24 11:48 +0100
http://bitbucket.org/pypy/pypy.org/changeset/137a96636f8c/

Log:	Document more package.py.

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -246,6 +246,8 @@
 machine with insufficient RAM!  It will just swap forever.  See
 notes below in that case.)</p>
 </li>
+<li><p class="first">If you want to install this PyPy as root, please read the next section.</p>
+</li>
 </ol>
 <p>Notes:</p>
 <ul>
@@ -282,6 +284,15 @@
 python package.py --help #for information
 python package.py --archive-name pypy-my-own-package-name
 </pre>
+<p>It is recommended to use package.py because custom scripts will
+invariably become out-of-date.  If you want to write custom scripts
+anyway, note an easy-to-miss point: some modules are written with CFFI,
+and require some compilation.  If you install PyPy as root without
+pre-compiling them, normal users will get permission errors.  This means
+you need to run for example <tt class="docutils literal">pypy <span class="pre">-c</span> &ldquo;import gdbm&rdquo;</tt> during the
+installation process (see the exact list in <a class="reference external" href="https://bitbucket.org/pypy/pypy/src/default/pypy/tool/release/package.py">package.py</a>).  Users
+seeing a broken installation of PyPy can also fix it after-the-fact, if
+they have sudo rights, by running <tt class="docutils literal">sudo pypy <span class="pre">-c</span> &ldquo;import gdbm&rdquo;</tt>.</p>
 </div>
 <div class="section" id="checksums">
 <h1>Checksums</h1>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -252,6 +252,8 @@
    machine with insufficient RAM!  It will just swap forever.  See
    notes below in that case.)
 
+6. If you want to install this PyPy as root, please read the next section.
+
 Notes:
 
 * It is recommended to use PyPy to do translations, instead of using CPython,
@@ -297,6 +299,18 @@
     python package.py --help #for information
     python package.py --archive-name pypy-my-own-package-name 
 
+It is recommended to use package.py because custom scripts will
+invariably become out-of-date.  If you want to write custom scripts
+anyway, note an easy-to-miss point: some modules are written with CFFI,
+and require some compilation.  If you install PyPy as root without
+pre-compiling them, normal users will get permission errors.  This means
+you need to run for example ``pypy -c "import gdbm"`` during the
+installation process (see the exact list in `package.py`_).  Users
+seeing a broken installation of PyPy can also fix it after-the-fact, if
+they have sudo rights, by running ``sudo pypy -c "import gdbm"``.
+
+.. _`package.py`: https://bitbucket.org/pypy/pypy/src/default/pypy/tool/release/package.py
+
 Checksums
 ---------
 

From noreply at buildbot.pypy.org  Wed Dec 24 11:56:01 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 24 Dec 2014 11:56:01 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: A branch to try and slim
 the guard resop descr
Message-ID: <20141224105601.DA0941C10F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75100:f5afc6c7549d
Date: 2014-12-24 12:47 +0200
http://bitbucket.org/pypy/pypy/changeset/f5afc6c7549d/

Log:	A branch to try and slim the guard resop descr

diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1479,15 +1479,15 @@
             assert kind == 'v'
         return lltype.nullptr(rclass.OBJECTPTR.TO)
 
-    def _prepare_resume_from_failure(self, opnum, dont_change_position,
-                                     deadframe):
+    def _prepare_resume_from_failure(self, opnum, deadframe):
         from rpython.jit.metainterp.resoperation import rop
         #
-        if opnum == rop.GUARD_TRUE:
+        if opnum == rop.GUARD_FUTURE_CONDITION:
+            pass
+        elif opnum == rop.GUARD_TRUE:
             # Produced directly by some goto_if_not_xxx() opcode that did not
             # jump, but which must now jump.  The pc is just after the opcode.
-            if not dont_change_position:
-                self.position = self.jitcode.follow_jump(self.position)
+            self.position = self.jitcode.follow_jump(self.position)
         #
         elif opnum == rop.GUARD_FALSE:
             # Produced directly by some goto_if_not_xxx() opcode that jumped,
@@ -1517,8 +1517,7 @@
         elif opnum == rop.GUARD_NO_OVERFLOW:
             # Produced by int_xxx_ovf().  The pc is just after the opcode.
             # We get here because it did not used to overflow, but now it does.
-            if not dont_change_position:
-                return get_llexception(self.cpu, OverflowError())
+            return get_llexception(self.cpu, OverflowError())
         #
         elif opnum == rop.GUARD_OVERFLOW:
             # Produced by int_xxx_ovf().  The pc is just after the opcode.
@@ -1649,13 +1648,9 @@
         resumedescr,
         deadframe,
         all_virtuals)
-    if isinstance(resumedescr, ResumeAtPositionDescr):
-        dont_change_position = True
-    else:
-        dont_change_position = False
 
     current_exc = blackholeinterp._prepare_resume_from_failure(
-        resumedescr.guard_opnum, dont_change_position, deadframe)
+        resumedescr.guard_opnum, deadframe)
 
     _run_forever(blackholeinterp, current_exc)
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -479,22 +479,36 @@
     return d
 
 class ResumeDescr(AbstractFailDescr):
-    pass
+    _attrs_ = ()
 
 class ResumeGuardDescr(ResumeDescr):
-    # this class also gets the following attributes stored by resume.py code
-    # XXX move all of unused stuff to guard_op, now that we can have
-    #     a separate class, so it does not survive that long
-    rd_snapshot = None
-    rd_frame_info_list = None
+    _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
+               'rd_frame_info_list', 'rd_pendingfields', 'status')
+    
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
     rd_consts = None
     rd_virtuals = None
+    rd_frame_info_list = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
 
     status = r_uint(0)
 
+    def clone(self):
+        new = self.__class__()
+        new.rd_frame_info_list = self.rd_frame_info_list
+        return new
+
+    def copy_all_attributes_from(self, other):
+        assert isinstance(other, ResumeGuardDescr)
+        self.rd_count = other.rd_count
+        self.rd_consts = other.rd_consts
+        self.rd_frame_info_list = other.rd_frame_info_list
+        self.rd_pendingfields = other.rd_pendingfields
+        self.rd_virtuals = other.rd_virtuals
+        self.rd_numb = other.rd_numb
+        # we don't copy status
+
     ST_BUSY_FLAG    = 0x01     # if set, busy tracing from the guard
     ST_TYPE_MASK    = 0x06     # mask for the type (TY_xxx)
     ST_SHIFT        = 3        # in "status >> ST_SHIFT" is stored:
@@ -509,31 +523,12 @@
     def store_final_boxes(self, guard_op, boxes, metainterp_sd):
         guard_op.setfailargs(boxes)
         self.rd_count = len(boxes)
-        self.guard_opnum = guard_op.getopnum()
         #
         if metainterp_sd.warmrunnerdesc is not None:   # for tests
             jitcounter = metainterp_sd.warmrunnerdesc.jitcounter
             hash = jitcounter.fetch_next_hash()
             self.status = hash & self.ST_SHIFT_MASK
 
-    def make_a_counter_per_value(self, guard_value_op):
-        assert guard_value_op.getopnum() == rop.GUARD_VALUE
-        box = guard_value_op.getarg(0)
-        try:
-            i = guard_value_op.getfailargs().index(box)
-        except ValueError:
-            return     # xxx probably very rare
-        else:
-            if box.type == history.INT:
-                ty = self.TY_INT
-            elif box.type == history.REF:
-                ty = self.TY_REF
-            elif box.type == history.FLOAT:
-                ty = self.TY_FLOAT
-            else:
-                assert 0, box.type
-            self.status = ty | (r_uint(i) << self.ST_SHIFT)
-
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
             self.start_compiling()
@@ -632,32 +627,62 @@
                                self, inputargs, new_loop.operations,
                                new_loop.original_jitcell_token)
 
-    def copy_all_attributes_into(self, res):
-        # XXX a bit ugly to have to list them all here
-        res.rd_snapshot = self.rd_snapshot
-        res.rd_frame_info_list = self.rd_frame_info_list
-        res.rd_numb = self.rd_numb
-        res.rd_consts = self.rd_consts
-        res.rd_virtuals = self.rd_virtuals
-        res.rd_pendingfields = self.rd_pendingfields
-        res.rd_count = self.rd_count
+    def make_a_counter_per_value(self, guard_value_op):
+        assert guard_value_op.getopnum() == rop.GUARD_VALUE
+        box = guard_value_op.getarg(0)
+        try:
+            i = guard_value_op.getfailargs().index(box)
+        except ValueError:
+            return     # xxx probably very rare
+        else:
+            if box.type == history.INT:
+                ty = self.TY_INT
+            elif box.type == history.REF:
+                ty = self.TY_REF
+            elif box.type == history.FLOAT:
+                ty = self.TY_FLOAT
+            else:
+                assert 0, box.type
+            self.status = ty | (r_uint(i) << self.ST_SHIFT)
 
-    def _clone_if_mutable(self):
-        res = ResumeGuardDescr()
-        self.copy_all_attributes_into(res)
-        return res
+class ResumeGuardNonnullDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NONNULL
+
+class ResumeGuardIsnullDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_ISNULL
+
+class ResumeGuardClassDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_CLASS
+
+class ResumeGuardTrueDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_TRUE
+
+class ResumeGuardFalseDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_FALSE
+
+class ResumeGuardNonnullClassDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NONNULL_CLASS
+
+class ResumeGuardExceptionDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_EXCEPTION
+
+class ResumeGuardNoExceptionDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NO_EXCEPTION
+
+class ResumeGuardOverflowDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_OVERFLOW
+
+class ResumeGuardNoOverflowDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NO_OVERFLOW
+
+class ResumeGuardValueDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_VALUE
 
 class ResumeGuardNotInvalidated(ResumeGuardDescr):
-    def _clone_if_mutable(self):
-        res = ResumeGuardNotInvalidated()
-        self.copy_all_attributes_into(res)
-        return res
+    guard_opnum = rop.GUARD_NOT_INVALIDATED
 
 class ResumeAtPositionDescr(ResumeGuardDescr):
-    def _clone_if_mutable(self):
-        res = ResumeAtPositionDescr()
-        self.copy_all_attributes_into(res)
-        return res
+    guard_opnum = rop.GUARD_FUTURE_CONDITION
 
 class AllVirtuals:
     llopaque = True
@@ -678,11 +703,19 @@
 
 
 class ResumeGuardForcedDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NOT_FORCED
 
-    def __init__(self, metainterp_sd, jitdriver_sd):
+    def _init(self, metainterp_sd, jitdriver_sd):
+        # to please the annotator
         self.metainterp_sd = metainterp_sd
         self.jitdriver_sd = jitdriver_sd
 
+    def clone(self):
+        new = ResumeGuardForcedDescr()
+        new._init(self.metainterp_sd, self.jitdriver_sd)
+        new.rd_frame_info_list = self.rd_frame_info_list
+        return new
+
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         # Failures of a GUARD_NOT_FORCED are never compiled, but
         # always just blackholed.  First fish for the data saved when
@@ -740,13 +773,6 @@
         hidden_all_virtuals = obj.hide(metainterp_sd.cpu)
         metainterp_sd.cpu.set_savedata_ref(deadframe, hidden_all_virtuals)
 
-    def _clone_if_mutable(self):
-        res = ResumeGuardForcedDescr(self.metainterp_sd,
-                                     self.jitdriver_sd)
-        self.copy_all_attributes_into(res)
-        return res
-
-
 class ResumeFromInterpDescr(ResumeDescr):
     def __init__(self, original_greenkey):
         self.original_greenkey = original_greenkey
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -134,14 +134,6 @@
     def repr_of_descr(self):
         return '%r' % (self,)
 
-    def _clone_if_mutable(self):
-        return self
-    def clone_if_mutable(self):
-        clone = self._clone_if_mutable()
-        if not we_are_translated():
-            assert clone.__class__ is self.__class__
-        return clone
-
     def hide(self, cpu):
         descr_ptr = cpu.ts.cast_instance_to_base_ref(self)
         return cpu.ts.cast_to_ref(descr_ptr)
@@ -159,6 +151,8 @@
     index = -1
     final_descr = False
 
+    _attrs_ = ()
+
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         raise NotImplementedError
     def compile_and_attach(self, metainterp, new_loop):
diff --git a/rpython/jit/metainterp/inliner.py b/rpython/jit/metainterp/inliner.py
--- a/rpython/jit/metainterp/inliner.py
+++ b/rpython/jit/metainterp/inliner.py
@@ -1,5 +1,6 @@
 from rpython.jit.metainterp.history import Const
 from rpython.jit.metainterp.resume import Snapshot
+from rpython.jit.metainterp.resoperation import GuardResOp
 
 
 class Inliner(object):
@@ -26,21 +27,16 @@
                 newop.setfailargs([self.inline_arg(a) for a in args])
             else:
                 newop.setfailargs([])
+            assert isinstance(newop, GuardResOp)
+            newop.rd_snapshot = self.inline_snapshot(newop.rd_snapshot)
 
         if newop.result and not ignore_result:
             old_result = newop.result
             newop.result = newop.result.clonebox()
             self.argmap[old_result] = newop.result
 
-        self.inline_descr_inplace(newop.getdescr())
-
         return newop
 
-    def inline_descr_inplace(self, descr):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
-        if isinstance(descr, ResumeGuardDescr):
-            descr.rd_snapshot = self.inline_snapshot(descr.rd_snapshot)
-
     def inline_arg(self, arg):
         if arg is None:
             return None
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -7,7 +7,8 @@
                                                      IntLowerBound, MININT,\
                                                      MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractResOp
+from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+     AbstractResOp, GuardResOp
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.tool.pairtype import extendabletype
 from rpython.rlib.debug import debug_print
@@ -735,6 +736,11 @@
         while i > 0:
             i -= 1
             if self._newoperations[i] is old_op:
+                # a bit of dance to make sure we copy all the attrs on
+                # an already emitted descr
+                newdescr = new_op.getdescr()
+                olddescr = old_op.getdescr()
+                newdescr.copy_all_attributes_from(olddescr)
                 self._newoperations[i] = new_op
                 break
         else:
@@ -744,7 +750,9 @@
         assert pendingfields is not None
         descr = op.getdescr()
         assert isinstance(descr, compile.ResumeGuardDescr)
-        modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo)
+        assert isinstance(op, GuardResOp)
+        modifier = resume.ResumeDataVirtualAdder(descr, op,
+                                                 self.resumedata_memo)
         try:
             newboxes = modifier.finish(self, pendingfields)
             if len(newboxes) > self.metainterp_sd.options.failargs_limit:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -312,15 +312,11 @@
                 if not previous_classbox.same_constant(expected_classbox):
                     r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
                     raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r)
+            descr = compile.ResumeGuardValueDescr()
             op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+                        args = [old_guard_op.getarg(0), op.getarg(1)],
+                        descr = descr)
             self.optimizer.replaces_guard[op] = old_guard_op
-            # hack hack hack.  Change the guard_opnum on
-            # new_guard_op.getdescr() so that when resuming,
-            # the operation is not skipped by pyjitpl.py.
-            descr = op.getdescr()
-            assert isinstance(descr, compile.ResumeGuardDescr)
-            descr.guard_opnum = rop.GUARD_VALUE
             descr.make_a_counter_per_value(op)
             # to be safe
             if isinstance(value, PtrOptValue):
@@ -364,15 +360,11 @@
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
+                descr = compile.ResumeGuardNonnullClassDescr()
                 op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
-                                         args = [old_guard_op.getarg(0), op.getarg(1)])
+                            args = [old_guard_op.getarg(0), op.getarg(1)],
+                            descr=descr)
                 self.optimizer.replaces_guard[op] = old_guard_op
-                # hack hack hack.  Change the guard_opnum on
-                # new_guard_op.getdescr() so that when resuming,
-                # the operation is not skipped by pyjitpl.py.
-                descr = op.getdescr()
-                assert isinstance(descr, compile.ResumeGuardDescr)
-                descr.guard_opnum = rop.GUARD_NONNULL_CLASS
         self.emit_operation(op)
         value.make_constant_class(expectedclassbox, op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -24,7 +24,7 @@
     fi0 = resume.FrameInfo(None, "code0", 11)
     fdescr.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33)
     snapshot0 = resume.Snapshot(None, [b0])
-    fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1])
+    op.rd_snapshot = resume.Snapshot(snapshot0, [b1])
     #
     opt.store_final_boxes_in_guard(op, [])
     if op.getfailargs() == [b0, b1]:
@@ -97,7 +97,8 @@
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
-        loop = self.parse(ops)
+                
+        loop = self.parse(ops, postprocess=self.postprocess)
         token = JitCellToken()
         loop.operations = [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \
                           loop.operations
@@ -108,42 +109,6 @@
         print '\n'.join([str(o) for o in loop.operations])
         self.assert_equal(loop, expected)
 
-    def setup_method(self, meth=None):
-        class FailDescr(compile.ResumeGuardDescr):
-            oparse = None
-            def _oparser_uses_descr_of_guard(self, oparse, fail_args):
-                # typically called 3 times: once when parsing 'ops',
-                # once when parsing 'preamble', once when parsing 'expected'.
-                self.oparse = oparse
-                self.rd_frame_info_list, self.rd_snapshot = snapshot(fail_args)
-            def _clone_if_mutable(self):
-                assert self is fdescr
-                return fdescr2
-            def __repr__(self):
-                if self is fdescr:
-                    return 'fdescr'
-                if self is fdescr2:
-                    return 'fdescr2'
-                return compile.ResumeGuardDescr.__repr__(self)
-        #
-        def snapshot(fail_args, got=[]):
-            if not got:    # only the first time, i.e. when parsing 'ops'
-                rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-                rd_snapshot = resume.Snapshot(None, fail_args)
-                got.append(rd_frame_info_list)
-                got.append(rd_snapshot)
-            return got
-        #
-        fdescr = instantiate(FailDescr)
-        self.namespace['fdescr'] = fdescr
-        fdescr2 = instantiate(FailDescr)
-        self.namespace['fdescr2'] = fdescr2
-
-    def teardown_method(self, meth):
-        self.namespace.pop('fdescr', None)
-        self.namespace.pop('fdescr2', None)
-
-
 
 class BaseTestOptimizeBasic(BaseTestBasic):
 
@@ -2065,14 +2030,14 @@
     def test_merge_guard_nonnull_guard_class(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         expected = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull_class(p1, ConstClass(node_vtable), descr=fdescr) [i0]
+        guard_nonnull_class(p1, ConstClass(node_vtable)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3, p2)
         """
@@ -2082,14 +2047,14 @@
     def test_merge_guard_nonnull_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_value(p1, ConstPtr(myptr)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         expected = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3, p2)
         """
@@ -2099,7 +2064,7 @@
     def test_merge_guard_nonnull_guard_class_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i2]
         i4 = int_sub(i3, 1)
@@ -2108,7 +2073,7 @@
         """
         expected = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         i4 = int_sub(i3, 1)
         jump(p2, i0, i1, i4, p2)
@@ -2527,7 +2492,7 @@
         reader = ResumeDataFakeReader(fdescr, fail_args,
                                       MyMetaInterp(self.cpu))
         boxes = reader.consume_boxes()
-        self._verify_fail_args(boxes, fdescr.oparse, expectedtext)
+        self._verify_fail_args(boxes, self.oparse, expectedtext)
 
     def test_expand_fail_1(self):
         ops = """
@@ -2538,12 +2503,12 @@
         i4 = getfield_gc(p1, descr=valuedescr)
         #
         i2 = int_add(10, 5)
-        guard_true(i1, descr=fdescr) [i2, i4]
+        guard_true(i1) [i2, i4]
         jump(i1, i4)
         """
         expected = """
         [i1, i3]
-        guard_true(i1, descr=fdescr) [i3]
+        guard_true(i1) [i3]
         jump(1, i3)
         """
         self.optimize_loop(ops, expected)
@@ -2555,12 +2520,12 @@
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i2, descr=valuedescr)
         setfield_gc(p1, p1, descr=nextdescr)
-        guard_true(i1, descr=fdescr) [p1]
+        guard_true(i1) [p1]
         jump(i1, i2)
         """
         expected = """
         [i1, i2]
-        guard_true(i1, descr=fdescr) [i2]
+        guard_true(i1) [i2]
         jump(1, i2)
         """
         self.optimize_loop(ops, expected)
@@ -2577,12 +2542,12 @@
         setfield_gc(p1, p2, descr=nextdescr)
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(p2, p3, descr=nextdescr)
-        guard_true(i1, descr=fdescr) [i3, p1]
+        guard_true(i1) [i3, p1]
         jump(i2, i1, i3, p3)
         """
         expected = """
         [i1, i2, i3, p3]
-        guard_true(i1, descr=fdescr) [i3, i2, p3]
+        guard_true(i1) [i3, i2, p3]
         jump(i2, 1, i3, p3)
         """
         self.optimize_loop(ops, expected)
@@ -2594,7 +2559,6 @@
     def test_expand_fail_4(self):
         for arg in ['p1', 'i2,p1', 'p1,p2', 'p2,p1',
                     'i2,p1,p2', 'i2,p2,p1']:
-            self.setup_method() # humpf
             ops = """
             [i1, i2, i3]
             p1 = new_with_vtable(ConstClass(node_vtable))
@@ -2604,12 +2568,12 @@
             setfield_gc(p1, i2, descr=valuedescr)
             setfield_gc(p1, p2, descr=nextdescr)
             setfield_gc(p2, i2, descr=valuedescr)
-            guard_true(i1, descr=fdescr) [i4, i3, %s]
+            guard_true(i1) [i4, i3, %s]
             jump(i1, i2, i3)
             """
             expected = """
             [i1, i2, i3]
-            guard_true(i1, descr=fdescr) [i3, i2]
+            guard_true(i1) [i3, i2]
             jump(1, i2, i3)
             """
             self.optimize_loop(ops % arg, expected)
@@ -2627,12 +2591,12 @@
         setfield_gc(p1, p2, descr=nextdescr)
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(p2, p1, descr=nextdescr)      # a cycle
-        guard_true(i1, descr=fdescr) [i3, i4, p1, p2]
+        guard_true(i1) [i3, i4, p1, p2]
         jump(i2, i1, i3, i4)
         """
         expected = """
         [i1, i2, i3, i4]
-        guard_true(i1, descr=fdescr) [i3, i4, i2]
+        guard_true(i1) [i3, i4, i2]
         jump(i2, 1, i3, i4)
         """
         self.optimize_loop(ops, expected)
@@ -2644,14 +2608,14 @@
     def test_expand_fail_6(self):
         ops = """
         [p0, i0, i1]
-        guard_true(i0, descr=fdescr) [p0]
+        guard_true(i0) [p0]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
         jump(p1, i1, i1)
         """
         expected = """
         [i1b, i0, i1]
-        guard_true(i0, descr=fdescr) [i1b]
+        guard_true(i0) [i1b]
         jump(i1, i1, i1)
         """
         py.test.skip("XXX")
@@ -2667,13 +2631,13 @@
         p1 = new_array(3, descr=arraydescr)
         setarrayitem_gc(p1, 1, i1, descr=arraydescr)
         setarrayitem_gc(p1, 0, 25, descr=arraydescr)
-        guard_true(i1, descr=fdescr) [p1]
+        guard_true(i1) [p1]
         i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
         jump(i2)
         """
         expected = """
         [i1]
-        guard_true(i1, descr=fdescr) [i1]
+        guard_true(i1) [i1]
         jump(1)
         """
         self.optimize_loop(ops, expected)
@@ -2687,14 +2651,14 @@
         p2 = new(descr=ssize)
         setfield_gc(p2, i1, descr=adescr)
         setfield_gc(p2, p1, descr=bdescr)
-        guard_true(i1, descr=fdescr) [p2]
+        guard_true(i1) [p2]
         i3 = getfield_gc(p2, descr=adescr)
         p3 = getfield_gc(p2, descr=bdescr)
         jump(i3, p3)
         """
         expected = """
         [i1, p1]
-        guard_true(i1, descr=fdescr) [i1, p1]
+        guard_true(i1) [i1, p1]
         jump(1, p1)
         """
         self.optimize_loop(ops, expected)
@@ -2711,7 +2675,7 @@
         setfield_gc(p5s, i2, descr=adescr)
         setfield_gc(p5s, p7v, descr=bdescr)
         setarrayitem_gc(p1a, 1, p5s, descr=arraydescr2)
-        guard_true(i1, descr=fdescr) [p1a]
+        guard_true(i1) [p1a]
         p2s = new(descr=ssize)
         p3v = new_with_vtable(ConstClass(node_vtable))
         p4a = new_array(2, descr=arraydescr2)
@@ -2723,7 +2687,7 @@
         """
         expected = """
         [i1, ia, iv, pnull, i2]
-        guard_true(i1, descr=fdescr) [ia, iv, i2]
+        guard_true(i1) [ia, iv, i2]
         jump(1, 1, i2, NULL, i2)
         """
         py.test.skip("XXX")
@@ -2749,14 +2713,14 @@
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        guard_true(i3, descr=fdescr) []
+        guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         jump(p1, i2, i4)
         """
         expected = """
         [p1, i2, i3]
-        guard_true(i3, descr=fdescr) [p1, i2]
+        guard_true(i3) [p1, i2]
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         jump(p1, i2, i4)
@@ -2774,14 +2738,14 @@
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(ConstPtr(myptr), p2, descr=nextdescr)
-        guard_true(i3, descr=fdescr) []
+        guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr)
         jump(i2, i4)
         """
         expected = """
         [i2, i3]
-        guard_true(i3, descr=fdescr) [i2]
+        guard_true(i3) [i2]
         i4 = int_neg(i2)
         setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr)
         jump(i2, i4)
@@ -3096,7 +3060,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=nextdescr)
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr) [p2, p1]
+        guard_not_forced() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=nextdescr)
         jump(p0, i1)
@@ -3111,7 +3075,7 @@
         setfield_gc(p0, p2, descr=nextdescr)
         #
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr) [p2, i1]
+        guard_not_forced() [p2, i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
         p1 = new_with_vtable(ConstClass(node_vtable))
@@ -3142,7 +3106,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=refdescr)
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p2, p1]
+        guard_no_exception() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=refdescr)
         jump(p0, i1)
@@ -3151,7 +3115,7 @@
         [p0, i1]
         p3 = force_token()
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p3, i1, p0]
+        guard_no_exception() [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         jump(p0, i1)
         """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -45,7 +45,7 @@
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
-        loop = self.parse(ops)
+        loop = self.parse(ops, postprocess=self.postprocess)
         if expected != "crash!":
             expected = self.parse(expected)
         if expected_preamble:
@@ -96,41 +96,6 @@
 
 
 class OptimizeOptTest(BaseTestWithUnroll):
-    def setup_method(self, meth=None):
-        class FailDescr(compile.ResumeGuardDescr):
-            oparse = None
-            def _oparser_uses_descr_of_guard(self, oparse, fail_args):
-                # typically called 3 times: once when parsing 'ops',
-                # once when parsing 'preamble', once when parsing 'expected'.
-                self.oparse = oparse
-                self.rd_frame_info_list, self.rd_snapshot = snapshot(fail_args)
-            def _clone_if_mutable(self):
-                assert self is fdescr
-                return fdescr2
-            def __repr__(self):
-                if self is fdescr:
-                    return 'fdescr'
-                if self is fdescr2:
-                    return 'fdescr2'
-                return compile.ResumeGuardDescr.__repr__(self)
-        #
-        def snapshot(fail_args, got=[]):
-            if not got:    # only the first time, i.e. when parsing 'ops'
-                rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-                rd_snapshot = resume.Snapshot(None, fail_args)
-                got.append(rd_frame_info_list)
-                got.append(rd_snapshot)
-            return got
-        #
-        fdescr = instantiate(FailDescr)
-        self.namespace['fdescr'] = fdescr
-        fdescr2 = instantiate(FailDescr)
-        self.namespace['fdescr2'] = fdescr2
-
-    def teardown_method(self, meth):
-        self.namespace.pop('fdescr', None)
-        self.namespace.pop('fdescr2', None)
-
     def test_simple(self):
         ops = """
         []
@@ -3004,20 +2969,20 @@
     def test_merge_guard_nonnull_guard_class(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         preamble = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull_class(p1, ConstClass(node_vtable), descr=fdescr) [i0]
+        guard_nonnull_class(p1, ConstClass(node_vtable)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3)
         """
         expected = """
         [p2, i0, i1, i2]
-        guard_nonnull_class(p2, ConstClass(node_vtable), descr=fdescr2) [i0]
+        guard_nonnull_class(p2, ConstClass(node_vtable)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3)
         """
@@ -3027,20 +2992,20 @@
     def test_merge_guard_nonnull_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_value(p1, ConstPtr(myptr)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         preamble = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3)
         """
         expected = """
         [p2, i0, i1, i2]
-        guard_value(p2, ConstPtr(myptr), descr=fdescr2) [i0]
+        guard_value(p2, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         jump(ConstPtr(myptr), i0, i1, i3)
         """
@@ -3050,7 +3015,7 @@
     def test_merge_guard_nonnull_guard_class_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i2]
         i4 = int_sub(i3, 1)
@@ -3059,14 +3024,14 @@
         """
         preamble = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         i4 = int_sub(i3, 1)
         jump(p2, i0, i1, i4)
         """
         expected = """
         [p2, i0, i1, i2]
-        guard_value(p2, ConstPtr(myptr), descr=fdescr2) [i0]
+        guard_value(p2, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         i4 = int_sub(i3, 1)
         jump(ConstPtr(myptr), i0, i1, i4)
@@ -3860,7 +3825,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=nextdescr)
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr) [p2, p1]
+        guard_not_forced() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=nextdescr)
         jump(p0, i1)
@@ -3875,7 +3840,7 @@
         setfield_gc(p0, p2, descr=nextdescr)
         #
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr2) [p2, i1]
+        guard_not_forced() [p2, i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
         p1 = new_with_vtable(ConstClass(node_vtable))
@@ -3906,7 +3871,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=refdescr)
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p2, p1]
+        guard_no_exception() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=refdescr)
         escape()
@@ -3916,7 +3881,7 @@
         [p0, i1]
         p3 = force_token()
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p3, i1, p0]
+        guard_no_exception() [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         escape()
         jump(p0, i1)
@@ -3925,7 +3890,7 @@
         [p0, i1]
         p3 = force_token()
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr2) [p3, i1, p0]
+        guard_no_exception() [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         escape()
         jump(p0, i1)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -8,12 +8,13 @@
 from rpython.jit.backend.llgraph import runner
 from rpython.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr,
                                          Const, TreeLoop, AbstractDescr,
-                                         JitCellToken, TargetToken)
+                                         JitCellToken, TargetToken,
+    BasicFinalDescr)
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.codewriter.heaptracker import register_known_gctype, adr2int
-from rpython.jit.tool.oparser import parse, pure_parse
+from rpython.jit.tool.oparser import OpParser, pure_parse
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
 from rpython.jit.metainterp.jitprof import EmptyProfiler
@@ -348,40 +349,46 @@
     def store_final_boxes(self, op, boxes, metainterp_sd):
         op.setfailargs(boxes)
     def __eq__(self, other):
-        return type(self) is type(other)      # xxx obscure
-    def clone_if_mutable(self):
-        res = Storage(self.metainterp_sd, self.original_greenkey)
-        self.copy_all_attributes_into(res)
-        return res
+        return True # screw this
+        #return type(self) is type(other)      # xxx obscure
 
 def _sortboxes(boxes):
     _kind2count = {history.INT: 1, history.REF: 2, history.FLOAT: 3}
     return sorted(boxes, key=lambda box: _kind2count[box.type])
 
+final_descr = BasicFinalDescr()
+
 class BaseTest(object):
 
-    def parse(self, s, boxkinds=None, want_fail_descr=True):
+    def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None):
         if want_fail_descr:
             invent_fail_descr = self.invent_fail_descr
         else:
             invent_fail_descr = lambda *args: None
-        return parse(s, self.cpu, self.namespace,
-                     type_system=self.type_system,
-                     boxkinds=boxkinds,
-                     invent_fail_descr=invent_fail_descr)
+        self.oparse = OpParser(s, self.cpu, self.namespace, 'lltype',
+                               boxkinds,
+                               invent_fail_descr, False, postprocess)
+        return self.oparse.parse()
+
+    def postprocess(self, op):
+        if op.is_guard():
+            op.rd_snapshot = resume.Snapshot(None, op.getfailargs())
+            op.getdescr().rd_frame_info_list = resume.FrameInfo(None, "code", 11)
 
     def add_guard_future_condition(self, res):
         # invent a GUARD_FUTURE_CONDITION to not have to change all tests
         if res.operations[-1].getopnum() == rop.JUMP:
-            guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None, descr=self.invent_fail_descr(None, -1, []))
+            guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None, descr=self.invent_fail_descr(None, rop.GUARD_FUTURE_CONDITION, []))
+            guard.rd_snapshot = resume.Snapshot(None, [])
             res.operations.insert(-1, guard)
 
     def invent_fail_descr(self, model, opnum, fail_args):
-        if fail_args is None:
-            return None
-        descr = Storage()
-        descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-        descr.rd_snapshot = resume.Snapshot(None, _sortboxes(fail_args))
+        if rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST:
+            descr = Storage()
+            descr.guard_opnum = opnum
+            descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
+        else:
+            descr = final_descr
         return descr
 
     def assert_equal(self, optimized, expected, text_right=None):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -15,6 +15,7 @@
 from rpython.jit.metainterp.history import TreeLoop, JitCellToken
 from rpython.jit.metainterp.optimizeopt.test.test_optimizeopt import FakeMetaInterpStaticData
 from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.metainterp import resume
 
 class BaseTestGenerateGuards(BaseTest):
 
@@ -804,8 +805,9 @@
     def optimize_bridge(self, loops, bridge, expected, expected_target='Loop', **boxvalues):
         if isinstance(loops, str):
             loops = (loops, )
-        loops = [self.parse(loop) for loop in loops]
-        bridge = self.parse(bridge)
+        loops = [self.parse(loop, postprocess=self.postprocess)
+                 for loop in loops]
+        bridge = self.parse(bridge, postprocess=self.postprocess)
         self.add_guard_future_condition(bridge)
         for loop in loops:
             loop.preamble = self.unroll_and_optimize(loop)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
@@ -13,7 +13,7 @@
 
         def optimize_loop(self, ops, expected, expected_preamble=None,
                           call_pure_results=None, expected_short=None):
-            loop = self.parse(ops)
+            loop = self.parse(ops, postprocess=self.postprocess)
             if expected != "crash!":
                 expected = self.parse(expected)
             if expected_preamble:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,6 +1,5 @@
 import sys
 
-from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
 from rpython.jit.metainterp.inliner import Inliner
 from rpython.jit.metainterp.optimize import InvalidLoop
@@ -8,7 +7,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
         ShortBoxes, BadVirtualState, VirtualStatesCantMatch)
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp
 from rpython.jit.metainterp.resume import Snapshot
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
 
@@ -560,7 +559,10 @@
 
             for guard in extra_guards:
                 if guard.is_guard():
-                    descr = patchguardop.getdescr().clone_if_mutable()
+                    descr = patchguardop.getdescr().clone()
+                    assert isinstance(guard, GuardResOp)
+                    assert isinstance(patchguardop, GuardResOp)
+                    guard.rd_snapshot = patchguardop.rd_snapshot
                     guard.setdescr(descr)
                 self.optimizer.send_extra_operation(guard)
 
@@ -590,7 +592,10 @@
             if newop.is_guard():
                 if not patchguardop:
                     raise InvalidLoop("would like to have short preamble, but it has a guard and there's no guard_future_condition")
-                descr = patchguardop.getdescr().clone_if_mutable()
+                descr = patchguardop.getdescr().clone()
+                assert isinstance(newop, GuardResOp)
+                assert isinstance(patchguardop, GuardResOp)
+                newop.rd_snapshot = patchguardop.rd_snapshot
                 newop.setdescr(descr)
             self.optimizer.send_extra_operation(newop)
             if shop.result in assumed_classes:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.jitprof import EmptyProfiler
 from rpython.jit.metainterp.logger import Logger
 from rpython.jit.metainterp.optimizeopt.util import args_dict
-from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.resoperation import rop, GuardResOp
 from rpython.rlib import nonconst, rstack
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.rlib.debug import have_debug_prints, make_sure_not_resized
@@ -1875,22 +1875,45 @@
         else:
             moreargs = list(extraargs)
         if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
-            resumedescr = compile.ResumeGuardForcedDescr(self.staticdata,
-                                                         self.jitdriver_sd)
+            resumedescr = compile.ResumeGuardForcedDescr()
+            resumedescr._init(self.staticdata, self.jitdriver_sd)
         elif opnum == rop.GUARD_NOT_INVALIDATED:
             resumedescr = compile.ResumeGuardNotInvalidated()
         elif opnum == rop.GUARD_FUTURE_CONDITION:
             resumedescr = compile.ResumeAtPositionDescr()
+        elif opnum == rop.GUARD_VALUE:
+            resumedescr = compile.ResumeGuardValueDescr()
+        elif opnum == rop.GUARD_NONNULL:
+            resumedescr = compile.ResumeGuardNonnullDescr()
+        elif opnum == rop.GUARD_ISNULL:
+            resumedescr = compile.ResumeGuardIsnullDescr()
+        elif opnum == rop.GUARD_NONNULL_CLASS:
+            resumedescr = compile.ResumeGuardNonnullClassDescr()
+        elif opnum == rop.GUARD_CLASS:
+            resumedescr = compile.ResumeGuardClassDescr()
+        elif opnum == rop.GUARD_TRUE:
+            resumedescr = compile.ResumeGuardTrueDescr()
+        elif opnum == rop.GUARD_FALSE:
+            resumedescr = compile.ResumeGuardFalseDescr()
+        elif opnum == rop.GUARD_EXCEPTION:
+            resumedescr = compile.ResumeGuardExceptionDescr()
+        elif opnum == rop.GUARD_NO_EXCEPTION:
+            resumedescr = compile.ResumeGuardNoExceptionDescr()
+        elif opnum == rop.GUARD_OVERFLOW:
+            resumedescr = compile.ResumeGuardOverflowDescr()
+        elif opnum == rop.GUARD_NO_OVERFLOW:
+            resumedescr = compile.ResumeGuardNoOverflowDescr()
         else:
-            resumedescr = compile.ResumeGuardDescr()
+            assert False
         guard_op = self.history.record(opnum, moreargs, None, descr=resumedescr)
-        self.capture_resumedata(resumedescr, resumepc)
+        assert isinstance(guard_op, GuardResOp)
+        self.capture_resumedata(guard_op, resumedescr, resumepc)
         self.staticdata.profiler.count_ops(opnum, Counters.GUARDS)
         # count
         self.attach_debug_info(guard_op)
         return guard_op
 
-    def capture_resumedata(self, resumedescr, resumepc=-1):
+    def capture_resumedata(self, guard_op, resumedescr, resumepc=-1):
         virtualizable_boxes = None
         if (self.jitdriver_sd.virtualizable_info is not None or
             self.jitdriver_sd.greenfield_info is not None):
@@ -1902,7 +1925,7 @@
             if resumepc >= 0:
                 frame.pc = resumepc
         resume.capture_resumedata(self.framestack, virtualizable_boxes,
-                                  self.virtualref_boxes, resumedescr)
+                                  self.virtualref_boxes, resumedescr, guard_op)
         if self.framestack:
             self.framestack[-1].pc = saved_pc
 
@@ -2163,13 +2186,8 @@
         self.seen_loop_header_for_jdindex = -1
         if isinstance(key, compile.ResumeAtPositionDescr):
             self.seen_loop_header_for_jdindex = self.jitdriver_sd.index
-            dont_change_position = True
-        else:
-            dont_change_position = False
         try:
-            self.prepare_resume_from_failure(key.guard_opnum,
-                                             dont_change_position,
-                                             deadframe)
+            self.prepare_resume_from_failure(key.guard_opnum, deadframe)
             if self.resumekey_original_loop_token is None:   # very rare case
                 raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
             self.interpret()
@@ -2312,12 +2330,12 @@
             else: assert 0
         self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args)
 
-    def prepare_resume_from_failure(self, opnum, dont_change_position,
-                                    deadframe):
+    def prepare_resume_from_failure(self, opnum, deadframe):
         frame = self.framestack[-1]
-        if opnum == rop.GUARD_TRUE:     # a goto_if_not that jumps only now
-            if not dont_change_position:
-                frame.pc = frame.jitcode.follow_jump(frame.pc)
+        if opnum == rop.GUARD_FUTURE_CONDITION:
+            pass
+        elif opnum == rop.GUARD_TRUE:     # a goto_if_not that jumps only now
+            frame.pc = frame.jitcode.follow_jump(frame.pc)
         elif opnum == rop.GUARD_FALSE:     # a goto_if_not that stops jumping;
             pass                  # or a switch that was in its "default" case
         elif opnum == rop.GUARD_VALUE or opnum == rop.GUARD_CLASS:
@@ -2341,12 +2359,11 @@
             pass # XXX we want to do something special in resume descr,
                  # but not now
         elif opnum == rop.GUARD_NO_OVERFLOW:   # an overflow now detected
-            if not dont_change_position:
-                self.execute_raised(OverflowError(), constant=True)
-                try:
-                    self.finishframe_exception()
-                except ChangeFrame:
-                    pass
+            self.execute_raised(OverflowError(), constant=True)
+            try:
+                self.finishframe_exception()
+            except ChangeFrame:
+                pass
         elif opnum == rop.GUARD_OVERFLOW:      # no longer overflowing
             self.clear_exception()
         else:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -91,8 +91,6 @@
     def clone(self):
         args = self.getarglist()
         descr = self.getdescr()
-        if descr is not None:
-            descr = descr.clone_if_mutable()
         op = ResOperation(self.getopnum(), args[:], self.result, descr)
         if not we_are_translated():
             op.name = self.name
@@ -217,6 +215,8 @@
 
     _fail_args = None
 
+    rd_snapshot = None
+
     def getfailargs(self):
         return self._fail_args
 
@@ -225,12 +225,20 @@
 
     def copy_and_change(self, opnum, args=None, result=None, descr=None):
         newop = AbstractResOp.copy_and_change(self, opnum, args, result, descr)
+        assert isinstance(newop, GuardResOp)
         newop.setfailargs(self.getfailargs())
+        newop.rd_snapshot = self.rd_snapshot
         return newop
 
     def clone(self):
         newop = AbstractResOp.clone(self)
+        assert isinstance(newop, GuardResOp)
         newop.setfailargs(self.getfailargs())
+        newop.rd_snapshot = self.rd_snapshot
+        descr = self.getdescr()
+        if descr is not None: # for short preamble which is a mess
+            newdescr = descr.clone()
+            newop.setdescr(newdescr)
         return newop
 
 # ============
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -55,7 +55,7 @@
                                          back.get_list_of_active_boxes(True))
 
 def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes,
-                       storage):
+                       storage, snapshot_storage):
     n = len(framestack) - 1
     if virtualizable_boxes is not None:
         boxes = virtualref_boxes + virtualizable_boxes
@@ -70,10 +70,10 @@
         snapshot = Snapshot(top.parent_resumedata_snapshot,
                             top.get_list_of_active_boxes(False))
         snapshot = Snapshot(snapshot, boxes)
-        storage.rd_snapshot = snapshot
+        snapshot_storage.rd_snapshot = snapshot
     else:
         storage.rd_frame_info_list = None
-        storage.rd_snapshot = Snapshot(None, boxes)
+        snapshot_storage.rd_snapshot = Snapshot(None, boxes)
 
 #
 # The following is equivalent to the RPython-level declaration:
@@ -270,8 +270,9 @@
 
 class ResumeDataVirtualAdder(VirtualVisitor):
 
-    def __init__(self, storage, memo):
+    def __init__(self, storage, snapshot_storage, memo):
         self.storage = storage
+        self.snapshot_storage = snapshot_storage
         self.memo = memo
 
     def make_virtual_info(self, value, fieldnums):
@@ -357,13 +358,13 @@
         storage = self.storage
         # make sure that nobody attached resume data to this guard yet
         assert not storage.rd_numb
-        snapshot = storage.rd_snapshot
+        snapshot = self.snapshot_storage.rd_snapshot
         assert snapshot is not None # is that true?
         numb, liveboxes_from_env, v = self.memo.number(optimizer, snapshot)
         self.liveboxes_from_env = liveboxes_from_env
         self.liveboxes = {}
         storage.rd_numb = numb
-        storage.rd_snapshot = None
+        self.snapshot_storage.rd_snapshot = None
 
         # collect liveboxes and virtuals
         n = len(liveboxes_from_env) - v
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -123,7 +123,7 @@
     class FakeVirtualValue(AbstractVirtualValue):
         def visitor_dispatch_virtual_type(self, *args):
             return FakeVInfo()
-    modifier = ResumeDataVirtualAdder(None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None)
     v1 = FakeVirtualValue(None, None)
     vinfo1 = modifier.make_virtual_info(v1, [1, 2, 4])
     vinfo2 = modifier.make_virtual_info(v1, [1, 2, 4])
@@ -542,7 +542,7 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2)]
 
     storage = Storage()
-    capture_resumedata(fs, None, [], storage)
+    capture_resumedata(fs, None, [], storage, storage)
 
     assert fs[0].parent_resumedata_snapshot is None
     assert fs[0].parent_resumedata_frame_info_list is None
@@ -558,7 +558,7 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2),
           FakeFrame("code1", 3, b3, c2, b1),
           FakeFrame("code2", 9, c3, b2)]
-    capture_resumedata(fs, None, [], storage)
+    capture_resumedata(fs, None, [], storage, storage)
 
     frame_info_list = storage.rd_frame_info_list
     assert frame_info_list.prev is fs[2].parent_resumedata_frame_info_list
@@ -588,7 +588,7 @@
     fs[2].pc = 15
     vbs = [b1, b2]
     vrs = [b3]
-    capture_resumedata(fs, vbs, vrs, storage)
+    capture_resumedata(fs, vbs, vrs, storage, storage)
 
     frame_info_list = storage.rd_frame_info_list
     assert frame_info_list.prev is fs[2].parent_resumedata_frame_info_list
@@ -616,9 +616,9 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2),
           FakeFrame("code1", 3, b3, c2, b1),
           FakeFrame("code2", 9, c3, b2)]
-    capture_resumedata(fs, None, [], storage)
+    capture_resumedata(fs, None, [], storage, storage)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     metainterp = MyMetaInterp()
 
@@ -640,7 +640,7 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2),
           FakeFrame("code1", 3, b3, c2, b1),
           FakeFrame("code2", 9, c3, b2)]
-    capture_resumedata(fs, [b4], [], storage)
+    capture_resumedata(fs, [b4], [], storage, storage)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     modifier = ResumeDataVirtualAdder(storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
@@ -665,10 +665,10 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2),
           FakeFrame("code1", 3, b3, c2, b1),
           FakeFrame("code2", 9, c3, b2)]
-    capture_resumedata(fs, None, [], storage)
+    capture_resumedata(fs, None, [], storage, storage)
     storage2 = Storage()
     fs = fs[:-1] + [FakeFrame("code2", 10, c3, b2, b4)]
-    capture_resumedata(fs, None, [], storage2)
+    capture_resumedata(fs, None, [], storage2, storage2)
     
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     modifier = ResumeDataVirtualAdder(storage, memo)
@@ -728,10 +728,10 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2),
           FakeFrame("code1", 3, b3, c2, b1),
           FakeFrame("code2", 9, c3, b2)]
-    capture_resumedata(fs, None, [], storage)
+    capture_resumedata(fs, None, [], storage, storage)
     storage2 = Storage()
     fs = fs[:-1] + [FakeFrame("code2", 10, c3, b2, b4)]
-    capture_resumedata(fs, None, [], storage2)
+    capture_resumedata(fs, None, [], storage2, storage2)
     
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     values = {b2: virtual_value(b2, b5, c4)}
@@ -788,7 +788,7 @@
                       LLtypeMixin.nodebox.constbox()]
     storage = Storage()
     fs = [FakeFrame("code0", 0, c1, b2, b3)]
-    capture_resumedata(fs, None, [], storage)
+    capture_resumedata(fs, None, [], storage, storage)
     
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     values = {b2: virtual_value(b2, b5, c4)}
@@ -800,7 +800,7 @@
 
     storage2 = Storage()
     fs = [FakeFrame("code0", 0, b1, b4, b2)]
-    capture_resumedata(fs, None, [], storage2)
+    capture_resumedata(fs, None, [], storage2, storage2)
     values[b4] = virtual_value(b4, b6, c4)
     modifier = ResumeDataVirtualAdder(storage2, memo)
     liveboxes = modifier.finish(FakeOptimizer(values))
@@ -814,7 +814,7 @@
     b1, b2, b3 = [BoxPtr(), BoxPtr(), BoxInt()]
     storage = Storage()
     fs = [FakeFrame("code0", 0, b1, b2)]
-    capture_resumedata(fs, None, [], storage)
+    capture_resumedata(fs, None, [], storage, storage)
     
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     v1 = virtual_value(b1, b3, None)
@@ -1022,7 +1022,7 @@
 def test_register_virtual_fields():
     b1, b2 = BoxInt(), BoxInt()
     vbox = BoxPtr()
-    modifier = ResumeDataVirtualAdder(None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1031,7 +1031,7 @@
                                   b2: UNASSIGNED}
     assert modifier.vfieldboxes == {vbox: [b1, b2]}
 
-    modifier = ResumeDataVirtualAdder(None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None)
     modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1061,7 +1061,7 @@
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())    
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     assert storage.rd_snapshot is None
     cpu = MyCPU([])
@@ -1075,14 +1075,14 @@
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.finish(FakeOptimizer({}))
     assert len(memo.consts) == 2
     assert storage.rd_consts is memo.consts
 
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)]
     storage2 = make_storage(b1s, b2s, b3s)
-    modifier2 = ResumeDataVirtualAdder(storage2, memo)
+    modifier2 = ResumeDataVirtualAdder(storage2, storage2, memo)
     modifier2.finish(FakeOptimizer({}))
     assert len(memo.consts) == 3    
     assert storage2.rd_consts is memo.consts
@@ -1137,7 +1137,7 @@
     b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     b1_2 = BoxInt()
     class FakeValue(object):
 
@@ -1169,7 +1169,7 @@
     b1s = ConstInt(111)
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())        
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     b2t, b3t = [BoxPtr(demo55o), BoxInt(33)]
     newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t)
@@ -1190,7 +1190,7 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1273,7 +1273,7 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1321,7 +1321,7 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1364,7 +1364,7 @@
     b2s, b4s = [BoxPtr(), BoxPtr()]
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1403,7 +1403,7 @@
     class Storage(object):
         pass
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, None)
+    modifier = ResumeDataVirtualAdder(storage, storage, None)
     modifier._add_pending_fields([])
     assert not storage.rd_pendingfields
     #
@@ -1411,7 +1411,7 @@
         pass
     field_a = FieldDescr()
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, None)
+    modifier = ResumeDataVirtualAdder(storage, storage, None)
     modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042),
                                    61: rffi.cast(rffi.SHORT, 1061)}
     modifier._add_pending_fields([(field_a, 42, 61, -1)])
@@ -1425,7 +1425,7 @@
     #
     array_a = FieldDescr()
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, None)
+    modifier = ResumeDataVirtualAdder(storage, storage, None)
     modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042),
                                    61: rffi.cast(rffi.SHORT, 1061),
                                    62: rffi.cast(rffi.SHORT, 1062),
@@ -1506,7 +1506,7 @@
     metainterp_sd = FakeMetaInterpStaticData()
     metainterp_sd.options = options
     memo = ResumeDataLoopMemo(metainterp_sd)
-    modifier = ResumeDataVirtualAdder(None, memo)
+    modifier = ResumeDataVirtualAdder(None, None, memo)
 
     for i in range(5):
         assert not modifier._invalidation_needed(5, i)
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -61,9 +61,10 @@
 
     def __init__(self, input, cpu, namespace, type_system, boxkinds,
                  invent_fail_descr=default_fail_descr,
-                 nonstrict=False):
+                 nonstrict=False, postproces=None):
         self.input = input
         self.vars = {}
+        self._postproces = postproces
         self.cpu = cpu
         self._consts = namespace
         self.type_system = type_system
@@ -237,8 +238,6 @@
                     fail_args.append(fail_arg)
             if descr is None and self.invent_fail_descr:
                 descr = self.invent_fail_descr(self.model, opnum, fail_args)
-            if hasattr(descr, '_oparser_uses_descr_of_guard'):
-                descr._oparser_uses_descr_of_guard(self, fail_args)
         else:
             fail_args = None
             if opnum == rop.FINISH:
@@ -250,7 +249,7 @@
 
         return opnum, args, descr, fail_args
 
-    def create_op(self, opnum, args, result, descr):
+    def create_op(self, opnum, args, result, descr, fail_args):
         if opnum == ESCAPE_OP.OPNUM:
             op = ESCAPE_OP(result)
             op.initarglist(args)
@@ -262,7 +261,12 @@
             assert descr is None
             return op
         else:
-            return ResOperation(opnum, args, result, descr)
+            res = ResOperation(opnum, args, result, descr)
+            if fail_args is not None:
+                res.setfailargs(fail_args)
+            if self._postproces:
+                self._postproces(res)
+            return res
 
     def parse_result_op(self, line):
         res, op = line.split("=", 1)
@@ -273,16 +277,12 @@
             raise ParseError("Double assign to var %s in line: %s" % (res, line))
         rvar = self.box_for_var(res)
         self.vars[res] = rvar
-        res = self.create_op(opnum, args, rvar, descr)
-        if fail_args is not None:
-            res.setfailargs(fail_args)
+        res = self.create_op(opnum, args, rvar, descr, fail_args)
         return res
 
     def parse_op_no_result(self, line):
         opnum, args, descr, fail_args = self.parse_op(line)
-        res = self.create_op(opnum, args, None, descr)
-        if fail_args is not None:
-            res.setfailargs(fail_args)
+        res = self.create_op(opnum, args, None, descr, fail_args)
         return res
 
     def parse_next_op(self, line):
@@ -377,11 +377,12 @@
 
 def parse(input, cpu=None, namespace=None, type_system='lltype',
           boxkinds=None, invent_fail_descr=default_fail_descr,
-          no_namespace=False, nonstrict=False, OpParser=OpParser):
+          no_namespace=False, nonstrict=False, OpParser=OpParser,
+          postprocess=None):
     if namespace is None and not no_namespace:
         namespace = {}
     return OpParser(input, cpu, namespace, type_system, boxkinds,
-                    invent_fail_descr, nonstrict).parse()
+                    invent_fail_descr, nonstrict, postprocess).parse()
 
 def pure_parse(*args, **kwds):
     kwds['invent_fail_descr'] = None

From noreply at buildbot.pypy.org  Wed Dec 24 11:56:03 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 24 Dec 2014 11:56:03 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: Make adr_jump_offset
 official and put it in attrs, I did not realize the
Message-ID: <20141224105603.1FB061C10F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75101:8a5a8258f0a8
Date: 2014-12-24 12:55 +0200
http://bitbucket.org/pypy/pypy/changeset/8a5a8258f0a8/

Log:	Make adr_jump_offset official and put it in attrs, I did not realize
	the extent to which backends store garbage on the descrs, next step
	is to fight it

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -901,7 +901,7 @@
             descr = tok.faildescr
             assert isinstance(descr, AbstractFailDescr)
             failure_recovery_pos = block_start + tok.pos_recovery_stub
-            descr._arm_failure_recovery_block = failure_recovery_pos
+            descr.adr_jump_offset = failure_recovery_pos
             relative_offset = tok.pos_recovery_stub - tok.offset
             guard_pos = block_start + tok.offset
             if not tok.is_guard_not_invalidated:
@@ -968,11 +968,11 @@
 
     def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
         b = InstrBuilder(self.cpu.cpuinfo.arch_version)
-        patch_addr = faildescr._arm_failure_recovery_block
+        patch_addr = faildescr.adr_jump_offset
         assert patch_addr != 0
         b.B(bridge_addr)
         b.copy_to_raw_memory(patch_addr)
-        faildescr._arm_failure_recovery_block = 0
+        faildescr.adr_jump_offset = 0
 
     # regalloc support
     def load(self, loc, value):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -572,13 +572,13 @@
 
     def patch_pending_failure_recoveries(self, rawstart):
         # after we wrote the assembler to raw memory, set up
-        # tok.faildescr._x86_adr_jump_offset to contain the raw address of
+        # tok.faildescr.adr_jump_offset to contain the raw address of
         # the 4-byte target field in the JMP/Jcond instruction, and patch
         # the field in question to point (initially) to the recovery stub
         clt = self.current_clt
         for tok in self.pending_guard_tokens:
             addr = rawstart + tok.pos_jump_offset
-            tok.faildescr._x86_adr_jump_offset = addr
+            tok.faildescr.adr_jump_offset = addr
             relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4)
             assert rx86.fits_in_32bits(relative_target)
             #
@@ -685,7 +685,7 @@
                                    self.cpu.gc_ll_descr.gcrootmap)
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
-        adr_jump_offset = faildescr._x86_adr_jump_offset
+        adr_jump_offset = faildescr.adr_jump_offset
         assert adr_jump_offset != 0
         offset = adr_new_target - (adr_jump_offset + 4)
         # If the new target fits within a rel32 of the jump, just patch
@@ -705,7 +705,7 @@
             p = rffi.cast(rffi.INTP, adr_jump_offset)
             adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0])
             mc.copy_to_raw_memory(adr_target)
-        faildescr._x86_adr_jump_offset = 0    # means "patched"
+        faildescr.adr_jump_offset = 0    # means "patched"
 
     def fixup_target_tokens(self, rawstart):
         for targettoken in self.target_tokens_currently_compiling:
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -151,7 +151,7 @@
     index = -1
     final_descr = False
 
-    _attrs_ = ()
+    _attrs_ = ('adr_jump_offset',)
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         raise NotImplementedError

From noreply at buildbot.pypy.org  Wed Dec 24 11:57:58 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 24 Dec 2014 11:57:58 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: give up for now
Message-ID: <20141224105758.D750A1C33C1@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75102:0b63bcf4575c
Date: 2014-12-24 12:57 +0200
http://bitbucket.org/pypy/pypy/changeset/0b63bcf4575c/

Log:	give up for now

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -151,7 +151,7 @@
     index = -1
     final_descr = False
 
-    _attrs_ = ('adr_jump_offset',)
+    _attrs_ = ('adr_jump_offset', 'rd_locs', 'rd_loop_token')
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         raise NotImplementedError

From noreply at buildbot.pypy.org  Wed Dec 24 12:55:41 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 24 Dec 2014 12:55:41 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: Comment out the wref
 stuff, I'm not sure what it does (but there is a strong
Message-ID: <20141224115541.3ED271D28DF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75103:4b641be64135
Date: 2014-12-24 13:55 +0200
http://bitbucket.org/pypy/pypy/changeset/4b641be64135/

Log:	Comment out the wref stuff, I'm not sure what it does (but there is
	a strong reference stored by the backend anyway)

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -65,12 +65,14 @@
     wref = weakref.ref(original_jitcell_token)
     for op in loop.operations:
         descr = op.getdescr()
-        if isinstance(descr, ResumeDescr):
-            descr.wref_original_loop_token = wref   # stick it there
-            n = descr.index
-            if n >= 0:       # we also record the resumedescr number
-                original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
-        elif isinstance(descr, JitCellToken):
+        # not sure what descr.index is about
+        #if isinstance(descr, ResumeDescr):
+            #descr.wref_original_loop_token = wref   # stick it there
+            #n = descr.index
+            #if n >= 0:       # we also record the resumedescr number
+            #    original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
+        #    pass
+        if isinstance(descr, JitCellToken):
             # for a CALL_ASSEMBLER: record it as a potential jump.
             if descr is not original_jitcell_token:
                 original_jitcell_token.record_jump_to(descr)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2170,7 +2170,8 @@
         assert isinstance(key, compile.ResumeGuardDescr)
         # store the resumekey.wref_original_loop_token() on 'self' to make
         # sure that it stays alive as long as this MetaInterp
-        self.resumekey_original_loop_token = key.wref_original_loop_token()
+        # XXXX this is stored these days via rd_loop_token on resumedescr
+        #self.resumekey_original_loop_token = key.wref_original_loop_token()
         self.staticdata.try_to_free_some_loops()
         self.initialize_state_from_guard_failure(key, deadframe)
         try:

From noreply at buildbot.pypy.org  Wed Dec 24 16:27:34 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Dec 2014 16:27:34 +0100 (CET)
Subject: [pypy-commit] cffi default: Add all standard types from stdint.h.
 Note that it still fails with
Message-ID: <20141224152735.004511D22DE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1598:ecf4fb44f740
Date: 2014-12-24 16:27 +0100
http://bitbucket.org/cffi/cffi/changeset/ecf4fb44f740/

Log:	Add all standard types from stdint.h. Note that it still fails with
	NotImplementedError if you try to use one of them that would be
	defined, on your platform, as an integer of size not in (1, 2, 4,
	8).

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -27,6 +27,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include <stdint.h>
 # endif
@@ -3323,8 +3341,26 @@
        EPTYPE(u32, uint32_t, CT_PRIMITIVE_UNSIGNED)             \
        EPTYPE(i64, int64_t, CT_PRIMITIVE_SIGNED)                \
        EPTYPE(u64, uint64_t, CT_PRIMITIVE_UNSIGNED)             \
+       EPTYPE(il8, int_least8_t, CT_PRIMITIVE_SIGNED)           \
+       EPTYPE(ul8, uint_least8_t, CT_PRIMITIVE_UNSIGNED)        \
+       EPTYPE(il16, int_least16_t, CT_PRIMITIVE_SIGNED)         \
+       EPTYPE(ul16, uint_least16_t, CT_PRIMITIVE_UNSIGNED)      \
+       EPTYPE(il32, int_least32_t, CT_PRIMITIVE_SIGNED)         \
+       EPTYPE(ul32, uint_least32_t, CT_PRIMITIVE_UNSIGNED)      \
+       EPTYPE(il64, int_least64_t, CT_PRIMITIVE_SIGNED)         \
+       EPTYPE(ul64, uint_least64_t, CT_PRIMITIVE_UNSIGNED)      \
+       EPTYPE(if8, int_fast8_t, CT_PRIMITIVE_SIGNED)            \
+       EPTYPE(uf8, uint_fast8_t, CT_PRIMITIVE_UNSIGNED)         \
+       EPTYPE(if16, int_fast16_t, CT_PRIMITIVE_SIGNED)          \
+       EPTYPE(uf16, uint_fast16_t, CT_PRIMITIVE_UNSIGNED)       \
+       EPTYPE(if32, int_fast32_t, CT_PRIMITIVE_SIGNED)          \
+       EPTYPE(uf32, uint_fast32_t, CT_PRIMITIVE_UNSIGNED)       \
+       EPTYPE(if64, int_fast64_t, CT_PRIMITIVE_SIGNED)          \
+       EPTYPE(uf64, uint_fast64_t, CT_PRIMITIVE_UNSIGNED)       \
        EPTYPE(ip, intptr_t, CT_PRIMITIVE_SIGNED)                \
        EPTYPE(up, uintptr_t, CT_PRIMITIVE_UNSIGNED)             \
+       EPTYPE(im, intmax_t, CT_PRIMITIVE_SIGNED)                \
+       EPTYPE(um, uintmax_t, CT_PRIMITIVE_UNSIGNED)             \
        EPTYPE(pd, ptrdiff_t, CT_PRIMITIVE_SIGNED)               \
        EPTYPE(sz, size_t, CT_PRIMITIVE_UNSIGNED)                \
        EPTYPE(ssz, ssize_t, CT_PRIMITIVE_SIGNED)
@@ -3430,7 +3466,8 @@
 
  bad_ffi_type:
     PyErr_Format(PyExc_NotImplementedError,
-                 "primitive type '%s' with a non-standard size %d",
+                 "primitive type '%s' has size %d; "
+                 "the supported sizes are 1, 2, 4, 8",
                  name, (int)ptypes->size);
     return NULL;
 }
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2729,7 +2729,16 @@
 def test_nonstandard_integer_types():
     for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t',
                      'uint32_t', 'int64_t', 'uint64_t', 'intptr_t',
-                     'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t']:
+                     'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t',
+                     'int_least8_t',  'uint_least8_t',
+                     'int_least16_t', 'uint_least16_t',
+                     'int_least32_t', 'uint_least32_t',
+                     'int_least64_t', 'uint_least64_t',
+                     'int_fast8_t',  'uint_fast8_t',
+                     'int_fast16_t', 'uint_fast16_t',
+                     'int_fast32_t', 'uint_fast32_t',
+                     'int_fast64_t', 'uint_fast64_t',
+                     'intmax_t', 'uintmax_t']:
         new_primitive_type(typename)    # works
 
 def test_cannot_convert_unicode_to_charp():

From noreply at buildbot.pypy.org  Wed Dec 24 16:32:06 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Dec 2014 16:32:06 +0100 (CET)
Subject: [pypy-commit] pypy default: Add some "jit.dont_look_inside" on
 these Win32 functions.
Message-ID: <20141224153206.7E0961D287A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75104:ed493f85222e
Date: 2014-12-24 16:32 +0100
http://bitbucket.org/pypy/pypy/changeset/ed493f85222e/

Log:	Add some "jit.dont_look_inside" on these Win32 functions.

diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py
--- a/rpython/rlib/rwin32.py
+++ b/rpython/rlib/rwin32.py
@@ -123,8 +123,10 @@
     _SetLastError = winexternal('SetLastError', [DWORD], lltype.Void,
                                 _nowrapper=True, sandboxsafe=True)
 
+    @jit.dont_look_inside
     def GetLastError():
         return rffi.cast(lltype.Signed, _GetLastError())
+    @jit.dont_look_inside
     def SetLastError(err):
         _SetLastError(rffi.cast(DWORD, err))
 

From noreply at buildbot.pypy.org  Wed Dec 24 16:51:55 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Dec 2014 16:51:55 +0100 (CET)
Subject: [pypy-commit] cffi default: Document ecf4fb44f740.
Message-ID: <20141224155155.43EDC1D28DF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1599:c3d175e55b8b
Date: 2014-12-24 16:51 +0100
http://bitbucket.org/cffi/cffi/changeset/c3d175e55b8b/

Log:	Document ecf4fb44f740.

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -435,12 +435,17 @@
 * *New in version 0.6:* all `common Windows types`_ are defined if you run
   on Windows (``DWORD``, ``LPARAM``, etc.).
 
+* *New in version 0.8.7:* the other standard integer types from stdint.h,
+  as long as they map to integers of 1, 2, 4 or 8 bytes.  Larger integers
+  are not supported.
+
 .. _`common Windows types`: http://msdn.microsoft.com/en-us/library/windows/desktop/aa383751%28v=vs.85%29.aspx
 
 .. "versionadded:: 0.4": _Bool
 .. "versionadded:: 0.6": bool
 .. "versionadded:: 0.4": FILE
 .. "versionadded:: 0.6": Wintypes
+.. "versionadded:: 0.8.7": intmax_t etc.
 
 As we will see on `the verification step`_ below, the declarations can
 also contain "``...``" at various places; these are placeholders that will

From noreply at buildbot.pypy.org  Wed Dec 24 16:51:56 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Dec 2014 16:51:56 +0100 (CET)
Subject: [pypy-commit] cffi default: Add the keyword "relative_to=..." to
	verify(), as per
Message-ID: <20141224155156.93FF61D28DF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1600:b97105235b12
Date: 2014-12-24 16:52 +0100
http://bitbucket.org/cffi/cffi/changeset/b97105235b12/

Log:	Add the keyword "relative_to=..." to verify(), as per
	https://bitbucket.org/cffi/cffi/pull-request/53

diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py
--- a/cffi/ffiplatform.py
+++ b/cffi/ffiplatform.py
@@ -11,6 +11,9 @@
     """
 
 
+LIST_OF_FILE_NAMES = ['include_dirs', 'library_dirs',
+                      'extra_objects', 'depends']
+
 def get_extension(srcfilename, modname, sources=(), **kwds):
     from distutils.core import Extension
     allsources = [srcfilename]
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -17,7 +17,7 @@
 
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
                  ext_package=None, tag='', force_generic_engine=False,
-                 source_extension='.c', flags=None, **kwds):
+                 source_extension='.c', flags=None, relative_to=None, **kwds):
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
@@ -26,7 +26,7 @@
         self._vengine = vengine_class(self)
         self._vengine.patch_extension_kwds(kwds)
         self.flags = flags
-        self.kwds = kwds
+        self.kwds = self.make_relative_to(kwds, relative_to)
         #
         if modulename:
             if tag:
@@ -109,6 +109,20 @@
     def generates_python_module(self):
         return self._vengine._gen_python_module
 
+    def make_relative_to(self, kwds, relative_to):
+        if relative_to and os.path.dirname(relative_to):
+            dirname = os.path.dirname(relative_to)
+            kwds = kwds.copy()
+            for key in ffiplatform.LIST_OF_FILE_NAMES:
+                if key in kwds:
+                    lst = kwds[key]
+                    if not isinstance(lst, (list, tuple)):
+                        raise TypeError("keyword '%s' should be a list or tuple"
+                                        % (key,))
+                    lst = [os.path.join(dirname, fn) for fn in lst]
+                    kwds[key] = lst
+        return kwds
+
     # ----------
 
     def _locate_module(self):
diff --git a/testing/test_verify.py b/testing/test_verify.py
--- a/testing/test_verify.py
+++ b/testing/test_verify.py
@@ -1349,7 +1349,7 @@
     """)
 
 def test_tmpdir():
-    import tempfile, os, shutil
+    import tempfile, os
     from testing.udir import udir
     tmpdir = tempfile.mkdtemp(dir=str(udir))
     ffi = FFI()
@@ -1358,6 +1358,20 @@
     assert os.listdir(tmpdir)
     assert lib.foo(100) == 142
 
+def test_relative_to():
+    import tempfile, os
+    from testing.udir import udir
+    tmpdir = tempfile.mkdtemp(dir=str(udir))
+    ffi = FFI()
+    ffi.cdef("int foo(int);")
+    f = open(os.path.join(tmpdir, 'foo.h'), 'w')
+    print >> f, "int foo(int a) { return a + 42; }"
+    f.close()
+    lib = ffi.verify('#include "foo.h"',
+                     include_dirs=['.'],
+                     relative_to=os.path.join(tmpdir, 'x'))
+    assert lib.foo(100) == 142
+
 def test_bug1():
     ffi = FFI()
     ffi.cdef("""

From noreply at buildbot.pypy.org  Wed Dec 24 16:57:54 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Dec 2014 16:57:54 +0100 (CET)
Subject: [pypy-commit] pypy default: Test and fix
Message-ID: <20141224155754.8A3681D28DF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75105:b950155cdd8a
Date: 2014-12-24 16:57 +0100
http://bitbucket.org/pypy/pypy/changeset/b950155cdd8a/

Log:	Test and fix

diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -254,7 +254,10 @@
         return self.__class__(can_be_None=False, no_nul=self.no_nul)
 
     def nonnulify(self):
-        return self.__class__(can_be_None=self.can_be_None, no_nul=True)
+        if self.can_be_None:
+            return self.__class__(can_be_None=True, no_nul=True)
+        else:
+            return self.__class__(no_nul=True)
 
 
 class SomeString(SomeStringOrUnicode):
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4326,6 +4326,13 @@
         assert isinstance(s, annmodel.SomeString)
         assert not s.can_be_none()
 
+    def test_nonnulify(self):
+        s = annmodel.SomeString(can_be_None=True).nonnulify()
+        assert s.can_be_None is True
+        assert s.no_nul is True
+        s = annmodel.SomeChar().nonnulify()
+        assert s.no_nul is True
+
 
 def g(n):
     return [0, 1, 2, n]

From noreply at buildbot.pypy.org  Wed Dec 24 17:24:48 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Dec 2014 17:24:48 +0100 (CET)
Subject: [pypy-commit] cffi default: Update the copies of that block of #ifs
Message-ID: <20141224162448.7327A1C0501@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1601:cef8630f9d37
Date: 2014-12-24 17:25 +0100
http://bitbucket.org/cffi/cffi/changeset/cef8630f9d37/

Log:	Update the copies of that block of #ifs

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -797,6 +797,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include <stdint.h>
 # endif
diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -566,6 +566,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include <stdint.h>
 # endif

From noreply at buildbot.pypy.org  Wed Dec 24 17:40:02 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Dec 2014 17:40:02 +0100 (CET)
Subject: [pypy-commit] cffi default: Ignore errors here
Message-ID: <20141224164002.F03C81C33C1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1602:957d8e48d4a4
Date: 2014-12-24 17:40 +0100
http://bitbucket.org/cffi/cffi/changeset/957d8e48d4a4/

Log:	Ignore errors here

diff --git a/testing/test_zdistutils.py b/testing/test_zdistutils.py
--- a/testing/test_zdistutils.py
+++ b/testing/test_zdistutils.py
@@ -17,7 +17,7 @@
 
     def teardown_class(self):
         if udir.isdir():
-            udir.remove()
+            udir.remove(ignore_errors=True)
 
     def test_locate_engine_class(self):
         cls = _locate_engine_class(FFI(), self.generic)

From noreply at buildbot.pypy.org  Wed Dec 24 17:41:51 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Dec 2014 17:41:51 +0100 (CET)
Subject: [pypy-commit] cffi default: ignore_errors here too
Message-ID: <20141224164151.974B91C33C1@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1603:a3896b35cbf8
Date: 2014-12-24 17:42 +0100
http://bitbucket.org/cffi/cffi/changeset/a3896b35cbf8/

Log:	ignore_errors here too

diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py
--- a/testing/test_zintegration.py
+++ b/testing/test_zintegration.py
@@ -75,7 +75,7 @@
 class TestZIntegration(object):
     def teardown_class(self):
         if udir.isdir():
-            udir.remove()
+            udir.remove(ignore_errors=True)
 
     def test_infrastructure(self):
         run_setup_and_program('infrastructure', '''

From noreply at buildbot.pypy.org  Wed Dec 24 18:00:54 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 24 Dec 2014 18:00:54 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: merge deault into branch
Message-ID: <20141224170054.6609B1D287A@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r75106:f649f767bd94
Date: 2014-12-24 18:59 +0200
http://bitbucket.org/pypy/pypy/changeset/f649f767bd94/

Log:	merge deault into branch

diff too long, truncating to 2000 out of 9967 lines

diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -478,6 +478,29 @@
         except TypeError:
             pass
 
+    def CheckCurDescription(self):
+        self.cu.execute("select * from test")
+
+        actual = self.cu.description
+        expected = [
+            ('id', None, None, None, None, None, None),
+            ('name', None, None, None, None, None, None),
+            ('income', None, None, None, None, None, None),
+        ]
+        self.assertEqual(expected, actual)
+
+    def CheckCurDescriptionVoidStatement(self):
+        self.cu.execute("insert into test(name) values (?)", ("foo",))
+        self.assertIsNone(self.cu.description)
+
+    def CheckCurDescriptionWithoutStatement(self):
+        cu = self.cx.cursor()
+        try:
+            self.assertIsNone(cu.description)
+        finally:
+            cu.close()
+
+
 @unittest.skipUnless(threading, 'This test requires threading.')
 class ThreadTests(unittest.TestCase):
     def setUp(self):
diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -1108,6 +1108,16 @@
             od.popitem()
         self.assertEqual(len(od), 0)
 
+    def test_popitem_first(self):
+        pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
+        shuffle(pairs)
+        od = OrderedDict(pairs)
+        while pairs:
+            self.assertEqual(od.popitem(last=False), pairs.pop(0))
+        with self.assertRaises(KeyError):
+            od.popitem(last=False)
+        self.assertEqual(len(od), 0)
+
     def test_pop(self):
         pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
         shuffle(pairs)
@@ -1179,7 +1189,11 @@
         od = OrderedDict(pairs)
         # yaml.dump(od) -->
         # '!!python/object/apply:__main__.OrderedDict\n- - [a, 1]\n  - [b, 2]\n'
-        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1]))
+
+        # PyPy bug fix: added [0] at the end of this line, because the
+        # test is really about the 2-tuples that need to be 2-lists
+        # inside the list of 6 of them
+        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1][0]))
 
     def test_reduce_not_too_fat(self):
         # do not save instance dictionary if not needed
@@ -1189,6 +1203,16 @@
         od.x = 10
         self.assertEqual(len(od.__reduce__()), 3)
 
+    def test_reduce_exact_output(self):
+        # PyPy: test that __reduce__() produces the exact same answer as
+        # CPython does, even though in the 'all_ordered_dicts' branch we
+        # have to emulate it.
+        pairs = [['c', 1], ['b', 2], ['d', 4]]
+        od = OrderedDict(pairs)
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,)))
+        od.x = 10
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,), {'x': 10}))
+
     def test_repr(self):
         od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)])
         self.assertEqual(repr(od),
diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -225,9 +225,9 @@
     >>> element.remove(subelement)
     >>> serialize(element) # 5
     '<tag key="value" />'
-    >>> element.remove(subelement)
+    >>> element.remove(subelement)    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ValueError: list.remove(x): x not in list
+    ValueError: list.remove(x): ...
     >>> serialize(element) # 6
     '<tag key="value" />'
     >>> element[0:0] = [subelement, subelement, subelement]
diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt
--- a/lib-python/stdlib-upgrade.txt
+++ b/lib-python/stdlib-upgrade.txt
@@ -7,7 +7,7 @@
 
 1. check out the branch vendor/stdlib
 2. upgrade the files there
-3. update stdlib-versions.txt with the output of hg -id from the cpython repo
+3. update stdlib-version.txt with the output of hg -id from the cpython repo
 4. commit
 5. update to default/py3k
 6. create a integration branch for the new stdlib
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1175,8 +1175,9 @@
         try:
             return self.__description
         except AttributeError:
-            self.__description = self.__statement._get_description()
-            return self.__description
+            if self.__statement:
+                self.__description = self.__statement._get_description()
+                return self.__description
     description = property(__get_description)
 
     def __get_lastrowid(self):
diff --git a/lib_pypy/readline.py b/lib_pypy/readline.py
--- a/lib_pypy/readline.py
+++ b/lib_pypy/readline.py
@@ -6,4 +6,11 @@
 are only stubs at the moment.
 """
 
-from pyrepl.readline import *
+try:
+    from pyrepl.readline import *
+except ImportError:
+    import sys
+    if sys.platform == 'win32':
+        raise ImportError("the 'readline' module is not available on Windows"
+                          " (on either PyPy or CPython)")
+    raise
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,6 +6,10 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
+**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
+on linux, linux64 and windows. We will make it the default on all platforms
+by the time of the next release.
+
 The first thing that you need is to compile PyPy yourself with the option
 ``--shared``. We plan to make ``--shared`` the default in the future. Consult
 the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -35,6 +35,13 @@
 PyPy's bytearray type is very inefficient. It would be an interesting
 task to look into possible optimizations on this.
 
+Implement AF_XXX packet types for PyPy
+--------------------------------------
+
+PyPy is missing AF_XXX types of sockets. Implementing it is easy-to-medium
+task. `bug report`_
+
+.. _`bug report`: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets#more
 
 Implement copy-on-write list slicing
 ------------------------------------
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -125,6 +125,8 @@
         else:
             return self.space.builtin
 
+    _NO_CELLS = []
+
     @jit.unroll_safe
     def initialize_frame_scopes(self, outer_func, code):
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
@@ -143,7 +145,7 @@
         nfreevars = len(code.co_freevars)
         if not nfreevars:
             if not ncellvars:
-                self.cells = []
+                self.cells = self._NO_CELLS
                 return            # no self.cells needed - fast path
         elif outer_func is None:
             space = self.space
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -397,7 +397,7 @@
 def test_invalid_indexing():
     p = new_primitive_type("int")
     x = cast(p, 42)
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises(TypeError, "x[0]")
 
 def test_default_str():
     BChar = new_primitive_type("char")
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -14,6 +14,7 @@
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.module._rawffi.alt.type_converter import FromAppLevelConverter, ToAppLevelConverter
 from pypy.module._rawffi.interp_rawffi import got_libffi_error, wrap_dlopenerror
+from pypy.module._rawffi import lasterror
 
 import os
 if os.name == 'nt':
@@ -201,11 +202,23 @@
         self.func = func
         self.argchain = argchain
 
+    def before(self):
+        lasterror.restore_last_error(self.space)
+
+    def after(self):
+        lasterror.save_last_error(self.space)
+
     def get_longlong(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.LONGLONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.LONGLONG)
+        self.after()
+        return x
 
     def get_ulonglong(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.ULONGLONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.ULONGLONG)
+        self.after()
+        return x
 
     def get_signed(self, w_ffitype):
         # if the declared return type of the function is smaller than LONG,
@@ -216,64 +229,94 @@
         # to space.wrap in order to get a nice applevel <int>.
         #
         restype = w_ffitype.get_ffitype()
+        self.before()
         call = self.func.call
         if restype is libffi.types.slong:
-            return call(self.argchain, rffi.LONG)
+            x = call(self.argchain, rffi.LONG)
         elif restype is libffi.types.sint:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.INT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.INT))
         elif restype is libffi.types.sshort:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT))
         elif restype is libffi.types.schar:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR))
         else:
-            self.error(w_ffitype)
+            raise self.error(w_ffitype)
+        self.after()
+        return x
 
     def get_unsigned(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.ULONG)
+        self.before()
+        x = self.func.call(self.argchain, rffi.ULONG)
+        self.after()
+        return x
 
     def get_unsigned_which_fits_into_a_signed(self, w_ffitype):
         # the same comment as get_signed apply
         restype = w_ffitype.get_ffitype()
+        self.before()
         call = self.func.call
         if restype is libffi.types.uint:
             assert not libffi.IS_32_BIT
             # on 32bit machines, we should never get here, because it's a case
             # which has already been handled by get_unsigned above.
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT))
         elif restype is libffi.types.ushort:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT))
         elif restype is libffi.types.uchar:
-            return rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR))
+            x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR))
         else:
-            self.error(w_ffitype)
+            raise self.error(w_ffitype)
+        self.after()
+        return x
 
 
     def get_pointer(self, w_ffitype):
+        self.before()
         ptrres = self.func.call(self.argchain, rffi.VOIDP)
+        self.after()
         return rffi.cast(rffi.ULONG, ptrres)
 
     def get_char(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.UCHAR)
+        self.before()
+        x = self.func.call(self.argchain, rffi.UCHAR)
+        self.after()
+        return x
 
     def get_unichar(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.WCHAR_T)
+        self.before()
+        x = self.func.call(self.argchain, rffi.WCHAR_T)
+        self.after()
+        return x
 
     def get_float(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.DOUBLE)
+        self.before()
+        x = self.func.call(self.argchain, rffi.DOUBLE)
+        self.after()
+        return x
 
     def get_singlefloat(self, w_ffitype):
-        return self.func.call(self.argchain, rffi.FLOAT)
+        self.before()
+        x = self.func.call(self.argchain, rffi.FLOAT)
+        self.after()
+        return x
 
     def get_struct(self, w_ffitype, w_structdescr):
+        self.before()
         addr = self.func.call(self.argchain, rffi.LONG, is_struct=True)
+        self.after()
         return w_structdescr.fromaddress(self.space, addr)
 
     def get_struct_rawffi(self, w_ffitype, w_structdescr):
+        self.before()
         uintval = self.func.call(self.argchain, rffi.ULONG, is_struct=True)
+        self.after()
         return w_structdescr.fromaddress(self.space, uintval)
 
     def get_void(self, w_ffitype):
-        return self.func.call(self.argchain, lltype.Void)
+        self.before()
+        x = self.func.call(self.argchain, lltype.Void)
+        self.after()
+        return x
 
 
 def unpack_argtypes(space, w_argtypes, w_restype):
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -18,6 +18,7 @@
 from rpython.rlib.rarithmetic import intmask, r_uint
 from pypy.module._rawffi.buffer import RawFFIBuffer
 from pypy.module._rawffi.tracker import tracker
+from pypy.module._rawffi import lasterror
 
 TYPEMAP = {
     # XXX A mess with unsigned/signed/normal chars :-/
@@ -495,10 +496,14 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                lasterror.restore_last_error(space)
                 self.ptr.call(args_ll, result.ll_buffer)
+                lasterror.save_last_error(space)
                 return space.wrap(result)
             else:
+                lasterror.restore_last_error(space)
                 self.ptr.call(args_ll, lltype.nullptr(rffi.VOIDP.TO))
+                lasterror.save_last_error(space)
                 return space.w_None
         except StackCheckError, e:
             raise OperationError(space.w_ValueError, space.wrap(e.message))
@@ -615,12 +620,10 @@
 
 if sys.platform == 'win32':
     def get_last_error(space):
-        from rpython.rlib.rwin32 import GetLastError
-        return space.wrap(GetLastError())
+        return space.wrap(lasterror.fetch_last_error(space))
     @unwrap_spec(error=int)
     def set_last_error(space, error):
-        from rpython.rlib.rwin32 import SetLastError
-        SetLastError(error)
+        lasterror.store_last_error(space, error)
 else:
     # always have at least a dummy version of these functions
     # (https://bugs.pypy.org/issue1242)
diff --git a/pypy/module/_rawffi/lasterror.py b/pypy/module/_rawffi/lasterror.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_rawffi/lasterror.py
@@ -0,0 +1,40 @@
+# For Windows only.
+# https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
+
+import os
+
+_MS_WINDOWS = os.name == "nt"
+
+
+if _MS_WINDOWS:
+    from rpython.rlib import rwin32
+    from pypy.interpreter.executioncontext import ExecutionContext
+
+
+    ExecutionContext._rawffi_last_error = 0
+
+    def fetch_last_error(space):
+        ec = space.getexecutioncontext()
+        return ec._rawffi_last_error
+
+    def store_last_error(space, last_error):
+        ec = space.getexecutioncontext()
+        ec._rawffi_last_error = last_error
+
+    def restore_last_error(space):
+        ec = space.getexecutioncontext()
+        lasterror = ec._rawffi_last_error
+        rwin32.SetLastError(lasterror)
+
+    def save_last_error(space):
+        lasterror = rwin32.GetLastError()
+        ec = space.getexecutioncontext()
+        ec._rawffi_last_error = lasterror
+
+else:
+
+    def restore_last_error(space):
+        pass
+
+    def save_last_error(space):
+        pass
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -16,6 +16,7 @@
         #include "src/precommondefs.h"
         #include <stdlib.h>
         #include <stdio.h>
+        #include <errno.h>
 
         struct x
         {
@@ -204,6 +205,24 @@
             return inp;
         }
 
+        RPY_EXPORTED
+        int check_errno(int incoming)
+        {
+            int old_errno = errno;
+            errno = incoming;
+            return old_errno;
+        }
+
+        #ifdef _WIN32
+        #include <Windows.h>
+        RPY_EXPORTED
+        int check_last_error(int incoming)
+        {
+            int old_errno = GetLastError();
+            SetLastError(incoming);
+            return old_errno;
+        }
+        #endif
         '''))
         eci = ExternalCompilationInfo(include_dirs=[cdir])
         return str(platform.compile([c_file], eci, 'x', standalone=False))
@@ -1150,6 +1169,37 @@
         raises(OverflowError, "arg1[0] = 10**900")
         arg1.free()
 
+    def test_errno(self):
+        import _rawffi
+        lib = _rawffi.CDLL(self.lib_name)
+        A = _rawffi.Array('i')
+        f = lib.ptr('check_errno', ['i'], 'i')
+        _rawffi.set_errno(42)
+        arg = A(1)
+        arg[0] = 43
+        res = f(arg)
+        assert res[0] == 42
+        z = _rawffi.get_errno()
+        assert z == 43
+        arg.free()
+
+    def test_last_error(self):
+        import sys
+        if sys.platform != 'win32':
+            skip("Windows test")
+        import _rawffi
+        lib = _rawffi.CDLL(self.lib_name)
+        A = _rawffi.Array('i')
+        f = lib.ptr('check_last_error', ['i'], 'i')
+        _rawffi.set_last_error(42)
+        arg = A(1)
+        arg[0] = 43
+        res = f(arg)
+        assert res[0] == 42
+        z = _rawffi.get_last_error()
+        assert z == 43
+        arg.free()
+
 
 class AppTestAutoFree:
     spaceconfig = dict(usemodules=['_rawffi', 'struct'])
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -52,7 +52,8 @@
 
 if not OPENSSL_NO_SSL2:
     constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
-constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
+if not OPENSSL_NO_SSL3:
+    constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
 
@@ -656,7 +657,7 @@
     # set up context
     if protocol == PY_SSL_VERSION_TLS1:
         method = libssl_TLSv1_method()
-    elif protocol == PY_SSL_VERSION_SSL3:
+    elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3:
         method = libssl_SSLv3_method()
     elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
         method = libssl_SSLv2_method()
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -58,7 +58,7 @@
     w_globals = from_ref(space, py_frame.c_f_globals)
 
     frame = space.FrameClass(space, code, w_globals, outer_func=None)
-    frame.f_lineno = py_frame.c_f_lineno
+    frame.f_lineno = rffi.getintfield(py_frame, 'c_f_lineno')
     w_obj = space.wrap(frame)
     track_reference(space, py_obj, w_obj)
     return w_obj
diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -30,6 +30,7 @@
                 'get_referrers': 'referents.get_referrers',
                 '_dump_rpy_heap': 'referents._dump_rpy_heap',
                 'get_typeids_z': 'referents.get_typeids_z',
+                'get_typeids_list': 'referents.get_typeids_list',
                 'GcRef': 'referents.W_GcRef',
                 })
         MixedModule.__init__(self, space, w_name)
diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -16,7 +16,8 @@
     [0][0][0][-1] inserted after all GC roots, before all non-roots.
 
     If the argument is a filename and the 'zlib' module is available,
-    we also write a 'typeids.txt' in the same directory, if none exists.
+    we also write 'typeids.txt' and 'typeids.lst' in the same directory,
+    if they don't already exist.
     """
     if isinstance(file, str):
         f = open(file, 'wb')
@@ -30,7 +31,13 @@
             filename2 = os.path.join(os.path.dirname(file), 'typeids.txt')
             if not os.path.exists(filename2):
                 data = zlib.decompress(gc.get_typeids_z())
-                f = open(filename2, 'wb')
+                f = open(filename2, 'w')
+                f.write(data)
+                f.close()
+            filename2 = os.path.join(os.path.dirname(file), 'typeids.lst')
+            if not os.path.exists(filename2):
+                data = ''.join(['%d\n' % n for n in gc.get_typeids_list()])
+                f = open(filename2, 'w')
                 f.write(data)
                 f.close()
     else:
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -228,3 +228,8 @@
     a = rgc.get_typeids_z()
     s = ''.join([a[i] for i in range(len(a))])
     return space.wrap(s)
+
+def get_typeids_list(space):
+    l = rgc.get_typeids_list()
+    list_w = [space.wrap(l[i]) for i in range(len(l))]
+    return space.newlist(list_w)
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -60,9 +60,9 @@
     _mixin_ = True
 
     def reduce(self, space):
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
+        _numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(_numpypy, MixedModule)
+        multiarray = _numpypy.get("multiarray")
         assert isinstance(multiarray, MixedModule)
         scalar = multiarray.get("scalar")
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -34,8 +34,8 @@
 
 
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
-                        "unegative", "flat", "tostring","count_nonzero",
-                        "argsort"]
+                        "unegative", "flat", "tostring", "count_nonzero",
+                        "argsort", "cumsum", "logical_xor_reduce"]
 TWO_ARG_FUNCTIONS = ["dot", 'take', 'searchsorted']
 TWO_ARG_FUNCTIONS_OR_NONE = ['view', 'astype']
 THREE_ARG_FUNCTIONS = ['where']
@@ -559,6 +559,11 @@
                 w_res = arr.descr_any(interp.space)
             elif self.name == "all":
                 w_res = arr.descr_all(interp.space)
+            elif self.name == "cumsum":
+                w_res = arr.descr_cumsum(interp.space)
+            elif self.name == "logical_xor_reduce":
+                logical_xor = ufuncs.get(interp.space).logical_xor
+                w_res = logical_xor.reduce(interp.space, arr, None)
             elif self.name == "unegative":
                 neg = ufuncs.get(interp.space).negative
                 w_res = neg.call(interp.space, [arr], None, None, None)
diff --git a/pypy/module/micronumpy/flagsobj.py b/pypy/module/micronumpy/flagsobj.py
--- a/pypy/module/micronumpy/flagsobj.py
+++ b/pypy/module/micronumpy/flagsobj.py
@@ -5,46 +5,26 @@
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module.micronumpy import constants as NPY
-
+from pypy.module.micronumpy.strides import is_c_contiguous, is_f_contiguous
 
 def enable_flags(arr, flags):
     arr.flags |= flags
 
-
 def clear_flags(arr, flags):
     arr.flags &= ~flags
 
-
- at jit.unroll_safe
 def _update_contiguous_flags(arr):
-    shape = arr.shape
-    strides = arr.strides
-
-    is_c_contig = True
-    sd = arr.dtype.elsize
-    for i in range(len(shape) - 1, -1, -1):
-        dim = shape[i]
-        if strides[i] != sd:
-            is_c_contig = False
-            break
-        if dim == 0:
-            break
-        sd *= dim
+    is_c_contig = is_c_contiguous(arr)
     if is_c_contig:
         enable_flags(arr, NPY.ARRAY_C_CONTIGUOUS)
     else:
         clear_flags(arr, NPY.ARRAY_C_CONTIGUOUS)
 
-    sd = arr.dtype.elsize
-    for i in range(len(shape)):
-        dim = shape[i]
-        if strides[i] != sd:
-            clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
-            return
-        if dim == 0:
-            break
-        sd *= dim
-    enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
+    is_f_contig = is_f_contiguous(arr)
+    if is_f_contig:
+        enable_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
+    else:
+        clear_flags(arr, NPY.ARRAY_F_CONTIGUOUS)
 
 
 class W_FlagsObject(W_Root):
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -250,6 +250,7 @@
 def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity):
     obj_iter, obj_state = obj.create_iter()
     out_iter, out_state = out.create_iter()
+    out_iter.track_index = False
     if identity is None:
         cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
         out_iter.setitem(out_state, cur_value)
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -19,7 +19,7 @@
     order_converter, shape_converter, searchside_converter
 from pypy.module.micronumpy.flagsobj import W_FlagsObject
 from pypy.module.micronumpy.strides import get_shape_from_iterable, \
-    shape_agreement, shape_agreement_multiple
+    shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous
 
 
 def _match_dot_shapes(space, left, right):
@@ -824,7 +824,15 @@
                 raise OperationError(space.w_ValueError, space.wrap(
                     "new type not compatible with array."))
         else:
-            if dims == 1 or impl.get_strides()[0] < impl.get_strides()[-1]:
+            if not is_c_contiguous(impl) and not is_f_contiguous(impl):
+                if old_itemsize != new_itemsize:
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "new type not compatible with array."))
+                # Strides, shape does not change
+                v = impl.astype(space, dtype)
+                return wrap_impl(space, w_type, self, v) 
+            strides = impl.get_strides()
+            if dims == 1 or strides[0] <strides[-1]:
                 # Column-major, resize first dimension
                 if new_shape[0] * old_itemsize % new_itemsize != 0:
                     raise OperationError(space.w_ValueError, space.wrap(
@@ -1131,9 +1139,9 @@
         from pypy.interpreter.mixedmodule import MixedModule
         from pypy.module.micronumpy.concrete import SliceArray
 
-        numpypy = space.getbuiltinmodule("_numpypy")
-        assert isinstance(numpypy, MixedModule)
-        multiarray = numpypy.get("multiarray")
+        _numpypy = space.getbuiltinmodule("_numpypy")
+        assert isinstance(_numpypy, MixedModule)
+        multiarray = _numpypy.get("multiarray")
         assert isinstance(multiarray, MixedModule)
         reconstruct = multiarray.get("_reconstruct")
         parameters = space.newtuple([self.getclass(space), space.newtuple(
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -429,3 +429,35 @@
                     n_old_elems_to_use *= old_shape[oldI]
     assert len(new_strides) == len(new_shape)
     return new_strides[:]
+
+ at jit.unroll_safe
+def is_c_contiguous(arr):
+    shape = arr.get_shape()
+    strides = arr.get_strides()
+    ret = True
+    sd = arr.dtype.elsize
+    for i in range(len(shape) - 1, -1, -1):
+        dim = shape[i]
+        if strides[i] != sd:
+            ret = False
+            break
+        if dim == 0:
+            break
+        sd *= dim
+    return ret
+
+ at jit.unroll_safe
+def is_f_contiguous(arr):
+    shape = arr.get_shape()
+    strides = arr.get_strides()
+    ret = True
+    sd = arr.dtype.elsize
+    for i in range(len(shape)):
+        dim = shape[i]
+        if strides[i] != sd:
+            ret = False
+            break
+        if dim == 0:
+            break
+        sd *= dim
+    return ret
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -3,13 +3,13 @@
 
 class AppTestNumSupport(BaseNumpyAppTest):
     def test_zeros(self):
-        from numpypy import zeros
+        from numpy import zeros
         a = zeros(3)
         assert len(a) == 3
         assert a[0] == a[1] == a[2] == 0
 
     def test_empty(self):
-        from numpypy import empty
+        from numpy import empty
         import gc
         for i in range(1000):
             a = empty(3)
@@ -26,26 +26,26 @@
                 "empty() returned a zeroed out array every time")
 
     def test_where(self):
-        from numpypy import where, ones, zeros, array
+        from numpy import where, ones, zeros, array
         a = [1, 2, 3, 0, -3]
         a = where(array(a) > 0, ones(5), zeros(5))
         assert (a == [1, 1, 1, 0, 0]).all()
 
     def test_where_differing_dtypes(self):
-        from numpypy import array, ones, zeros, where
+        from numpy import array, ones, zeros, where
         a = [1, 2, 3, 0, -3]
         a = where(array(a) > 0, ones(5, dtype=int), zeros(5, dtype=float))
         assert (a == [1, 1, 1, 0, 0]).all()
 
     def test_where_broadcast(self):
-        from numpypy import array, where
+        from numpy import array, where
         a = where(array([[1, 2, 3], [4, 5, 6]]) > 3, [1, 1, 1], 2)
         assert (a == [[2, 2, 2], [1, 1, 1]]).all()
         a = where(True, [1, 1, 1], 2)
         assert (a == [1, 1, 1]).all()
 
     def test_where_errors(self):
-        from numpypy import where, array
+        from numpy import where, array
         raises(ValueError, "where([1, 2, 3], [3, 4, 5])")
         raises(ValueError, "where([1, 2, 3], [3, 4, 5], [6, 7])")
         assert where(True, 1, 2) == array(1)
@@ -58,14 +58,14 @@
     #    xxx
 
     def test_where_invalidates(self):
-        from numpypy import where, ones, zeros, array
+        from numpy import where, ones, zeros, array
         a = array([1, 2, 3, 0, -3])
         b = where(a > 0, ones(5), zeros(5))
         a[0] = 0
         assert (b == [1, 1, 1, 0, 0]).all()
 
     def test_dot_basic(self):
-        from numpypy import array, dot, arange
+        from numpy import array, dot, arange
         a = array(range(5))
         assert dot(a, a) == 30.0
 
@@ -100,7 +100,7 @@
         assert (dot([[1,2],[3,4]],[5,6]) == [17, 39]).all()
 
     def test_dot_constant(self):
-        from numpypy import array, dot
+        from numpy import array, dot
         a = array(range(5))
         b = a.dot(2.5)
         for i in xrange(5):
@@ -111,7 +111,7 @@
         assert c == 12.0
 
     def test_dot_out(self):
-        from numpypy import arange, dot
+        from numpy import arange, dot
         a = arange(12).reshape(3, 4)
         b = arange(12).reshape(4, 3)
         out = arange(9).reshape(3, 3)
@@ -124,19 +124,19 @@
                                 'right type, nr dimensions, and be a C-Array)')
 
     def test_choose_basic(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), array([4, 5, 6]), array([7, 8, 9])
         r = array([2, 1, 0]).choose([a, b, c])
         assert (r == [7, 5, 3]).all()
 
     def test_choose_broadcast(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         r = array([2, 1, 0]).choose([a, b, c])
         assert (r == [13, 5, 3]).all()
 
     def test_choose_out(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         r = array([2, 1, 0]).choose([a, b, c], out=None)
         assert (r == [13, 5, 3]).all()
@@ -146,7 +146,7 @@
         assert (a == [13, 5, 3]).all()
 
     def test_choose_modes(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         raises(ValueError, "array([3, 1, 0]).choose([a, b, c])")
         raises(ValueError, "array([3, 1, 0]).choose([a, b, c], mode='raises')")
@@ -158,20 +158,20 @@
         assert (r == [4, 5, 3]).all()
 
     def test_choose_dtype(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1.2, 2, 3]), [4, 5, 6], 13
         r = array([2, 1, 0]).choose([a, b, c])
         assert r.dtype == float
 
     def test_choose_dtype_out(self):
-        from numpypy import array
+        from numpy import array
         a, b, c = array([1, 2, 3]), [4, 5, 6], 13
         x = array([0, 0, 0], dtype='i2')
         r = array([2, 1, 0]).choose([a, b, c], out=x)
         assert r.dtype == 'i2'
 
     def test_put_basic(self):
-        from numpypy import arange, array
+        from numpy import arange, array
         a = arange(5)
         a.put([0, 2], [-44, -55])
         assert (a == array([-44, 1, -55, 3, 4])).all()
@@ -183,7 +183,7 @@
         assert (a == array([0, 7, 2, 3, 4])).all()
 
     def test_put_modes(self):
-        from numpypy import array, arange
+        from numpy import array, arange
         a = arange(5)
         a.put(22, -5, mode='clip')
         assert (a == array([0, 1, 2, 3, -5])).all()
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -14,15 +14,12 @@
             else:
                 from . import dummy_module as numpy
                 sys.modules['numpy'] = numpy
-            sys.modules['numpypy'] = numpy
         else:
             import os
             path = os.path.dirname(__file__) + '/dummy_module.py'
             cls.space.appexec([cls.space.wrap(path)], """(path):
             import imp
-            numpy = imp.load_source('numpy', path)
-            import sys
-            sys.modules['numpypy'] = numpy
+            imp.load_source('numpy', path)
             """)
         cls.w_non_native_prefix = cls.space.wrap(NPY.OPPBYTE)
         cls.w_native_prefix = cls.space.wrap(NPY.NATBYTE)
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -132,13 +132,13 @@
             cls.w_c_pow = cls.space.wrap(interp2app(cls_c_pow))
 
     def test_fabs(self):
-        from numpypy import fabs, dtype
+        from numpy import fabs, dtype
 
         a = dtype('complex128').type(complex(-5., 5.))
         raises(TypeError, fabs, a)
 
     def test_fmax(self):
-        from numpypy import fmax, array
+        from numpy import fmax, array
         nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
         a = array((complex(ninf, 10), complex(10, ninf),
                    complex( inf, 10), complex(10,  inf),
@@ -165,7 +165,7 @@
         assert (fmax(a, b) == res).all()
 
     def test_fmin(self):
-        from numpypy import fmin, array
+        from numpy import fmin, array
         nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
         a = array((complex(ninf, 10), complex(10, ninf),
                    complex( inf, 10), complex(10,  inf),
@@ -192,11 +192,11 @@
         assert (fmin(a, b) == res).all()
 
     def test_signbit(self):
-        from numpypy import signbit
+        from numpy import signbit
         raises(TypeError, signbit, complex(1,1))
 
     def test_reciprocal(self):
-        from numpypy import array, reciprocal
+        from numpy import array, reciprocal
         inf = float('inf')
         nan = float('nan')
         #complex
@@ -218,21 +218,21 @@
                 assert (a[0].imag - e.imag) < rel_err
 
     def test_floorceiltrunc(self):
-        from numpypy import array, floor, ceil, trunc
+        from numpy import array, floor, ceil, trunc
         a = array([ complex(-1.4, -1.4), complex(-1.5, -1.5)])
         raises(TypeError, floor, a)
         raises(TypeError, ceil, a)
         raises(TypeError, trunc, a)
 
     def test_copysign(self):
-        from numpypy import copysign, dtype
+        from numpy import copysign, dtype
         complex128 = dtype('complex128').type
         a = complex128(complex(-5., 5.))
         b = complex128(complex(0., 0.))
         raises(TypeError, copysign, a, b)
 
     def test_exp2(self):
-        from numpypy import array, exp2
+        from numpy import array, exp2
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
@@ -268,7 +268,7 @@
 
     def test_expm1(self):
         import math, cmath
-        from numpypy import array, expm1
+        from numpy import array, expm1
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
@@ -307,7 +307,7 @@
                 self.rAlmostEqual(t1, t2, rel_err=rel_err, msg=msg)
 
     def test_not_complex(self):
-        from numpypy import (radians, deg2rad, degrees, rad2deg,
+        from numpy import (radians, deg2rad, degrees, rad2deg,
                   logaddexp, logaddexp2, fmod,
                   arctan2)
         raises(TypeError, radians, complex(90,90))
@@ -320,7 +320,7 @@
         raises (TypeError, fmod, complex(90,90), 3)
 
     def test_isnan_isinf(self):
-        from numpypy import isnan, isinf, array
+        from numpy import isnan, isinf, array
         assert (isnan(array([0.2+2j, complex(float('inf'),0),
                 complex(0,float('inf')), complex(0,float('nan')),
                 complex(float('nan'), 0)], dtype=complex)) == \
@@ -333,7 +333,7 @@
 
 
     def test_square(self):
-        from numpypy import square
+        from numpy import square
         assert square(complex(3, 4)) == complex(3,4) * complex(3, 4)
 
     def test_power_simple(self):
@@ -364,8 +364,8 @@
             self.rAlmostEqual(float(n_r_a[i].imag), float(p_r[i].imag), msg=msg)
 
     def test_conjugate(self):
-        from numpypy import conj, conjugate, dtype
-        import numpypy as np
+        from numpy import conj, conjugate, dtype
+        import numpy as np
 
         c0 = dtype('complex128').type(complex(2.5, 0))
         c1 = dtype('complex64').type(complex(1, 2))
@@ -390,7 +390,7 @@
     def test_logn(self):
         import math, cmath
         # log and log10 are tested in math (1:1 from rcomplex)
-        from numpypy import log2, array, log1p
+        from numpy import log2, array, log1p
         inf = float('inf')
         ninf = -float('inf')
         nan = float('nan')
@@ -447,7 +447,7 @@
                 self.rAlmostEqual(t1, t2, rel_err=rel_err, msg=msg)
 
     def test_logical_ops(self):
-        from numpypy import logical_and, logical_or, logical_xor, logical_not
+        from numpy import logical_and, logical_or, logical_xor, logical_not
 
         c1 = complex(1, 1)
         c3 = complex(3, 0)
@@ -461,7 +461,7 @@
         assert (logical_not([c1, c0]) == [False, True]).all()
 
     def test_minimum(self):
-        from numpypy import array, minimum
+        from numpy import array, minimum
 
         a = array([-5.0+5j, -5.0-5j, -0.0-10j, 1.0+10j])
         b = array([ 3.0+10.0j, 3.0, -2.0+2.0j, -3.0+4.0j])
@@ -470,7 +470,7 @@
             assert c[i] == min(a[i], b[i])
 
     def test_maximum(self):
-        from numpypy import array, maximum
+        from numpy import array, maximum
 
         a = array([-5.0+5j, -5.0-5j, -0.0-10j, 1.0+10j])
         b = array([ 3.0+10.0j, 3.0, -2.0+2.0j, -3.0+4.0j])
@@ -480,10 +480,10 @@
 
     def test_basic(self):
         import sys
-        from numpypy import (dtype, add, array, dtype,
+        from numpy import (dtype, add, array, dtype,
             subtract as sub, multiply, divide, negative, absolute as abs,
             floor_divide, real, imag, sign)
-        from numpypy import (equal, not_equal, greater, greater_equal, less,
+        from numpy import (equal, not_equal, greater, greater_equal, less,
                 less_equal, isnan)
         assert real(4.0) == 4.0
         assert imag(0.0) == 0.0
@@ -579,7 +579,7 @@
             assert repr(abs(inf_c)) == 'inf'
             assert repr(abs(complex(float('nan'), float('nan')))) == 'nan'
             # numpy actually raises an AttributeError,
-            # but numpypy raises a TypeError
+            # but numpy.raises a TypeError
             if '__pypy__' in sys.builtin_module_names:
                 exct, excm = TypeError, 'readonly attribute'
             else:
@@ -592,7 +592,7 @@
             assert(imag(c2) == 4.0)
 
     def test_conj(self):
-        from numpypy import array
+        from numpy import array
 
         a = array([1 + 2j, 1 - 2j])
         assert (a.conj() == [1 - 2j, 1 + 2j]).all()
@@ -603,7 +603,7 @@
         if self.isWindows:
             skip('windows does not support c99 complex')
         import sys
-        import numpypy as np
+        import numpy as np
         rAlmostEqual = self.rAlmostEqual
 
         for t, testcases in (
@@ -658,6 +658,6 @@
             sys.stderr.write('\n')
 
     def test_complexbox_to_pycomplex(self):
-        from numpypy import dtype
+        from numpy import dtype
         x = dtype('complex128').type(3.4j)
         assert complex(x) == 3.4j
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -23,7 +23,7 @@
             from numpy.core.multiarray import typeinfo
         except ImportError:
             # running on dummy module
-            from numpypy import typeinfo
+            from numpy import typeinfo
         assert typeinfo['Number'] == np.number
         assert typeinfo['LONGLONG'] == ('q', 9, 64, 8, 9223372036854775807L,
                                         -9223372036854775808L, np.longlong)
@@ -39,7 +39,7 @@
                                     np.dtype('int').type)
 
     def test_dtype_basic(self):
-        from numpypy import dtype
+        from numpy import dtype
         import sys
 
         d = dtype('?')
@@ -104,7 +104,7 @@
         assert d.shape == ()
 
     def test_dtype_eq(self):
-        from numpypy import dtype
+        from numpy import dtype
 
         assert dtype("int8") == "int8"
         assert "int8" == dtype("int8")
@@ -112,7 +112,7 @@
         assert dtype(bool) == bool
 
     def test_dtype_aliases(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype('bool8') is dtype('bool')
         assert dtype('byte') is dtype('int8')
         assert dtype('ubyte') is dtype('uint8')
@@ -132,7 +132,7 @@
         assert dtype('clongdouble').num in (15, 16)
 
     def test_dtype_with_types(self):
-        from numpypy import dtype
+        from numpy import dtype
 
         assert dtype(bool).num == 0
         if self.ptr_size == 4:
@@ -156,7 +156,7 @@
         assert dtype('complex').num == 15
 
     def test_array_dtype_attr(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
 
         a = array(range(5), long)
         assert a.dtype is dtype(long)
@@ -182,7 +182,7 @@
         assert np.dtype('S5').isbuiltin == 0
 
     def test_repr_str(self):
-        from numpypy import dtype
+        from numpy import dtype
         b = dtype(int).newbyteorder().newbyteorder().byteorder
         assert '.dtype' in repr(dtype)
         d = dtype('?')
@@ -208,7 +208,7 @@
         assert str(d) == "|V16"
 
     def test_bool_array(self):
-        from numpypy import array, False_, True_
+        from numpy import array, False_, True_
 
         a = array([0, 1, 2, 2.5], dtype='?')
         assert a[0] is False_
@@ -216,7 +216,7 @@
             assert a[i] is True_
 
     def test_copy_array_with_dtype(self):
-        from numpypy import array, longlong, False_
+        from numpy import array, longlong, False_
 
         a = array([0, 1, 2, 3], dtype=long)
         # int on 64-bit, long in 32-bit
@@ -230,35 +230,35 @@
         assert b[0] is False_
 
     def test_zeros_bool(self):
-        from numpypy import zeros, False_
+        from numpy import zeros, False_
 
         a = zeros(10, dtype=bool)
         for i in range(10):
             assert a[i] is False_
 
     def test_ones_bool(self):
-        from numpypy import ones, True_
+        from numpy import ones, True_
 
         a = ones(10, dtype=bool)
         for i in range(10):
             assert a[i] is True_
 
     def test_zeros_long(self):
-        from numpypy import zeros, longlong
+        from numpy import zeros, longlong
         a = zeros(10, dtype=long)
         for i in range(10):
             assert isinstance(a[i], longlong)
             assert a[1] == 0
 
     def test_ones_long(self):
-        from numpypy import ones, longlong
+        from numpy import ones, longlong
         a = ones(10, dtype=long)
         for i in range(10):
             assert isinstance(a[i], longlong)
             assert a[1] == 1
 
     def test_overflow(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         assert array([128], 'b')[0] == -128
         assert array([256], 'B')[0] == 0
         assert array([32768], 'h')[0] == -32768
@@ -273,7 +273,7 @@
         raises(OverflowError, "array([2**64], 'Q')")
 
     def test_bool_binop_types(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         types = [
             '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd',
             'e'
@@ -285,7 +285,7 @@
             assert (a + array([0], t)).dtype is dtype(t)
 
     def test_binop_types(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
                  ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
                  ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
@@ -309,7 +309,7 @@
             assert (d1, d2) == (d1, d2) and d3 is dtype(dout)
 
     def test_add(self):
-        import numpypy as np
+        import numpy as np
         for dtype in ["int8", "int16", "I"]:
             a = np.array(range(5), dtype=dtype)
             b = a + a
@@ -325,22 +325,22 @@
         assert len(d) == 2
 
     def test_shape(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype(long).shape == ()
 
     def test_cant_subclass(self):
-        from numpypy import dtype
+        from numpy import dtype
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
 
     def test_can_subclass(self):
-        import numpypy
-        class xyz(numpypy.void):
+        import numpy
+        class xyz(numpy.void):
             pass
         assert True
 
     def test_index(self):
-        import numpypy as np
+        import numpy as np
         for dtype in [np.int8, np.int16, np.int32, np.int64]:
             a = np.array(range(10), dtype=dtype)
             b = np.array([0] * 10, dtype=dtype)
@@ -348,7 +348,7 @@
                 a[idx] += 1
 
     def test_hash(self):
-        import numpypy as numpy
+        import numpy
         for tp, value in [
             (numpy.int8, 4),
             (numpy.int16, 5),
@@ -395,7 +395,7 @@
 
     def test_pickle(self):
         import numpy as np
-        from numpypy import array, dtype
+        from numpy import array, dtype
         from cPickle import loads, dumps
         a = array([1,2,3])
         if self.ptr_size == 8:
@@ -408,7 +408,7 @@
         assert np.dtype(('<f8', 2)).__reduce__() == (dtype, ('V16', 0, 1), (3, '|', (dtype('float64'), (2,)), None, None, 16, 1, 0))
 
     def test_newbyteorder(self):
-        import numpypy as np
+        import numpy as np
         import sys
         sys_is_le = sys.byteorder == 'little'
         native_code = sys_is_le and '<' or '>'
@@ -480,7 +480,7 @@
 
 class AppTestTypes(BaseAppTestDtypes):
     def test_abstract_types(self):
-        import numpypy as numpy
+        import numpy
 
         raises(TypeError, numpy.generic, 0)
         raises(TypeError, numpy.number, 0)
@@ -526,16 +526,16 @@
         #assert a.dtype is numpy.dtype('|V4')
 
     def test_new(self):
-        import numpypy as np
+        import numpy as np
         assert np.int_(4) == 4
         assert np.float_(3.4) == 3.4
 
     def test_pow(self):
-        from numpypy import int_
+        from numpy import int_
         assert int_(4) ** 2 == 16
 
     def test_bool(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.bool_.mro() == [numpy.bool_, numpy.generic, object]
         assert numpy.bool_(3) is numpy.True_
@@ -550,7 +550,7 @@
         assert numpy.bool_("False") is numpy.True_
 
     def test_int8(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int8.mro() == [numpy.int8, numpy.signedinteger,
                                     numpy.integer, numpy.number,
@@ -574,7 +574,7 @@
         assert numpy.int8('128') == -128
 
     def test_uint8(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.uint8.mro() == [numpy.uint8, numpy.unsignedinteger,
                                      numpy.integer, numpy.number,
@@ -599,7 +599,7 @@
         assert numpy.uint8('256') == 0
 
     def test_int16(self):
-        import numpypy as numpy
+        import numpy
 
         x = numpy.int16(3)
         assert x == 3
@@ -641,7 +641,7 @@
             assert numpy.uint32('4294967296') == 0
 
     def test_int_(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int_ is numpy.dtype(int).type
         assert numpy.int_.mro() == [numpy.int_, numpy.signedinteger,
@@ -752,7 +752,7 @@
 
     def test_complex_format(self):
         import sys
-        import numpypy as numpy
+        import numpy
 
         for complex_ in (numpy.complex128, numpy.complex64,):
             for real, imag, should in [
@@ -792,7 +792,7 @@
             assert "{:g}".format(numpy.complex_(0.5+1.5j)) == '{:g}'.format(0.5+1.5j)
 
     def test_complex(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.complex_ is numpy.complex128
         assert numpy.csingle is numpy.complex64
@@ -812,7 +812,7 @@
         assert d.char == 'F'
 
     def test_subclass_type(self):
-        import numpypy as numpy
+        import numpy
 
         class X(numpy.float64):
             def m(self):
@@ -826,12 +826,12 @@
         assert b.dtype.type is numpy.float64
 
     def test_long_as_index(self):
-        from numpypy import int_, float64
+        from numpy import int_, float64
         assert (1, 2, 3)[int_(1)] == 2
         raises(TypeError, lambda: (1, 2, 3)[float64(1)])
 
     def test_int(self):
-        from numpypy import int32, int64, int_
+        from numpy import int32, int64, int_
         import sys
         assert issubclass(int_, int)
         if sys.maxint == (1<<31) - 1:
@@ -842,7 +842,7 @@
             assert int_ is int64
 
     def test_various_types(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int16 is numpy.short
         assert numpy.int8 is numpy.byte
@@ -864,7 +864,7 @@
         assert not issubclass(numpy.longfloat, numpy.float64)
 
     def test_mro(self):
-        import numpypy as numpy
+        import numpy
 
         assert numpy.int16.__mro__ == (numpy.int16, numpy.signedinteger,
                                        numpy.integer, numpy.number,
@@ -873,7 +873,7 @@
 
     def test_operators(self):
         from operator import truediv
-        from numpypy import float64, int_, True_, False_
+        from numpy import float64, int_, True_, False_
         assert 5 / int_(2) == int_(2)
         assert truediv(int_(3), int_(2)) == float64(1.5)
         assert truediv(3, int_(2)) == float64(1.5)
@@ -899,7 +899,7 @@
 
     def test_alternate_constructs(self):
         import numpy as np
-        from numpypy import dtype
+        from numpy import dtype
         nnp = self.non_native_prefix
         byteorder = self.native_prefix
         assert dtype('i8') == dtype(byteorder + 'i8') == dtype('=i8') == dtype(long)
@@ -922,7 +922,7 @@
         assert d.str == '|S0'
 
     def test_dtype_str(self):
-        from numpypy import dtype
+        from numpy import dtype
         byteorder = self.native_prefix
         assert dtype('i8').str == byteorder + 'i8'
         assert dtype('<i8').str == '<i8'
@@ -949,7 +949,7 @@
         assert dtype(('f8', 2)).str == "|V16"
 
     def test_intp(self):
-        from numpypy import dtype
+        from numpy import dtype
         for s in ['p', 'int']:
             assert dtype(s) is dtype('intp')
         for s in ['P', 'uint']:
@@ -968,16 +968,16 @@
             assert dtype('P').name == 'uint64'
 
     def test_alignment(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype('i4').alignment == 4
 
     def test_isnative(self):
-        from numpypy import dtype
+        from numpy import dtype
         assert dtype('i4').isnative == True
         assert dtype('>i8').isnative == False
 
     def test_any_all_nonzero(self):
-        import numpypy as numpy
+        import numpy
         x = numpy.bool_(True)
         assert x.any() is numpy.True_
         assert x.all() is numpy.True_
@@ -1000,7 +1000,7 @@
         assert x.__nonzero__() is True
 
     def test_ravel(self):
-        from numpypy import float64, int8, array
+        from numpy import float64, int8, array
         x = float64(42.5).ravel()
         assert x.dtype == float64
         assert (x == array([42.5])).all()
@@ -1023,7 +1023,7 @@
 
 class AppTestStrUnicodeDtypes(BaseNumpyAppTest):
     def test_mro(self):
-        from numpypy import str_, unicode_, character, flexible, generic
+        from numpy import str_, unicode_, character, flexible, generic
         import sys
         if '__pypy__' in sys.builtin_module_names:
             assert str_.mro() == [str_, character, flexible, generic,
@@ -1037,7 +1037,7 @@
                                       flexible, generic, object]
 
     def test_str_dtype(self):
-        from numpypy import dtype, str_
+        from numpy import dtype, str_
 
         raises(TypeError, "dtype('Sx')")
         for t in ['S8', '|S8', '=S8']:
@@ -1058,7 +1058,7 @@
             assert d.str == '|S1'
 
     def test_unicode_dtype(self):
-        from numpypy import dtype, unicode_
+        from numpy import dtype, unicode_
 
         raises(TypeError, "dtype('Ux')")
         d = dtype('U8')
@@ -1070,11 +1070,11 @@
         assert d.num == 19
 
     def test_string_boxes(self):
-        from numpypy import str_
+        from numpy import str_
         assert isinstance(str_(3), str_)
 
     def test_unicode_boxes(self):
-        from numpypy import unicode_
+        from numpy import unicode_
         import sys
         if '__pypy__' in sys.builtin_module_names:
             exc = raises(NotImplementedError, unicode_, 3)
@@ -1085,7 +1085,7 @@
 
     def test_character_dtype(self):
         import numpy as np
-        from numpypy import array, character
+        from numpy import array, character
         x = array([["A", "B"], ["C", "D"]], character)
         assert (x == [["A", "B"], ["C", "D"]]).all()
         d = np.dtype('c')
@@ -1098,7 +1098,7 @@
     spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
 
     def test_create(self):
-        from numpypy import dtype, void
+        from numpy import dtype, void
 
         raises(ValueError, "dtype([('x', int), ('x', float)])")
         d = dtype([("x", "<i4"), ("y", "<f4"), ("z", "<u2"), ("v", "<f8")])
@@ -1157,7 +1157,7 @@
             raises(NotImplementedError, np.dtype, d)
 
     def test_create_subarrays(self):
-        from numpypy import dtype
+        from numpy import dtype
         d = dtype([("x", "float", (2,)), ("y", "int64", (2,))])
         assert d.itemsize == 32
         assert d.name == "void256"
@@ -1278,7 +1278,7 @@
         assert repr(d) == "dtype([('f0', '<f8'), ('f1', '<f8')])"
 
     def test_pickle_record(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         from cPickle import loads, dumps
 
         d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)])
@@ -1289,7 +1289,7 @@
         assert new_d.__reduce__() == d.__reduce__()
 
     def test_pickle_record_subarrays(self):
-        from numpypy import array, dtype
+        from numpy import array, dtype
         from cPickle import loads, dumps
 
         d = dtype([("x", "int32", (3,)), ("y", "int32", (2,)), ("z", "int32", (4,)), ("value", float, (5,))])
@@ -1321,7 +1321,7 @@
             cls.w_check_non_native = cls.space.wrap(interp2app(check_non_native))
 
     def test_non_native(self):
-        from numpypy import array
+        from numpy import array
         a = array([1, 2, 3], dtype=self.non_native_prefix + 'i2')
         assert a[0] == 1
         assert (a + a)[1] == 4
@@ -1344,7 +1344,7 @@
 
 class AppTestObjectDtypes(BaseNumpyAppTest):
     def test_scalar_from_object(self):
-        from numpypy import array
+        from numpy import array
         import sys
         class Polynomial(object):
             pass
diff --git a/pypy/module/micronumpy/test/test_iterators.py b/pypy/module/micronumpy/test/test_iterators.py
--- a/pypy/module/micronumpy/test/test_iterators.py
+++ b/pypy/module/micronumpy/test/test_iterators.py
@@ -13,6 +13,11 @@
         self.strides = strides
         self.start = start
 
+    def get_shape(self):
+        return self.shape
+
+    def get_strides(self):
+        return self.strides
 
 class TestIterDirect(object):
     def test_iterator_basic(self):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -269,13 +269,13 @@
                 assert not a.flags['F']
 
     def test_ndmin(self):
-        from numpypy import array
+        from numpy import array
 
         arr = array([[[1]]], ndmin=1)
         assert arr.shape == (1, 1, 1)
 
     def test_noop_ndmin(self):
-        from numpypy import array
+        from numpy import array
         arr = array([1], ndmin=3)
         assert arr.shape == (1, 1, 1)
 
@@ -289,7 +289,7 @@
         assert a.dtype == np.intp
 
     def test_array_copy(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(12)).reshape(3,4)
         b = array(a, ndmin=4)
         assert b.shape == (1, 1, 3, 4)
@@ -353,12 +353,12 @@
         assert str(exc.value) == 'buffer is read-only'
 
     def test_type(self):
-        from numpypy import array
+        from numpy import array
         ar = array(range(5))
         assert type(ar) is type(ar + ar)
 
     def test_ndim(self):
-        from numpypy import array
+        from numpy import array
         x = array(0.2)
         assert x.ndim == 0
         x = array([1, 2])
@@ -367,12 +367,12 @@
         assert x.ndim == 2
         x = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
         assert x.ndim == 3
-        # numpy actually raises an AttributeError, but numpypy raises an
+        # numpy actually raises an AttributeError, but numpy.raises an
         # TypeError
         raises((TypeError, AttributeError), 'x.ndim = 3')
 
     def test_zeros(self):
-        from numpypy import zeros
+        from numpy import zeros
         a = zeros(15)
         # Check that storage was actually zero'd.
         assert a[10] == 0.0
@@ -431,7 +431,7 @@
         assert type(b) is np.ndarray
 
     def test_size(self):
-        from numpypy import array,arange,cos
+        from numpy import array,arange,cos
         assert array(3).size == 1
         a = array([1, 2, 3])
         assert a.size == 3
@@ -440,13 +440,13 @@
         assert ten == 10
 
     def test_empty(self):
-        from numpypy import empty
+        from numpy import empty
         a = empty(2)
         a[1] = 1.0
         assert a[1] == 1.0
 
     def test_ones(self):
-        from numpypy import ones, dtype
+        from numpy import ones, dtype
         a = ones(3)
         assert len(a) == 3
         assert a[0] == 1
@@ -458,7 +458,7 @@
         assert b.dtype is dtype(complex)
 
     def test_arange(self):
-        from numpypy import arange, dtype
+        from numpy import arange, dtype
         a = arange(3)
         assert (a == [0, 1, 2]).all()
         assert a.dtype is dtype(int)
@@ -480,7 +480,7 @@
         assert arange(False, True, True).dtype is dtype(int)
 
     def test_copy(self):
-        from numpypy import arange, array
+        from numpy import arange, array
         a = arange(5)
         b = a.copy()
         for i in xrange(5):
@@ -522,7 +522,7 @@
             raises(NotImplementedError, a.copy, order=True)
 
     def test_iterator_init(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert a[3] == 3
 
@@ -546,7 +546,7 @@
         assert (a == [[1], [2]]).all()
 
     def test_getitem(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         raises(IndexError, "a[5]")
         a = a + a
@@ -555,14 +555,14 @@
         raises(IndexError, "a[-6]")
 
     def test_getitem_float(self):
-        from numpypy import array
+        from numpy import array
         a = array([1, 2, 3, 4])
         assert a[1.2] == 2
         assert a[1.6] == 2
         assert a[-1.2] == 4
 
     def test_getitem_tuple(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         raises(IndexError, "a[(1,2)]")
         for i in xrange(5):
@@ -572,28 +572,28 @@
             assert a[i] == b[i]
 
     def test_getitem_nd(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(15).reshape(3, 5)
         assert a[1, 3] == 8
         assert a.T[1, 2] == 11
 
     def test_getitem_obj_index(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         assert a[self.CustomIndexObject(1)] == 1
 
     def test_getitem_obj_prefer_index_to_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         assert a[self.CustomIndexIntObject(0, 1)] == 0
 
     def test_getitem_obj_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         assert a[self.CustomIntObject(1)] == 1
 
     def test_setitem(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         a[-1] = 5.0
         assert a[4] == 5.0
@@ -612,7 +612,7 @@
         assert a[2] == -0.005
 
     def test_setitem_tuple(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         raises(IndexError, "a[(1,2)] = [0,1]")
         for i in xrange(5):
@@ -630,25 +630,25 @@
         assert a[2] == 100
 
     def test_setitem_obj_index(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         a[self.CustomIndexObject(1)] = 100
         assert a[1] == 100
 
     def test_setitem_obj_prefer_index_to_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         a[self.CustomIndexIntObject(0, 1)] = 100
         assert a[0] == 100
 
     def test_setitem_obj_int(self):
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         a[self.CustomIntObject(1)] = 100
         assert a[1] == 100
 
     def test_delitem(self):
-        import numpypy as np
+        import numpy as np
         a = np.arange(10)
         exc = raises(ValueError, 'del a[2]')
         assert exc.value.message == 'cannot delete array elements'
@@ -665,7 +665,7 @@
         # numpy will swallow errors in __int__ and __index__ and
         # just raise IndexError.
 
-        from numpypy import arange
+        from numpy import arange
         a = arange(10)
         exc = raises(IndexError, "a[ErrorIndex()] == 0")
         assert exc.value.message == 'cannot convert index to integer'
@@ -673,7 +673,7 @@
         assert exc.value.message == 'cannot convert index to integer'
 
     def test_setslice_array(self):
-        from numpypy import array
+        from numpy import array
         a = array(5)
         exc = raises(ValueError, "a[:] = 4")
         assert exc.value[0] == "cannot slice a 0-d array"
@@ -687,7 +687,7 @@
         assert b[1] == 0.
 
     def test_setslice_of_slice_array(self):
-        from numpypy import array, zeros
+        from numpy import array, zeros
         a = zeros(5)
         a[::2] = array([9., 10., 11.])
         assert a[0] == 9.
@@ -706,7 +706,7 @@
         assert a[0] == 3.
 
     def test_setslice_list(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5), float)
         b = [0., 1.]
         a[1:4:2] = b
@@ -714,7 +714,7 @@
         assert a[3] == 1.
 
     def test_setslice_constant(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5), float)
         a[1:4:2] = 0.
         assert a[1] == 0.
@@ -722,7 +722,7 @@
 
     def test_newaxis(self):
         import math
-        from numpypy import array, cos, zeros, newaxis
+        from numpy import array, cos, zeros, newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
@@ -743,7 +743,7 @@
         assert o == 3
 
     def test_newaxis_slice(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
 
         a = array(range(5))
         b = array(range(1,5))
@@ -755,14 +755,14 @@
         assert (a[newaxis,1:] == c).all()
 
     def test_newaxis_assign(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
 
         a = array(range(5))
         a[newaxis,1] = [2]
         assert a[1] == 2
 
     def test_newaxis_virtual(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
 
         a = array(range(5))
         b = (a + a)[newaxis]
@@ -770,20 +770,20 @@
         assert (b == c).all()
 
     def test_newaxis_then_slice(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
         a = array(range(5))
         b = a[newaxis]
         assert b.shape == (1, 5)
         assert (b[0,1:] == a[1:]).all()
 
     def test_slice_then_newaxis(self):
-        from numpypy import array, newaxis
+        from numpy import array, newaxis
         a = array(range(5))
         b = a[2:]
         assert (b[newaxis] == [[2, 3, 4]]).all()
 
     def test_scalar(self):
-        from numpypy import array, dtype, int_
+        from numpy import array, dtype, int_
         a = array(3)
         exc = raises(IndexError, "a[0]")
         assert exc.value[0] == "0-d arrays can't be indexed"
@@ -816,13 +816,13 @@
         assert a == 2.5
 
     def test_len(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert len(a) == 5
         assert len(a + a) == 5
 
     def test_shape(self):
-        from numpypy import array
+        from numpy import array
         a = array(range(5))
         assert a.shape == (5,)

From noreply at buildbot.pypy.org  Wed Dec 24 20:36:10 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 24 Dec 2014 20:36:10 +0100 (CET)
Subject: [pypy-commit] pypy default: Clarify the assertion error.
Message-ID: <20141224193610.3A7C71C0501@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75107:8e44078cb486
Date: 2014-12-24 20:36 +0100
http://bitbucket.org/pypy/pypy/changeset/8e44078cb486/

Log:	Clarify the assertion error.

diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py
--- a/rpython/jit/metainterp/virtualizable.py
+++ b/rpython/jit/metainterp/virtualizable.py
@@ -49,7 +49,13 @@
             ARRAYPTR = fieldType(VTYPE, name)
             ARRAY = deref(ARRAYPTR)
             assert isinstance(ARRAYPTR, lltype.Ptr)
-            assert isinstance(ARRAY, lltype.GcArray)
+            if not isinstance(ARRAY, lltype.GcArray):
+                raise Exception(
+                    "The virtualizable field '%s' is not an array (found %r)."
+                    " It usually means that you must try harder to ensure that"
+                    " the list is not resized at run-time. You can do that by"
+                    " using rpython.rlib.debug.make_sure_not_resized()." %
+                    (name, ARRAY))
             ARRAYITEMTYPES.append(arrayItem(ARRAY))
         self.array_descrs = [cpu.arraydescrof(deref(fieldType(VTYPE, name)))
                              for name in array_fields]

From noreply at buildbot.pypy.org  Thu Dec 25 11:46:00 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 11:46:00 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: ups
Message-ID: <20141225104600.586671C31CF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75108:ad48827961e0
Date: 2014-12-25 12:45 +0200
http://bitbucket.org/pypy/pypy/changeset/ad48827961e0/

Log:	ups

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2171,7 +2171,7 @@
         # store the resumekey.wref_original_loop_token() on 'self' to make
         # sure that it stays alive as long as this MetaInterp
         # XXXX this is stored these days via rd_loop_token on resumedescr
-        #self.resumekey_original_loop_token = key.wref_original_loop_token()
+        self.resumekey_original_loop_token = key.rd_loop
         self.staticdata.try_to_free_some_loops()
         self.initialize_state_from_guard_failure(key, deadframe)
         try:

From noreply at buildbot.pypy.org  Thu Dec 25 11:48:21 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 11:48:21 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: another try
Message-ID: <20141225104821.DDB771C31CF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75109:193143ca545c
Date: 2014-12-25 12:48 +0200
http://bitbucket.org/pypy/pypy/changeset/193143ca545c/

Log:	another try

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -192,8 +192,6 @@
             positions[i] = rffi.cast(rffi.USHORT, position)
         # write down the positions of locs
         guardtok.faildescr.rd_locs = positions
-        # we want the descr to keep alive
-        guardtok.faildescr.rd_loop_token = self.current_clt
         return fail_descr, target
 
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -66,8 +66,8 @@
     for op in loop.operations:
         descr = op.getdescr()
         # not sure what descr.index is about
-        #if isinstance(descr, ResumeDescr):
-            #descr.wref_original_loop_token = wref   # stick it there
+        if isinstance(descr, ResumeDescr):
+            descr.rd_loop_token = original_jitcell_token   # stick it there
             #n = descr.index
             #if n >= 0:       # we also record the resumedescr number
             #    original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2171,7 +2171,7 @@
         # store the resumekey.wref_original_loop_token() on 'self' to make
         # sure that it stays alive as long as this MetaInterp
         # XXXX this is stored these days via rd_loop_token on resumedescr
-        self.resumekey_original_loop_token = key.rd_loop
+        self.resumekey_original_loop_token = key.rd_loop_token
         self.staticdata.try_to_free_some_loops()
         self.initialize_state_from_guard_failure(key, deadframe)
         try:

From noreply at buildbot.pypy.org  Thu Dec 25 11:56:20 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 11:56:20 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: skip that test,
 it has not been true on x86 backedn for a bit forever
Message-ID: <20141225105620.D9AF21D239C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75110:cfcdde2f0eb6
Date: 2014-12-25 12:56 +0200
http://bitbucket.org/pypy/pypy/changeset/cfcdde2f0eb6/

Log:	skip that test, it has not been true on x86 backedn for a bit
	forever

diff --git a/rpython/jit/metainterp/test/test_memmgr.py b/rpython/jit/metainterp/test/test_memmgr.py
--- a/rpython/jit/metainterp/test/test_memmgr.py
+++ b/rpython/jit/metainterp/test/test_memmgr.py
@@ -186,6 +186,7 @@
         self.check_enter_count(2 + 10*4)
 
     def test_call_assembler_keep_alive(self):
+        py.test.skip("descrs keep loops alive, not sure why")
         myjitdriver1 = JitDriver(greens=['m'], reds=['n'])
         myjitdriver2 = JitDriver(greens=['m'], reds=['n', 'rec'])
         def h(m, n):

From noreply at buildbot.pypy.org  Thu Dec 25 13:28:03 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 13:28:03 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: yet another attempt -
 dont allocate descrs for not emitted guards at all
Message-ID: <20141225122803.4C83F1D2859@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75111:cd94f2a8fe4a
Date: 2014-12-25 14:27 +0200
http://bitbucket.org/pypy/pypy/changeset/cd94f2a8fe4a/

Log:	yet another attempt - dont allocate descrs for not emitted guards at
	all (this nicely solves some nasty optimizeopt test questions in
	addition to being slightly better)

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -130,8 +130,8 @@
                       [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
 
     try:
-        start_state = optimize_trace(metainterp_sd, part, enable_opts,
-                                     export_state=True)
+        start_state = optimize_trace(metainterp_sd, jitdriver_sd, part,
+                                     enable_opts, export_state=True)
     except InvalidLoop:
         return None
     target_token = part.operations[0].getdescr()
@@ -158,7 +158,7 @@
         jumpargs = part.operations[-1].getarglist()
 
         try:
-            optimize_trace(metainterp_sd, part, enable_opts,
+            optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
                            start_state=start_state, export_state=False)
         except InvalidLoop:
             return None
@@ -211,7 +211,8 @@
     orignial_label = label.clone()
     assert label.getopnum() == rop.LABEL
     try:
-        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
+        optimize_trace(metainterp_sd, jitdriver_sd, part,
+                       jitdriver_sd.warmstate.enable_opts,
                        start_state=start_state, export_state=False)
     except InvalidLoop:
         # Fall back on jumping to preamble
@@ -221,7 +222,7 @@
                           [ResOperation(rop.JUMP, inputargs[:],
                                         None, descr=loop_jitcell_token)]
         try:
-            optimize_trace(metainterp_sd, part,
+            optimize_trace(metainterp_sd, jitdriver_sd, part,
                            jitdriver_sd.warmstate.enable_opts,
                            inline_short_preamble=False, start_state=start_state,
                            export_state=False)
@@ -496,11 +497,6 @@
 
     status = r_uint(0)
 
-    def clone(self):
-        new = self.__class__()
-        new.rd_frame_info_list = self.rd_frame_info_list
-        return new
-
     def copy_all_attributes_from(self, other):
         assert isinstance(other, ResumeGuardDescr)
         self.rd_count = other.rd_count
@@ -712,12 +708,6 @@
         self.metainterp_sd = metainterp_sd
         self.jitdriver_sd = jitdriver_sd
 
-    def clone(self):
-        new = ResumeGuardForcedDescr()
-        new._init(self.metainterp_sd, self.jitdriver_sd)
-        new.rd_frame_info_list = self.rd_frame_info_list
-        return new
-
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         # Failures of a GUARD_NOT_FORCED are never compiled, but
         # always just blackholed.  First fish for the data saved when
@@ -775,6 +765,41 @@
         hidden_all_virtuals = obj.hide(metainterp_sd.cpu)
         metainterp_sd.cpu.set_savedata_ref(deadframe, hidden_all_virtuals)
 
+def invent_fail_descr_for_op(op, optimizer):
+    opnum = op.getopnum()
+    if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
+        resumedescr = ResumeGuardForcedDescr()
+        resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd)
+    elif opnum == rop.GUARD_NOT_INVALIDATED:
+        resumedescr = ResumeGuardNotInvalidated()
+    elif opnum == rop.GUARD_FUTURE_CONDITION:
+        resumedescr = ResumeAtPositionDescr()
+    elif opnum == rop.GUARD_VALUE:
+        resumedescr = ResumeGuardValueDescr()
+    elif opnum == rop.GUARD_NONNULL:
+        resumedescr = ResumeGuardNonnullDescr()
+    elif opnum == rop.GUARD_ISNULL:
+        resumedescr = ResumeGuardIsnullDescr()
+    elif opnum == rop.GUARD_NONNULL_CLASS:
+        resumedescr = ResumeGuardNonnullClassDescr()
+    elif opnum == rop.GUARD_CLASS:
+        resumedescr = ResumeGuardClassDescr()
+    elif opnum == rop.GUARD_TRUE:
+        resumedescr = ResumeGuardTrueDescr()
+    elif opnum == rop.GUARD_FALSE:
+        resumedescr = ResumeGuardFalseDescr()
+    elif opnum == rop.GUARD_EXCEPTION:
+        resumedescr = ResumeGuardExceptionDescr()
+    elif opnum == rop.GUARD_NO_EXCEPTION:
+        resumedescr = ResumeGuardNoExceptionDescr()
+    elif opnum == rop.GUARD_OVERFLOW:
+        resumedescr = ResumeGuardOverflowDescr()
+    elif opnum == rop.GUARD_NO_OVERFLOW:
+        resumedescr = ResumeGuardNoOverflowDescr()
+    else:
+        assert False
+    return resumedescr
+
 class ResumeFromInterpDescr(ResumeDescr):
     def __init__(self, original_greenkey):
         self.original_greenkey = original_greenkey
@@ -813,13 +838,15 @@
 
     new_trace.operations = [op.clone() for op in metainterp.history.operations]
     metainterp_sd = metainterp.staticdata
-    state = metainterp.jitdriver_sd.warmstate
+    jitdriver_sd = metainterp.jitdriver_sd
+    state = jitdriver_sd.warmstate
     if isinstance(resumekey, ResumeAtPositionDescr):
         inline_short_preamble = False
     else:
         inline_short_preamble = True
     try:
-        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts,
+        state = optimize_trace(metainterp_sd, jitdriver_sd, new_trace,
+                               state.enable_opts,
                                inline_short_preamble, export_state=True)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
diff --git a/rpython/jit/metainterp/jitexc.py b/rpython/jit/metainterp/jitexc.py
--- a/rpython/jit/metainterp/jitexc.py
+++ b/rpython/jit/metainterp/jitexc.py
@@ -41,6 +41,8 @@
 
 class ExitFrameWithExceptionRef(JitException):
     def __init__(self, cpu, value):
+        import pdb
+        pdb.set_trace()
         assert lltype.typeOf(value) == cpu.ts.BASETYPE
         self.value = value
     def __str__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -47,7 +47,7 @@
 
     return optimizations, unroll
 
-def optimize_trace(metainterp_sd, loop, enable_opts,
+def optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts,
                    inline_short_preamble=True, start_state=None,
                    export_state=True):
     """Optimize loop.operations to remove internal overheadish operations.
@@ -59,11 +59,13 @@
                                                           loop.operations)
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
-            return optimize_unroll(metainterp_sd, loop, optimizations,
+            return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
+                                   optimizations,
                                    inline_short_preamble, start_state,
                                    export_state)
         else:
-            optimizer = Optimizer(metainterp_sd, loop, optimizations)
+            optimizer = Optimizer(metainterp_sd, jitdriver_sd, loop,
+                                  optimizations)
             optimizer.propagate_all_forward()
     finally:
         debug_stop("jit-optimize")
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -502,8 +502,9 @@
 
 class Optimizer(Optimization):
 
-    def __init__(self, metainterp_sd, loop, optimizations=None):
+    def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations=None):
         self.metainterp_sd = metainterp_sd
+        self.jitdriver_sd = jitdriver_sd
         self.cpu = metainterp_sd.cpu
         self.loop = loop
         self.values = {}
@@ -748,7 +749,12 @@
 
     def store_final_boxes_in_guard(self, op, pendingfields):
         assert pendingfields is not None
-        descr = op.getdescr()
+        if op.getdescr() is not None:
+            descr = op.getdescr()
+            assert isinstance(descr, compile.ResumeAtPositionDescr)
+        else:
+            descr = compile.invent_fail_descr_for_op(op, self)
+            op.setdescr(descr)
         assert isinstance(descr, compile.ResumeGuardDescr)
         assert isinstance(op, GuardResOp)
         modifier = resume.ResumeDataVirtualAdder(descr, op,
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -17,16 +17,16 @@
     b0 = BoxInt()
     b1 = BoxInt()
     opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu),
-                                None)
-    fdescr = ResumeGuardDescr()
-    op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr)
+                                None, None)
+    op = ResOperation(rop.GUARD_TRUE, ['dummy'], None)
     # setup rd data
     fi0 = resume.FrameInfo(None, "code0", 11)
-    fdescr.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33)
     snapshot0 = resume.Snapshot(None, [b0])
     op.rd_snapshot = resume.Snapshot(snapshot0, [b1])
+    op.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33)
     #
     opt.store_final_boxes_in_guard(op, [])
+    fdescr = op.getdescr()
     if op.getfailargs() == [b0, b1]:
         assert list(fdescr.rd_numb.nums)      == [tag(1, TAGBOX)]
         assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -361,36 +361,23 @@
 class BaseTest(object):
 
     def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None):
-        if want_fail_descr:
-            invent_fail_descr = self.invent_fail_descr
-        else:
-            invent_fail_descr = lambda *args: None
         self.oparse = OpParser(s, self.cpu, self.namespace, 'lltype',
                                boxkinds,
-                               invent_fail_descr, False, postprocess)
+                               None, False, postprocess)
         return self.oparse.parse()
 
     def postprocess(self, op):
         if op.is_guard():
             op.rd_snapshot = resume.Snapshot(None, op.getfailargs())
-            op.getdescr().rd_frame_info_list = resume.FrameInfo(None, "code", 11)
+            op.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
 
     def add_guard_future_condition(self, res):
         # invent a GUARD_FUTURE_CONDITION to not have to change all tests
         if res.operations[-1].getopnum() == rop.JUMP:
-            guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None, descr=self.invent_fail_descr(None, rop.GUARD_FUTURE_CONDITION, []))
+            guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None)
             guard.rd_snapshot = resume.Snapshot(None, [])
             res.operations.insert(-1, guard)
 
-    def invent_fail_descr(self, model, opnum, fail_args):
-        if rop._GUARD_FIRST <= opnum <= rop._GUARD_LAST:
-            descr = Storage()
-            descr.guard_opnum = opnum
-            descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-        else:
-            descr = final_descr
-        return descr
-
     def assert_equal(self, optimized, expected, text_right=None):
         from rpython.jit.metainterp.optimizeopt.util import equaloplists
         assert len(optimized.inputargs) == len(expected.inputargs)
@@ -417,7 +404,8 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        return optimize_trace(metainterp_sd, loop, self.enable_opts,
+        return optimize_trace(metainterp_sd, None, loop,
+                              self.enable_opts,
                               start_state=start_state,
                               export_state=export_state)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -799,7 +799,7 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        optimize_trace(metainterp_sd, bridge, self.enable_opts)
+        optimize_trace(metainterp_sd, None, bridge, self.enable_opts)
 
         
     def optimize_bridge(self, loops, bridge, expected, expected_target='Loop', **boxvalues):
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -9,15 +9,16 @@
         ShortBoxes, BadVirtualState, VirtualStatesCantMatch)
 from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp
 from rpython.jit.metainterp.resume import Snapshot
+from rpython.jit.metainterp import compile
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
 
 
 # FIXME: Introduce some VirtualOptimizer super class instead
 
-def optimize_unroll(metainterp_sd, loop, optimizations,
+def optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble=True, start_state=None,
                     export_state=True):
-    opt = UnrollOptimizer(metainterp_sd, loop, optimizations)
+    opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, loop, optimizations)
     opt.inline_short_preamble = inline_short_preamble
     return opt.propagate_all_forward(start_state, export_state)
 
@@ -51,8 +52,9 @@
 
     inline_short_preamble = True
 
-    def __init__(self, metainterp_sd, loop, optimizations):
-        self.optimizer = UnrollableOptimizer(metainterp_sd, loop, optimizations)
+    def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations):
+        self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
+                                             loop, optimizations)
         self.boxes_created_this_iteration = None
 
     def get_virtual_state(self, args):
@@ -559,11 +561,10 @@
 
             for guard in extra_guards:
                 if guard.is_guard():
-                    descr = patchguardop.getdescr().clone()
-                    assert isinstance(guard, GuardResOp)
                     assert isinstance(patchguardop, GuardResOp)
                     guard.rd_snapshot = patchguardop.rd_snapshot
-                    guard.setdescr(descr)
+                    guard.rd_frame_info_list = patchguardop.rd_frame_info_list
+                    guard.setdescr(compile.ResumeAtPositionDescr())
                 self.optimizer.send_extra_operation(guard)
 
             try:
@@ -592,11 +593,11 @@
             if newop.is_guard():
                 if not patchguardop:
                     raise InvalidLoop("would like to have short preamble, but it has a guard and there's no guard_future_condition")
-                descr = patchguardop.getdescr().clone()
                 assert isinstance(newop, GuardResOp)
                 assert isinstance(patchguardop, GuardResOp)
                 newop.rd_snapshot = patchguardop.rd_snapshot
-                newop.setdescr(descr)
+                newop.rd_frame_info_list = patchguardop.rd_frame_info_list
+                newop.setdescr(compile.ResumeAtPositionDescr())
             self.optimizer.send_extra_operation(newop)
             if shop.result in assumed_classes:
                 classbox = self.getvalue(newop.result).get_constant_class(self.optimizer.cpu)
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -164,8 +164,6 @@
                 assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
-        if op1.getopnum() not in (rop.JUMP, rop.LABEL):      # xxx obscure
-            assert op1.getdescr() == op2.getdescr()
         if op1.getfailargs() or op2.getfailargs():
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1874,46 +1874,15 @@
             moreargs = [box] + extraargs
         else:
             moreargs = list(extraargs)
-        if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
-            resumedescr = compile.ResumeGuardForcedDescr()
-            resumedescr._init(self.staticdata, self.jitdriver_sd)
-        elif opnum == rop.GUARD_NOT_INVALIDATED:
-            resumedescr = compile.ResumeGuardNotInvalidated()
-        elif opnum == rop.GUARD_FUTURE_CONDITION:
-            resumedescr = compile.ResumeAtPositionDescr()
-        elif opnum == rop.GUARD_VALUE:
-            resumedescr = compile.ResumeGuardValueDescr()
-        elif opnum == rop.GUARD_NONNULL:
-            resumedescr = compile.ResumeGuardNonnullDescr()
-        elif opnum == rop.GUARD_ISNULL:
-            resumedescr = compile.ResumeGuardIsnullDescr()
-        elif opnum == rop.GUARD_NONNULL_CLASS:
-            resumedescr = compile.ResumeGuardNonnullClassDescr()
-        elif opnum == rop.GUARD_CLASS:
-            resumedescr = compile.ResumeGuardClassDescr()
-        elif opnum == rop.GUARD_TRUE:
-            resumedescr = compile.ResumeGuardTrueDescr()
-        elif opnum == rop.GUARD_FALSE:
-            resumedescr = compile.ResumeGuardFalseDescr()
-        elif opnum == rop.GUARD_EXCEPTION:
-            resumedescr = compile.ResumeGuardExceptionDescr()
-        elif opnum == rop.GUARD_NO_EXCEPTION:
-            resumedescr = compile.ResumeGuardNoExceptionDescr()
-        elif opnum == rop.GUARD_OVERFLOW:
-            resumedescr = compile.ResumeGuardOverflowDescr()
-        elif opnum == rop.GUARD_NO_OVERFLOW:
-            resumedescr = compile.ResumeGuardNoOverflowDescr()
-        else:
-            assert False
-        guard_op = self.history.record(opnum, moreargs, None, descr=resumedescr)
+        guard_op = self.history.record(opnum, moreargs, None)
         assert isinstance(guard_op, GuardResOp)
-        self.capture_resumedata(guard_op, resumedescr, resumepc)
+        self.capture_resumedata(guard_op, resumepc)
         self.staticdata.profiler.count_ops(opnum, Counters.GUARDS)
         # count
         self.attach_debug_info(guard_op)
         return guard_op
 
-    def capture_resumedata(self, guard_op, resumedescr, resumepc=-1):
+    def capture_resumedata(self, guard_op, resumepc=-1):
         virtualizable_boxes = None
         if (self.jitdriver_sd.virtualizable_info is not None or
             self.jitdriver_sd.greenfield_info is not None):
@@ -1925,7 +1894,7 @@
             if resumepc >= 0:
                 frame.pc = resumepc
         resume.capture_resumedata(self.framestack, virtualizable_boxes,
-                                  self.virtualref_boxes, resumedescr, guard_op)
+                                  self.virtualref_boxes, guard_op)
         if self.framestack:
             self.framestack[-1].pc = saved_pc
 
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -216,6 +216,7 @@
     _fail_args = None
 
     rd_snapshot = None
+    rd_frame_info_list = None
 
     def getfailargs(self):
         return self._fail_args
@@ -228,6 +229,7 @@
         assert isinstance(newop, GuardResOp)
         newop.setfailargs(self.getfailargs())
         newop.rd_snapshot = self.rd_snapshot
+        newop.rd_frame_info_list = self.rd_frame_info_list
         return newop
 
     def clone(self):
@@ -235,10 +237,7 @@
         assert isinstance(newop, GuardResOp)
         newop.setfailargs(self.getfailargs())
         newop.rd_snapshot = self.rd_snapshot
-        descr = self.getdescr()
-        if descr is not None: # for short preamble which is a mess
-            newdescr = descr.clone()
-            newop.setdescr(newdescr)
+        newop.rd_frame_info_list = self.rd_frame_info_list
         return newop
 
 # ============
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -55,7 +55,7 @@
                                          back.get_list_of_active_boxes(True))
 
 def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes,
-                       storage, snapshot_storage):
+                       snapshot_storage):
     n = len(framestack) - 1
     if virtualizable_boxes is not None:
         boxes = virtualref_boxes + virtualizable_boxes
@@ -66,13 +66,13 @@
         _ensure_parent_resumedata(framestack, n)
         frame_info_list = FrameInfo(top.parent_resumedata_frame_info_list,
                                     top.jitcode, top.pc)
-        storage.rd_frame_info_list = frame_info_list
+        snapshot_storage.rd_frame_info_list = frame_info_list
         snapshot = Snapshot(top.parent_resumedata_snapshot,
                             top.get_list_of_active_boxes(False))
         snapshot = Snapshot(snapshot, boxes)
         snapshot_storage.rd_snapshot = snapshot
     else:
-        storage.rd_frame_info_list = None
+        snapshot_storage.rd_frame_info_list = None
         snapshot_storage.rd_snapshot = Snapshot(None, boxes)
 
 #
@@ -365,6 +365,7 @@
         self.liveboxes = {}
         storage.rd_numb = numb
         self.snapshot_storage.rd_snapshot = None
+        storage.rd_frame_info_list = self.snapshot_storage.rd_frame_info_list
 
         # collect liveboxes and virtuals
         n = len(liveboxes_from_env) - v
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -2764,7 +2764,8 @@
             return i
         #
         seen = []
-        def my_optimize_trace(metainterp_sd, loop, enable_opts, *args, **kwds):
+        def my_optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts,
+                              *args, **kwds):
             seen.append('unroll' in enable_opts)
             raise InvalidLoop
         old_optimize_trace = optimizeopt.optimize_trace

From noreply at buildbot.pypy.org  Thu Dec 25 13:28:51 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 13:28:51 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: don't commit pdb
Message-ID: <20141225122851.D92971D2859@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75112:f7b6d854393e
Date: 2014-12-25 14:28 +0200
http://bitbucket.org/pypy/pypy/changeset/f7b6d854393e/

Log:	don't commit pdb

diff --git a/rpython/jit/metainterp/jitexc.py b/rpython/jit/metainterp/jitexc.py
--- a/rpython/jit/metainterp/jitexc.py
+++ b/rpython/jit/metainterp/jitexc.py
@@ -41,8 +41,6 @@
 
 class ExitFrameWithExceptionRef(JitException):
     def __init__(self, cpu, value):
-        import pdb
-        pdb.set_trace()
         assert lltype.typeOf(value) == cpu.ts.BASETYPE
         self.value = value
     def __str__(self):

From noreply at buildbot.pypy.org  Thu Dec 25 13:54:26 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 13:54:26 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: ups, we missed it somehow
Message-ID: <20141225125426.3A0391C03D5@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75113:cf41cc016bb6
Date: 2014-12-25 14:54 +0200
http://bitbucket.org/pypy/pypy/changeset/cf41cc016bb6/

Log:	ups, we missed it somehow

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -562,6 +562,7 @@
             for guard in extra_guards:
                 if guard.is_guard():
                     assert isinstance(patchguardop, GuardResOp)
+                    assert isinstance(guard, GuardResOp)
                     guard.rd_snapshot = patchguardop.rd_snapshot
                     guard.rd_frame_info_list = patchguardop.rd_frame_info_list
                     guard.setdescr(compile.ResumeAtPositionDescr())

From noreply at buildbot.pypy.org  Thu Dec 25 20:34:02 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 20:34:02 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: Improve the logic to be
 exactly how it was before, but instead of storing
Message-ID: <20141225193402.E0B981D288E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75114:49886da188a7
Date: 2014-12-25 21:32 +0200
http://bitbucket.org/pypy/pypy/changeset/49886da188a7/

Log:	Improve the logic to be exactly how it was before, but instead of
	storing wref on each descr, store wref on compiled_loop_token (which
	has to be referenced from descr anyway)

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -63,11 +63,13 @@
     if metainterp_sd.warmrunnerdesc is not None:    # for tests
         assert original_jitcell_token.generation > 0     # has been registered with memmgr
     wref = weakref.ref(original_jitcell_token)
+    clt = original_jitcell_token.compiled_loop_token
+    clt.loop_token_wref = wref
     for op in loop.operations:
         descr = op.getdescr()
         # not sure what descr.index is about
         if isinstance(descr, ResumeDescr):
-            descr.rd_loop_token = original_jitcell_token   # stick it there
+            descr.rd_loop_token = clt   # stick it there
             #n = descr.index
             #if n >= 0:       # we also record the resumedescr number
             #    original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2139,8 +2139,9 @@
         assert isinstance(key, compile.ResumeGuardDescr)
         # store the resumekey.wref_original_loop_token() on 'self' to make
         # sure that it stays alive as long as this MetaInterp
-        # XXXX this is stored these days via rd_loop_token on resumedescr
-        self.resumekey_original_loop_token = key.rd_loop_token
+        self.resumekey_original_loop_token = key.rd_loop_token.loop_token_wref()
+        if self.resumekey_original_loop_token is None:
+            self.giveup() # should be rare
         self.staticdata.try_to_free_some_loops()
         self.initialize_state_from_guard_failure(key, deadframe)
         try:

From noreply at buildbot.pypy.org  Thu Dec 25 20:34:04 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 20:34:04 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: fix a test
Message-ID: <20141225193404.14C8B1D288E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75115:ce1577763894
Date: 2014-12-25 21:33 +0200
http://bitbucket.org/pypy/pypy/changeset/ce1577763894/

Log:	fix a test

diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -10,6 +10,9 @@
 from rpython.jit.metainterp.optimizeopt import ALL_OPTS_DICT
 
 class FakeCPU(object):
+    class Storage:
+        pass
+    
     class tracker:
         pass
 
@@ -18,6 +21,7 @@
         self.seen = []
     def compile_loop(self, inputargs, operations, token, log=True, name='',
                      logger=None):
+        token.compiled_loop_token = self.Storage()
         self.seen.append((inputargs, operations, token))
 
 class FakeLogger(object):

From noreply at buildbot.pypy.org  Thu Dec 25 20:38:04 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 20:38:04 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: now I know why, unskip
Message-ID: <20141225193804.A49411D288E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75116:5c4892d864fd
Date: 2014-12-25 21:38 +0200
http://bitbucket.org/pypy/pypy/changeset/5c4892d864fd/

Log:	now I know why, unskip

diff --git a/rpython/jit/metainterp/test/test_memmgr.py b/rpython/jit/metainterp/test/test_memmgr.py
--- a/rpython/jit/metainterp/test/test_memmgr.py
+++ b/rpython/jit/metainterp/test/test_memmgr.py
@@ -186,7 +186,6 @@
         self.check_enter_count(2 + 10*4)
 
     def test_call_assembler_keep_alive(self):
-        py.test.skip("descrs keep loops alive, not sure why")
         myjitdriver1 = JitDriver(greens=['m'], reds=['n'])
         myjitdriver2 = JitDriver(greens=['m'], reds=['n', 'rec'])
         def h(m, n):

From noreply at buildbot.pypy.org  Thu Dec 25 20:55:58 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 20:55:58 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: fix this test
Message-ID: <20141225195558.5AED91D2891@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75117:4d69e7d465fc
Date: 2014-12-25 21:39 +0200
http://bitbucket.org/pypy/pypy/changeset/4d69e7d465fc/

Log:	fix this test

diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -542,7 +542,7 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2)]
 
     storage = Storage()
-    capture_resumedata(fs, None, [], storage, storage)
+    capture_resumedata(fs, None, [], storage)
 
     assert fs[0].parent_resumedata_snapshot is None
     assert fs[0].parent_resumedata_frame_info_list is None
@@ -558,7 +558,7 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2),
           FakeFrame("code1", 3, b3, c2, b1),
           FakeFrame("code2", 9, c3, b2)]
-    capture_resumedata(fs, None, [], storage, storage)
+    capture_resumedata(fs, None, [], storage)
 
     frame_info_list = storage.rd_frame_info_list
     assert frame_info_list.prev is fs[2].parent_resumedata_frame_info_list
@@ -588,7 +588,7 @@
     fs[2].pc = 15
     vbs = [b1, b2]
     vrs = [b3]
-    capture_resumedata(fs, vbs, vrs, storage, storage)
+    capture_resumedata(fs, vbs, vrs, storage)
 
     frame_info_list = storage.rd_frame_info_list
     assert frame_info_list.prev is fs[2].parent_resumedata_frame_info_list
@@ -616,7 +616,7 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2),
           FakeFrame("code1", 3, b3, c2, b1),
           FakeFrame("code2", 9, c3, b2)]
-    capture_resumedata(fs, None, [], storage, storage)
+    capture_resumedata(fs, None, [], storage)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     modifier = ResumeDataVirtualAdder(storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
@@ -640,7 +640,7 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2),
           FakeFrame("code1", 3, b3, c2, b1),
           FakeFrame("code2", 9, c3, b2)]
-    capture_resumedata(fs, [b4], [], storage, storage)
+    capture_resumedata(fs, [b4], [], storage)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     modifier = ResumeDataVirtualAdder(storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
@@ -665,10 +665,10 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2),
           FakeFrame("code1", 3, b3, c2, b1),
           FakeFrame("code2", 9, c3, b2)]
-    capture_resumedata(fs, None, [], storage, storage)
+    capture_resumedata(fs, None, [], storage)
     storage2 = Storage()
     fs = fs[:-1] + [FakeFrame("code2", 10, c3, b2, b4)]
-    capture_resumedata(fs, None, [], storage2, storage2)
+    capture_resumedata(fs, None, [], storage2)
     
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     modifier = ResumeDataVirtualAdder(storage, memo)
@@ -728,10 +728,10 @@
     fs = [FakeFrame("code0", 0, b1, c1, b2),
           FakeFrame("code1", 3, b3, c2, b1),
           FakeFrame("code2", 9, c3, b2)]
-    capture_resumedata(fs, None, [], storage, storage)
+    capture_resumedata(fs, None, [], storage)
     storage2 = Storage()
     fs = fs[:-1] + [FakeFrame("code2", 10, c3, b2, b4)]
-    capture_resumedata(fs, None, [], storage2, storage2)
+    capture_resumedata(fs, None, [], storage2)
     
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     values = {b2: virtual_value(b2, b5, c4)}
@@ -788,7 +788,7 @@
                       LLtypeMixin.nodebox.constbox()]
     storage = Storage()
     fs = [FakeFrame("code0", 0, c1, b2, b3)]
-    capture_resumedata(fs, None, [], storage, storage)
+    capture_resumedata(fs, None, [], storage)
     
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     values = {b2: virtual_value(b2, b5, c4)}
@@ -800,7 +800,7 @@
 
     storage2 = Storage()
     fs = [FakeFrame("code0", 0, b1, b4, b2)]
-    capture_resumedata(fs, None, [], storage2, storage2)
+    capture_resumedata(fs, None, [], storage2)
     values[b4] = virtual_value(b4, b6, c4)
     modifier = ResumeDataVirtualAdder(storage2, memo)
     liveboxes = modifier.finish(FakeOptimizer(values))
@@ -814,7 +814,7 @@
     b1, b2, b3 = [BoxPtr(), BoxPtr(), BoxInt()]
     storage = Storage()
     fs = [FakeFrame("code0", 0, b1, b2)]
-    capture_resumedata(fs, None, [], storage, storage)
+    capture_resumedata(fs, None, [], storage)
     
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
     v1 = virtual_value(b1, b3, None)

From noreply at buildbot.pypy.org  Thu Dec 25 20:55:59 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 20:55:59 +0100 (CET)
Subject: [pypy-commit] pypy default: we can definbitely use a constant here
Message-ID: <20141225195559.A07B11D2891@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75118:a2cfe79ba785
Date: 2014-12-25 21:55 +0200
http://bitbucket.org/pypy/pypy/changeset/a2cfe79ba785/

Log:	we can definbitely use a constant here

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -624,7 +624,7 @@
             except KeyError:
                 pass
             else:
-                assert value.getlevel() != LEVEL_CONSTANT
+                #assert value.getlevel() != LEVEL_CONSTANT
                 assert cur_value.getlevel() != LEVEL_CONSTANT
                 # replacing with a different box
                 cur_value.copy_from(value)

From noreply at buildbot.pypy.org  Thu Dec 25 20:57:21 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 20:57:21 +0100 (CET)
Subject: [pypy-commit] pypy default: actually really kill that assert
Message-ID: <20141225195721.7FE981D2891@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75119:d3c2f3039388
Date: 2014-12-25 21:56 +0200
http://bitbucket.org/pypy/pypy/changeset/d3c2f3039388/

Log:	actually really kill that assert

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -624,7 +624,6 @@
             except KeyError:
                 pass
             else:
-                #assert value.getlevel() != LEVEL_CONSTANT
                 assert cur_value.getlevel() != LEVEL_CONSTANT
                 # replacing with a different box
                 cur_value.copy_from(value)

From noreply at buildbot.pypy.org  Thu Dec 25 20:57:22 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 20:57:22 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: grrr untested case
Message-ID: <20141225195722.B61291D2891@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75120:d2c413fe8562
Date: 2014-12-25 21:57 +0200
http://bitbucket.org/pypy/pypy/changeset/d2c413fe8562/

Log:	grrr untested case

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2141,7 +2141,7 @@
         # sure that it stays alive as long as this MetaInterp
         self.resumekey_original_loop_token = key.rd_loop_token.loop_token_wref()
         if self.resumekey_original_loop_token is None:
-            self.giveup() # should be rare
+            raise compile.giveup() # should be rare
         self.staticdata.try_to_free_some_loops()
         self.initialize_state_from_guard_failure(key, deadframe)
         try:

From noreply at buildbot.pypy.org  Thu Dec 25 21:47:15 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 21:47:15 +0100 (CET)
Subject: [pypy-commit] pypy pypy-dont-copy-ops: start working on lack of
	copying of ops
Message-ID: <20141225204715.688001C31CF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: pypy-dont-copy-ops
Changeset: r75121:1fd614fe5b30
Date: 2014-12-25 22:35 +0200
http://bitbucket.org/pypy/pypy/changeset/1fd614fe5b30/

Log:	start working on lack of copying of ops

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -123,9 +123,10 @@
     part = create_empty_loop(metainterp)
     part.inputargs = inputargs[:]
     h_ops = history.operations
-    part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \
-                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
-                      [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
+    label = ResOperation(rop.LABEL, inputargs, None,
+                         descr=TargetToken(jitcell_token))
+    end_label = ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)
+    part.operations = [label] + h_ops + [end_label]
 
     try:
         start_state = optimize_trace(metainterp_sd, part, enable_opts,
@@ -203,7 +204,7 @@
     h_ops = history.operations
 
     part.operations = [partial_trace.operations[-1]] + \
-                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
+                      h_ops + \
                       [ResOperation(rop.JUMP, jumpargs, None, descr=loop_jitcell_token)]
     label = part.operations[0]
     orignial_label = label.clone()
@@ -783,7 +784,7 @@
     new_trace.inputargs = metainterp.history.inputargs[:]
     # clone ops, as optimize_bridge can mutate the ops
 
-    new_trace.operations = [op.clone() for op in metainterp.history.operations]
+    new_trace.operations = metainterp.history.operations[:]
     metainterp_sd = metainterp.staticdata
     state = metainterp.jitdriver_sd.warmstate
     if isinstance(resumekey, ResumeAtPositionDescr):
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -74,7 +74,7 @@
         if other.getlevel() == LEVEL_CONSTANT:
             self.make_constant(other.get_key_box())
         elif other.getlevel() == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.get_known_class(), None)
+            self.make_constant_class(None, other.get_known_class())
         else:
             if other.getlevel() == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -192,7 +192,7 @@
         self.box = constbox
         self.setlevel(LEVEL_CONSTANT)
 
-    def get_last_guard(self):
+    def get_last_guard(self, optimizer):
         return None
 
     def get_known_class(self):
@@ -208,10 +208,10 @@
         return None
 
 class PtrOptValue(OptValue):
-    _attrs_ = ('known_class', 'last_guard', 'lenbound')
+    _attrs_ = ('known_class', 'last_guard_pos', 'lenbound')
 
     known_class = None
-    last_guard = None
+    last_guard_pos = -1
     lenbound = None
 
     def __init__(self, box, level=None, known_class=None, intbound=None):
@@ -224,7 +224,7 @@
         self.box = other_value.box
         self.known_class = other_value.known_class
         self._tag = other_value._tag
-        self.last_guard = other_value.last_guard
+        self.last_guard_pos = other_value.last_guard_pos
         self.lenbound = other_value.lenbound
 
     def make_len_gt(self, mode, descr, val):
@@ -235,16 +235,17 @@
         else:
             self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))
 
-    def make_nonnull(self, guardop):
+    def make_nonnull(self, optimizer):
         assert self.getlevel() < LEVEL_NONNULL
         self.setlevel(LEVEL_NONNULL)
-        self.last_guard = guardop
+        self.last_guard_pos = len(optimizer._newoperations)
 
-    def make_constant_class(self, classbox, guardop):
+    def make_constant_class(self, optimizer, classbox):
         assert self.getlevel() < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.setlevel(LEVEL_KNOWNCLASS)
-        self.last_guard = guardop
+        if optimizer is not None:
+            self.last_guard_pos = len(optimizer._newoperations)
 
     def import_from(self, other, optimizer):
         OptValue.import_from(self, other, optimizer)
@@ -299,8 +300,10 @@
     def getlenbound(self):
         return self.lenbound
 
-    def get_last_guard(self):
-        return self.last_guard
+    def get_last_guard(self, optimizer):
+        if self.last_guard_pos == -1:
+            return None
+        return optimizer._newoperations[self.last_guard_pos]
 
     def get_known_class(self):
         return self.known_class
@@ -345,7 +348,7 @@
                 return True
         return False
 
-    def make_nonnull(self, guardop):
+    def make_nonnull(self, optimizer):
         assert self.getlevel() < LEVEL_NONNULL
         self.setlevel(LEVEL_NONNULL)
 
@@ -545,6 +548,12 @@
 
         self.optimizations  = optimizations
 
+    def replace_guard(self, op, value):
+        assert isinstance(value, PtrOptValue)
+        if value.last_guard_pos == -1:
+            return
+        self.replaces_guard[op] = value.last_guard_pos
+
     def force_at_end_of_preamble(self):
         for o in self.optimizations:
             o.force_at_end_of_preamble()
@@ -705,6 +714,8 @@
     @specialize.argtype(0)
     def _emit_operation(self, op):
         assert op.getopnum() != rop.CALL_PURE
+        changed = False
+        orig_op = op
         for i in range(op.numargs()):
             arg = op.getarg(i)
             try:
@@ -713,32 +724,31 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box(self))
+                newbox = value.force_box(self)
+                if not changed and arg is not newbox:
+                    op = op.clone()
+                    changed = True
+                    op.setarg(i, newbox)
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.Counters.OPT_GUARDS)
             pendingfields = self.pendingfields
             self.pendingfields = None
-            if self.replaces_guard and op in self.replaces_guard:
-                self.replace_op(self.replaces_guard[op], op)
+            if self.replaces_guard and orig_op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[orig_op], op)
                 del self.replaces_guard[op]
                 return
             else:
-                op = self.store_final_boxes_in_guard(op, pendingfields)
+                guard_op = op.clone()
+                op = self.store_final_boxes_in_guard(guard_op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
         self._newoperations.append(op)
 
-    def replace_op(self, old_op, new_op):
-        # XXX: Do we want to cache indexes to prevent search?
-        i = len(self._newoperations)
-        while i > 0:
-            i -= 1
-            if self._newoperations[i] is old_op:
-                self._newoperations[i] = new_op
-                break
-        else:
-            assert False
+    def replace_op(self, old_op_pos, new_op):
+        old_op = self._newoperations[old_op_pos]
+        assert old_op.is_guard()
+        self._newoperations[old_op_pos] = new_op
 
     def store_final_boxes_in_guard(self, op, pendingfields):
         assert pendingfields is not None
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -284,7 +284,7 @@
             raise InvalidLoop('A GUARD_NONNULL (%s) was proven to always fail'
                               % r)
         self.emit_operation(op)
-        value.make_nonnull(op)
+        value.make_nonnull(self.optimizer)
 
     def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
@@ -296,11 +296,11 @@
             else:
                 name = "<unknown>"
             raise InvalidLoop('A promote of a virtual %s (a recently allocated object) never makes sense!' % name)
-        if value.get_last_guard():
+        old_guard_op = value.get_last_guard(self.optimizer)
+        if old_guard_op:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = value.get_last_guard()
             if old_guard_op.getopnum() != rop.GUARD_NONNULL:
                 # This is only safe if the class of the guard_value matches the
                 # class of the guard_*_class, otherwise the intermediate ops might
@@ -314,7 +314,7 @@
                     raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r)
             op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
                                       args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.replaces_guard[op] = old_guard_op
+            self.optimizer.replace_guard(op, value)
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
@@ -324,7 +324,7 @@
             descr.make_a_counter_per_value(op)
             # to be safe
             if isinstance(value, PtrOptValue):
-                value.last_guard = None
+                value.last_guard_pos = -1
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
         self.optimize_guard(op, constbox)
@@ -343,7 +343,7 @@
         if realclassbox is not None:
             assert realclassbox.same_constant(expectedclassbox)
             return
-        value.make_constant_class(expectedclassbox, None)
+        value.make_constant_class(None, expectedclassbox)
 
     def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
@@ -357,16 +357,16 @@
             raise InvalidLoop('A GUARD_CLASS (%s) was proven to always fail'
                               % r)
         assert isinstance(value, PtrOptValue)
-        if value.last_guard:
+        old_guard_op = value.get_last_guard(self.optimizer)
+        if old_guard_op:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
                 op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.replaces_guard[op] = old_guard_op
+                self.optimizer.replace_guard(op, value)
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
@@ -374,7 +374,7 @@
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
         self.emit_operation(op)
-        value.make_constant_class(expectedclassbox, op)
+        value.make_constant_class(self.optimizer, expectedclassbox)
 
     def optimize_GUARD_NONNULL_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -48,6 +48,7 @@
 
     def optimize_JUMP(self, op):
         if not self.unroll:
+            op = op.clone()
             descr = op.getdescr()
             assert isinstance(descr, JitCellToken)
             if not descr.target_tokens:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -134,6 +134,7 @@
                 else:
                     debug_print("Retrace count reached, jumping to preamble")
                     assert cell_token.target_tokens[0].virtual_state is None
+                    jumpop = jumpop.clone()
                     jumpop.setdescr(cell_token.target_tokens[0])
                     self.optimizer.send_extra_operation(jumpop)
                     return
@@ -506,6 +507,7 @@
             self.import_box(a, inputargs, short_jumpargs, jumpargs)
 
     def jump_to_already_compiled_trace(self, jumpop, patchguardop):
+        jumpop = jumpop.clone()
         assert jumpop.getopnum() == rop.JUMP
         cell_token = jumpop.getdescr()
 

From noreply at buildbot.pypy.org  Thu Dec 25 21:47:17 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 21:47:17 +0100 (CET)
Subject: [pypy-commit] pypy pypy-dont-copy-ops: merge slimming branch,
	it's easier that way
Message-ID: <20141225204717.570AD1C31CF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: pypy-dont-copy-ops
Changeset: r75122:a66a0e7c9caa
Date: 2014-12-25 22:37 +0200
http://bitbucket.org/pypy/pypy/changeset/a66a0e7c9caa/

Log:	merge slimming branch, it's easier that way

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -901,7 +901,7 @@
             descr = tok.faildescr
             assert isinstance(descr, AbstractFailDescr)
             failure_recovery_pos = block_start + tok.pos_recovery_stub
-            descr._arm_failure_recovery_block = failure_recovery_pos
+            descr.adr_jump_offset = failure_recovery_pos
             relative_offset = tok.pos_recovery_stub - tok.offset
             guard_pos = block_start + tok.offset
             if not tok.is_guard_not_invalidated:
@@ -968,11 +968,11 @@
 
     def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
         b = InstrBuilder(self.cpu.cpuinfo.arch_version)
-        patch_addr = faildescr._arm_failure_recovery_block
+        patch_addr = faildescr.adr_jump_offset
         assert patch_addr != 0
         b.B(bridge_addr)
         b.copy_to_raw_memory(patch_addr)
-        faildescr._arm_failure_recovery_block = 0
+        faildescr.adr_jump_offset = 0
 
     # regalloc support
     def load(self, loc, value):
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -192,8 +192,6 @@
             positions[i] = rffi.cast(rffi.USHORT, position)
         # write down the positions of locs
         guardtok.faildescr.rd_locs = positions
-        # we want the descr to keep alive
-        guardtok.faildescr.rd_loop_token = self.current_clt
         return fail_descr, target
 
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -572,13 +572,13 @@
 
     def patch_pending_failure_recoveries(self, rawstart):
         # after we wrote the assembler to raw memory, set up
-        # tok.faildescr._x86_adr_jump_offset to contain the raw address of
+        # tok.faildescr.adr_jump_offset to contain the raw address of
         # the 4-byte target field in the JMP/Jcond instruction, and patch
         # the field in question to point (initially) to the recovery stub
         clt = self.current_clt
         for tok in self.pending_guard_tokens:
             addr = rawstart + tok.pos_jump_offset
-            tok.faildescr._x86_adr_jump_offset = addr
+            tok.faildescr.adr_jump_offset = addr
             relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4)
             assert rx86.fits_in_32bits(relative_target)
             #
@@ -685,7 +685,7 @@
                                    self.cpu.gc_ll_descr.gcrootmap)
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
-        adr_jump_offset = faildescr._x86_adr_jump_offset
+        adr_jump_offset = faildescr.adr_jump_offset
         assert adr_jump_offset != 0
         offset = adr_new_target - (adr_jump_offset + 4)
         # If the new target fits within a rel32 of the jump, just patch
@@ -705,7 +705,7 @@
             p = rffi.cast(rffi.INTP, adr_jump_offset)
             adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0])
             mc.copy_to_raw_memory(adr_target)
-        faildescr._x86_adr_jump_offset = 0    # means "patched"
+        faildescr.adr_jump_offset = 0    # means "patched"
 
     def fixup_target_tokens(self, rawstart):
         for targettoken in self.target_tokens_currently_compiling:
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1479,15 +1479,15 @@
             assert kind == 'v'
         return lltype.nullptr(rclass.OBJECTPTR.TO)
 
-    def _prepare_resume_from_failure(self, opnum, dont_change_position,
-                                     deadframe):
+    def _prepare_resume_from_failure(self, opnum, deadframe):
         from rpython.jit.metainterp.resoperation import rop
         #
-        if opnum == rop.GUARD_TRUE:
+        if opnum == rop.GUARD_FUTURE_CONDITION:
+            pass
+        elif opnum == rop.GUARD_TRUE:
             # Produced directly by some goto_if_not_xxx() opcode that did not
             # jump, but which must now jump.  The pc is just after the opcode.
-            if not dont_change_position:
-                self.position = self.jitcode.follow_jump(self.position)
+            self.position = self.jitcode.follow_jump(self.position)
         #
         elif opnum == rop.GUARD_FALSE:
             # Produced directly by some goto_if_not_xxx() opcode that jumped,
@@ -1517,8 +1517,7 @@
         elif opnum == rop.GUARD_NO_OVERFLOW:
             # Produced by int_xxx_ovf().  The pc is just after the opcode.
             # We get here because it did not used to overflow, but now it does.
-            if not dont_change_position:
-                return get_llexception(self.cpu, OverflowError())
+            return get_llexception(self.cpu, OverflowError())
         #
         elif opnum == rop.GUARD_OVERFLOW:
             # Produced by int_xxx_ovf().  The pc is just after the opcode.
@@ -1649,13 +1648,9 @@
         resumedescr,
         deadframe,
         all_virtuals)
-    if isinstance(resumedescr, ResumeAtPositionDescr):
-        dont_change_position = True
-    else:
-        dont_change_position = False
 
     current_exc = blackholeinterp._prepare_resume_from_failure(
-        resumedescr.guard_opnum, dont_change_position, deadframe)
+        resumedescr.guard_opnum, deadframe)
 
     _run_forever(blackholeinterp, current_exc)
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -63,14 +63,18 @@
     if metainterp_sd.warmrunnerdesc is not None:    # for tests
         assert original_jitcell_token.generation > 0     # has been registered with memmgr
     wref = weakref.ref(original_jitcell_token)
+    clt = original_jitcell_token.compiled_loop_token
+    clt.loop_token_wref = wref
     for op in loop.operations:
         descr = op.getdescr()
+        # not sure what descr.index is about
         if isinstance(descr, ResumeDescr):
-            descr.wref_original_loop_token = wref   # stick it there
-            n = descr.index
-            if n >= 0:       # we also record the resumedescr number
-                original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
-        elif isinstance(descr, JitCellToken):
+            descr.rd_loop_token = clt   # stick it there
+            #n = descr.index
+            #if n >= 0:       # we also record the resumedescr number
+            #    original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
+        #    pass
+        if isinstance(descr, JitCellToken):
             # for a CALL_ASSEMBLER: record it as a potential jump.
             if descr is not original_jitcell_token:
                 original_jitcell_token.record_jump_to(descr)
@@ -129,8 +133,8 @@
     part.operations = [label] + h_ops + [end_label]
 
     try:
-        start_state = optimize_trace(metainterp_sd, part, enable_opts,
-                                     export_state=True)
+        start_state = optimize_trace(metainterp_sd, jitdriver_sd, part,
+                                     enable_opts, export_state=True)
     except InvalidLoop:
         return None
     target_token = part.operations[0].getdescr()
@@ -157,7 +161,7 @@
         jumpargs = part.operations[-1].getarglist()
 
         try:
-            optimize_trace(metainterp_sd, part, enable_opts,
+            optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
                            start_state=start_state, export_state=False)
         except InvalidLoop:
             return None
@@ -210,7 +214,8 @@
     orignial_label = label.clone()
     assert label.getopnum() == rop.LABEL
     try:
-        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
+        optimize_trace(metainterp_sd, jitdriver_sd, part,
+                       jitdriver_sd.warmstate.enable_opts,
                        start_state=start_state, export_state=False)
     except InvalidLoop:
         # Fall back on jumping to preamble
@@ -220,7 +225,7 @@
                           [ResOperation(rop.JUMP, inputargs[:],
                                         None, descr=loop_jitcell_token)]
         try:
-            optimize_trace(metainterp_sd, part,
+            optimize_trace(metainterp_sd, jitdriver_sd, part,
                            jitdriver_sd.warmstate.enable_opts,
                            inline_short_preamble=False, start_state=start_state,
                            export_state=False)
@@ -480,22 +485,31 @@
     return d
 
 class ResumeDescr(AbstractFailDescr):
-    pass
+    _attrs_ = ()
 
 class ResumeGuardDescr(ResumeDescr):
-    # this class also gets the following attributes stored by resume.py code
-    # XXX move all of unused stuff to guard_op, now that we can have
-    #     a separate class, so it does not survive that long
-    rd_snapshot = None
-    rd_frame_info_list = None
+    _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
+               'rd_frame_info_list', 'rd_pendingfields', 'status')
+    
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
     rd_consts = None
     rd_virtuals = None
+    rd_frame_info_list = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
 
     status = r_uint(0)
 
+    def copy_all_attributes_from(self, other):
+        assert isinstance(other, ResumeGuardDescr)
+        self.rd_count = other.rd_count
+        self.rd_consts = other.rd_consts
+        self.rd_frame_info_list = other.rd_frame_info_list
+        self.rd_pendingfields = other.rd_pendingfields
+        self.rd_virtuals = other.rd_virtuals
+        self.rd_numb = other.rd_numb
+        # we don't copy status
+
     ST_BUSY_FLAG    = 0x01     # if set, busy tracing from the guard
     ST_TYPE_MASK    = 0x06     # mask for the type (TY_xxx)
     ST_SHIFT        = 3        # in "status >> ST_SHIFT" is stored:
@@ -510,31 +524,12 @@
     def store_final_boxes(self, guard_op, boxes, metainterp_sd):
         guard_op.setfailargs(boxes)
         self.rd_count = len(boxes)
-        self.guard_opnum = guard_op.getopnum()
         #
         if metainterp_sd.warmrunnerdesc is not None:   # for tests
             jitcounter = metainterp_sd.warmrunnerdesc.jitcounter
             hash = jitcounter.fetch_next_hash()
             self.status = hash & self.ST_SHIFT_MASK
 
-    def make_a_counter_per_value(self, guard_value_op):
-        assert guard_value_op.getopnum() == rop.GUARD_VALUE
-        box = guard_value_op.getarg(0)
-        try:
-            i = guard_value_op.getfailargs().index(box)
-        except ValueError:
-            return     # xxx probably very rare
-        else:
-            if box.type == history.INT:
-                ty = self.TY_INT
-            elif box.type == history.REF:
-                ty = self.TY_REF
-            elif box.type == history.FLOAT:
-                ty = self.TY_FLOAT
-            else:
-                assert 0, box.type
-            self.status = ty | (r_uint(i) << self.ST_SHIFT)
-
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
             self.start_compiling()
@@ -633,32 +628,62 @@
                                self, inputargs, new_loop.operations,
                                new_loop.original_jitcell_token)
 
-    def copy_all_attributes_into(self, res):
-        # XXX a bit ugly to have to list them all here
-        res.rd_snapshot = self.rd_snapshot
-        res.rd_frame_info_list = self.rd_frame_info_list
-        res.rd_numb = self.rd_numb
-        res.rd_consts = self.rd_consts
-        res.rd_virtuals = self.rd_virtuals
-        res.rd_pendingfields = self.rd_pendingfields
-        res.rd_count = self.rd_count
+    def make_a_counter_per_value(self, guard_value_op):
+        assert guard_value_op.getopnum() == rop.GUARD_VALUE
+        box = guard_value_op.getarg(0)
+        try:
+            i = guard_value_op.getfailargs().index(box)
+        except ValueError:
+            return     # xxx probably very rare
+        else:
+            if box.type == history.INT:
+                ty = self.TY_INT
+            elif box.type == history.REF:
+                ty = self.TY_REF
+            elif box.type == history.FLOAT:
+                ty = self.TY_FLOAT
+            else:
+                assert 0, box.type
+            self.status = ty | (r_uint(i) << self.ST_SHIFT)
 
-    def _clone_if_mutable(self):
-        res = ResumeGuardDescr()
-        self.copy_all_attributes_into(res)
-        return res
+class ResumeGuardNonnullDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NONNULL
+
+class ResumeGuardIsnullDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_ISNULL
+
+class ResumeGuardClassDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_CLASS
+
+class ResumeGuardTrueDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_TRUE
+
+class ResumeGuardFalseDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_FALSE
+
+class ResumeGuardNonnullClassDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NONNULL_CLASS
+
+class ResumeGuardExceptionDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_EXCEPTION
+
+class ResumeGuardNoExceptionDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NO_EXCEPTION
+
+class ResumeGuardOverflowDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_OVERFLOW
+
+class ResumeGuardNoOverflowDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NO_OVERFLOW
+
+class ResumeGuardValueDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_VALUE
 
 class ResumeGuardNotInvalidated(ResumeGuardDescr):
-    def _clone_if_mutable(self):
-        res = ResumeGuardNotInvalidated()
-        self.copy_all_attributes_into(res)
-        return res
+    guard_opnum = rop.GUARD_NOT_INVALIDATED
 
 class ResumeAtPositionDescr(ResumeGuardDescr):
-    def _clone_if_mutable(self):
-        res = ResumeAtPositionDescr()
-        self.copy_all_attributes_into(res)
-        return res
+    guard_opnum = rop.GUARD_FUTURE_CONDITION
 
 class AllVirtuals:
     llopaque = True
@@ -679,8 +704,10 @@
 
 
 class ResumeGuardForcedDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NOT_FORCED
 
-    def __init__(self, metainterp_sd, jitdriver_sd):
+    def _init(self, metainterp_sd, jitdriver_sd):
+        # to please the annotator
         self.metainterp_sd = metainterp_sd
         self.jitdriver_sd = jitdriver_sd
 
@@ -741,12 +768,40 @@
         hidden_all_virtuals = obj.hide(metainterp_sd.cpu)
         metainterp_sd.cpu.set_savedata_ref(deadframe, hidden_all_virtuals)
 
-    def _clone_if_mutable(self):
-        res = ResumeGuardForcedDescr(self.metainterp_sd,
-                                     self.jitdriver_sd)
-        self.copy_all_attributes_into(res)
-        return res
-
+def invent_fail_descr_for_op(op, optimizer):
+    opnum = op.getopnum()
+    if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
+        resumedescr = ResumeGuardForcedDescr()
+        resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd)
+    elif opnum == rop.GUARD_NOT_INVALIDATED:
+        resumedescr = ResumeGuardNotInvalidated()
+    elif opnum == rop.GUARD_FUTURE_CONDITION:
+        resumedescr = ResumeAtPositionDescr()
+    elif opnum == rop.GUARD_VALUE:
+        resumedescr = ResumeGuardValueDescr()
+    elif opnum == rop.GUARD_NONNULL:
+        resumedescr = ResumeGuardNonnullDescr()
+    elif opnum == rop.GUARD_ISNULL:
+        resumedescr = ResumeGuardIsnullDescr()
+    elif opnum == rop.GUARD_NONNULL_CLASS:
+        resumedescr = ResumeGuardNonnullClassDescr()
+    elif opnum == rop.GUARD_CLASS:
+        resumedescr = ResumeGuardClassDescr()
+    elif opnum == rop.GUARD_TRUE:
+        resumedescr = ResumeGuardTrueDescr()
+    elif opnum == rop.GUARD_FALSE:
+        resumedescr = ResumeGuardFalseDescr()
+    elif opnum == rop.GUARD_EXCEPTION:
+        resumedescr = ResumeGuardExceptionDescr()
+    elif opnum == rop.GUARD_NO_EXCEPTION:
+        resumedescr = ResumeGuardNoExceptionDescr()
+    elif opnum == rop.GUARD_OVERFLOW:
+        resumedescr = ResumeGuardOverflowDescr()
+    elif opnum == rop.GUARD_NO_OVERFLOW:
+        resumedescr = ResumeGuardNoOverflowDescr()
+    else:
+        assert False
+    return resumedescr
 
 class ResumeFromInterpDescr(ResumeDescr):
     def __init__(self, original_greenkey):
@@ -786,13 +841,15 @@
 
     new_trace.operations = metainterp.history.operations[:]
     metainterp_sd = metainterp.staticdata
-    state = metainterp.jitdriver_sd.warmstate
+    jitdriver_sd = metainterp.jitdriver_sd
+    state = jitdriver_sd.warmstate
     if isinstance(resumekey, ResumeAtPositionDescr):
         inline_short_preamble = False
     else:
         inline_short_preamble = True
     try:
-        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts,
+        state = optimize_trace(metainterp_sd, jitdriver_sd, new_trace,
+                               state.enable_opts,
                                inline_short_preamble, export_state=True)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -134,14 +134,6 @@
     def repr_of_descr(self):
         return '%r' % (self,)
 
-    def _clone_if_mutable(self):
-        return self
-    def clone_if_mutable(self):
-        clone = self._clone_if_mutable()
-        if not we_are_translated():
-            assert clone.__class__ is self.__class__
-        return clone
-
     def hide(self, cpu):
         descr_ptr = cpu.ts.cast_instance_to_base_ref(self)
         return cpu.ts.cast_to_ref(descr_ptr)
@@ -159,6 +151,8 @@
     index = -1
     final_descr = False
 
+    _attrs_ = ('adr_jump_offset', 'rd_locs', 'rd_loop_token')
+
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         raise NotImplementedError
     def compile_and_attach(self, metainterp, new_loop):
diff --git a/rpython/jit/metainterp/inliner.py b/rpython/jit/metainterp/inliner.py
--- a/rpython/jit/metainterp/inliner.py
+++ b/rpython/jit/metainterp/inliner.py
@@ -1,5 +1,6 @@
 from rpython.jit.metainterp.history import Const
 from rpython.jit.metainterp.resume import Snapshot
+from rpython.jit.metainterp.resoperation import GuardResOp
 
 
 class Inliner(object):
@@ -26,21 +27,16 @@
                 newop.setfailargs([self.inline_arg(a) for a in args])
             else:
                 newop.setfailargs([])
+            assert isinstance(newop, GuardResOp)
+            newop.rd_snapshot = self.inline_snapshot(newop.rd_snapshot)
 
         if newop.result and not ignore_result:
             old_result = newop.result
             newop.result = newop.result.clonebox()
             self.argmap[old_result] = newop.result
 
-        self.inline_descr_inplace(newop.getdescr())
-
         return newop
 
-    def inline_descr_inplace(self, descr):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
-        if isinstance(descr, ResumeGuardDescr):
-            descr.rd_snapshot = self.inline_snapshot(descr.rd_snapshot)
-
     def inline_arg(self, arg):
         if arg is None:
             return None
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -47,7 +47,7 @@
 
     return optimizations, unroll
 
-def optimize_trace(metainterp_sd, loop, enable_opts,
+def optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts,
                    inline_short_preamble=True, start_state=None,
                    export_state=True):
     """Optimize loop.operations to remove internal overheadish operations.
@@ -59,11 +59,13 @@
                                                           loop.operations)
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
-            return optimize_unroll(metainterp_sd, loop, optimizations,
+            return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
+                                   optimizations,
                                    inline_short_preamble, start_state,
                                    export_state)
         else:
-            optimizer = Optimizer(metainterp_sd, loop, optimizations)
+            optimizer = Optimizer(metainterp_sd, jitdriver_sd, loop,
+                                  optimizations)
             optimizer.propagate_all_forward()
     finally:
         debug_stop("jit-optimize")
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -7,7 +7,8 @@
                                                      IntLowerBound, MININT,\
                                                      MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractResOp
+from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+     AbstractResOp, GuardResOp
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.tool.pairtype import extendabletype
 from rpython.rlib.debug import debug_print
@@ -504,8 +505,9 @@
 
 class Optimizer(Optimization):
 
-    def __init__(self, metainterp_sd, loop, optimizations=None):
+    def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations=None):
         self.metainterp_sd = metainterp_sd
+        self.jitdriver_sd = jitdriver_sd
         self.cpu = metainterp_sd.cpu
         self.loop = loop
         self.values = {}
@@ -748,13 +750,23 @@
     def replace_op(self, old_op_pos, new_op):
         old_op = self._newoperations[old_op_pos]
         assert old_op.is_guard()
+        old_descr = old_op.getdescr()
+        new_descr = new_op.getdescr()
+        new_descr.copy_all_attributes_from(old_descr)
         self._newoperations[old_op_pos] = new_op
 
     def store_final_boxes_in_guard(self, op, pendingfields):
         assert pendingfields is not None
-        descr = op.getdescr()
+        if op.getdescr() is not None:
+            descr = op.getdescr()
+            assert isinstance(descr, compile.ResumeAtPositionDescr)
+        else:
+            descr = compile.invent_fail_descr_for_op(op, self)
+            op.setdescr(descr)
         assert isinstance(descr, compile.ResumeGuardDescr)
-        modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo)
+        assert isinstance(op, GuardResOp)
+        modifier = resume.ResumeDataVirtualAdder(descr, op,
+                                                 self.resumedata_memo)
         try:
             newboxes = modifier.finish(self, pendingfields)
             if len(newboxes) > self.metainterp_sd.options.failargs_limit:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -312,15 +312,11 @@
                 if not previous_classbox.same_constant(expected_classbox):
                     r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
                     raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r)
+            descr = compile.ResumeGuardValueDescr()
             op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+                        args = [old_guard_op.getarg(0), op.getarg(1)],
+                        descr = descr)
             self.optimizer.replace_guard(op, value)
-            # hack hack hack.  Change the guard_opnum on
-            # new_guard_op.getdescr() so that when resuming,
-            # the operation is not skipped by pyjitpl.py.
-            descr = op.getdescr()
-            assert isinstance(descr, compile.ResumeGuardDescr)
-            descr.guard_opnum = rop.GUARD_VALUE
             descr.make_a_counter_per_value(op)
             # to be safe
             if isinstance(value, PtrOptValue):
@@ -364,15 +360,11 @@
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
+                descr = compile.ResumeGuardNonnullClassDescr()
                 op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
-                                         args = [old_guard_op.getarg(0), op.getarg(1)])
+                            args = [old_guard_op.getarg(0), op.getarg(1)],
+                            descr=descr)
                 self.optimizer.replace_guard(op, value)
-                # hack hack hack.  Change the guard_opnum on
-                # new_guard_op.getdescr() so that when resuming,
-                # the operation is not skipped by pyjitpl.py.
-                descr = op.getdescr()
-                assert isinstance(descr, compile.ResumeGuardDescr)
-                descr.guard_opnum = rop.GUARD_NONNULL_CLASS
         self.emit_operation(op)
         value.make_constant_class(self.optimizer, expectedclassbox)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -17,16 +17,16 @@
     b0 = BoxInt()
     b1 = BoxInt()
     opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu),
-                                None)
-    fdescr = ResumeGuardDescr()
-    op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr)
+                                None, None)
+    op = ResOperation(rop.GUARD_TRUE, ['dummy'], None)
     # setup rd data
     fi0 = resume.FrameInfo(None, "code0", 11)
-    fdescr.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33)
     snapshot0 = resume.Snapshot(None, [b0])
-    fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1])
+    op.rd_snapshot = resume.Snapshot(snapshot0, [b1])
+    op.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33)
     #
     opt.store_final_boxes_in_guard(op, [])
+    fdescr = op.getdescr()
     if op.getfailargs() == [b0, b1]:
         assert list(fdescr.rd_numb.nums)      == [tag(1, TAGBOX)]
         assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)]
@@ -97,7 +97,8 @@
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
-        loop = self.parse(ops)
+                
+        loop = self.parse(ops, postprocess=self.postprocess)
         token = JitCellToken()
         loop.operations = [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \
                           loop.operations
@@ -108,42 +109,6 @@
         print '\n'.join([str(o) for o in loop.operations])
         self.assert_equal(loop, expected)
 
-    def setup_method(self, meth=None):
-        class FailDescr(compile.ResumeGuardDescr):
-            oparse = None
-            def _oparser_uses_descr_of_guard(self, oparse, fail_args):
-                # typically called 3 times: once when parsing 'ops',
-                # once when parsing 'preamble', once when parsing 'expected'.
-                self.oparse = oparse
-                self.rd_frame_info_list, self.rd_snapshot = snapshot(fail_args)
-            def _clone_if_mutable(self):
-                assert self is fdescr
-                return fdescr2
-            def __repr__(self):
-                if self is fdescr:
-                    return 'fdescr'
-                if self is fdescr2:
-                    return 'fdescr2'
-                return compile.ResumeGuardDescr.__repr__(self)
-        #
-        def snapshot(fail_args, got=[]):
-            if not got:    # only the first time, i.e. when parsing 'ops'
-                rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-                rd_snapshot = resume.Snapshot(None, fail_args)
-                got.append(rd_frame_info_list)
-                got.append(rd_snapshot)
-            return got
-        #
-        fdescr = instantiate(FailDescr)
-        self.namespace['fdescr'] = fdescr
-        fdescr2 = instantiate(FailDescr)
-        self.namespace['fdescr2'] = fdescr2
-
-    def teardown_method(self, meth):
-        self.namespace.pop('fdescr', None)
-        self.namespace.pop('fdescr2', None)
-
-
 
 class BaseTestOptimizeBasic(BaseTestBasic):
 
@@ -2065,14 +2030,14 @@
     def test_merge_guard_nonnull_guard_class(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         expected = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull_class(p1, ConstClass(node_vtable), descr=fdescr) [i0]
+        guard_nonnull_class(p1, ConstClass(node_vtable)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3, p2)
         """
@@ -2082,14 +2047,14 @@
     def test_merge_guard_nonnull_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_value(p1, ConstPtr(myptr)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         expected = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3, p2)
         """
@@ -2099,7 +2064,7 @@
     def test_merge_guard_nonnull_guard_class_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i2]
         i4 = int_sub(i3, 1)
@@ -2108,7 +2073,7 @@
         """
         expected = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         i4 = int_sub(i3, 1)
         jump(p2, i0, i1, i4, p2)
@@ -2527,7 +2492,7 @@
         reader = ResumeDataFakeReader(fdescr, fail_args,
                                       MyMetaInterp(self.cpu))
         boxes = reader.consume_boxes()
-        self._verify_fail_args(boxes, fdescr.oparse, expectedtext)
+        self._verify_fail_args(boxes, self.oparse, expectedtext)
 
     def test_expand_fail_1(self):
         ops = """
@@ -2538,12 +2503,12 @@
         i4 = getfield_gc(p1, descr=valuedescr)
         #
         i2 = int_add(10, 5)
-        guard_true(i1, descr=fdescr) [i2, i4]
+        guard_true(i1) [i2, i4]
         jump(i1, i4)
         """
         expected = """
         [i1, i3]
-        guard_true(i1, descr=fdescr) [i3]
+        guard_true(i1) [i3]
         jump(1, i3)
         """
         self.optimize_loop(ops, expected)
@@ -2555,12 +2520,12 @@
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i2, descr=valuedescr)
         setfield_gc(p1, p1, descr=nextdescr)
-        guard_true(i1, descr=fdescr) [p1]
+        guard_true(i1) [p1]
         jump(i1, i2)
         """
         expected = """
         [i1, i2]
-        guard_true(i1, descr=fdescr) [i2]
+        guard_true(i1) [i2]
         jump(1, i2)
         """
         self.optimize_loop(ops, expected)
@@ -2577,12 +2542,12 @@
         setfield_gc(p1, p2, descr=nextdescr)
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(p2, p3, descr=nextdescr)
-        guard_true(i1, descr=fdescr) [i3, p1]
+        guard_true(i1) [i3, p1]
         jump(i2, i1, i3, p3)
         """
         expected = """
         [i1, i2, i3, p3]
-        guard_true(i1, descr=fdescr) [i3, i2, p3]
+        guard_true(i1) [i3, i2, p3]
         jump(i2, 1, i3, p3)
         """
         self.optimize_loop(ops, expected)
@@ -2594,7 +2559,6 @@
     def test_expand_fail_4(self):
         for arg in ['p1', 'i2,p1', 'p1,p2', 'p2,p1',
                     'i2,p1,p2', 'i2,p2,p1']:
-            self.setup_method() # humpf
             ops = """
             [i1, i2, i3]
             p1 = new_with_vtable(ConstClass(node_vtable))
@@ -2604,12 +2568,12 @@
             setfield_gc(p1, i2, descr=valuedescr)
             setfield_gc(p1, p2, descr=nextdescr)
             setfield_gc(p2, i2, descr=valuedescr)
-            guard_true(i1, descr=fdescr) [i4, i3, %s]
+            guard_true(i1) [i4, i3, %s]
             jump(i1, i2, i3)
             """
             expected = """
             [i1, i2, i3]
-            guard_true(i1, descr=fdescr) [i3, i2]
+            guard_true(i1) [i3, i2]
             jump(1, i2, i3)
             """
             self.optimize_loop(ops % arg, expected)
@@ -2627,12 +2591,12 @@
         setfield_gc(p1, p2, descr=nextdescr)
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(p2, p1, descr=nextdescr)      # a cycle
-        guard_true(i1, descr=fdescr) [i3, i4, p1, p2]
+        guard_true(i1) [i3, i4, p1, p2]
         jump(i2, i1, i3, i4)
         """
         expected = """
         [i1, i2, i3, i4]
-        guard_true(i1, descr=fdescr) [i3, i4, i2]
+        guard_true(i1) [i3, i4, i2]
         jump(i2, 1, i3, i4)
         """
         self.optimize_loop(ops, expected)
@@ -2644,14 +2608,14 @@
     def test_expand_fail_6(self):
         ops = """
         [p0, i0, i1]
-        guard_true(i0, descr=fdescr) [p0]
+        guard_true(i0) [p0]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
         jump(p1, i1, i1)
         """
         expected = """
         [i1b, i0, i1]
-        guard_true(i0, descr=fdescr) [i1b]
+        guard_true(i0) [i1b]
         jump(i1, i1, i1)
         """
         py.test.skip("XXX")
@@ -2667,13 +2631,13 @@
         p1 = new_array(3, descr=arraydescr)
         setarrayitem_gc(p1, 1, i1, descr=arraydescr)
         setarrayitem_gc(p1, 0, 25, descr=arraydescr)
-        guard_true(i1, descr=fdescr) [p1]
+        guard_true(i1) [p1]
         i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
         jump(i2)
         """
         expected = """
         [i1]
-        guard_true(i1, descr=fdescr) [i1]
+        guard_true(i1) [i1]
         jump(1)
         """
         self.optimize_loop(ops, expected)
@@ -2687,14 +2651,14 @@
         p2 = new(descr=ssize)
         setfield_gc(p2, i1, descr=adescr)
         setfield_gc(p2, p1, descr=bdescr)
-        guard_true(i1, descr=fdescr) [p2]
+        guard_true(i1) [p2]
         i3 = getfield_gc(p2, descr=adescr)
         p3 = getfield_gc(p2, descr=bdescr)
         jump(i3, p3)
         """
         expected = """
         [i1, p1]
-        guard_true(i1, descr=fdescr) [i1, p1]
+        guard_true(i1) [i1, p1]
         jump(1, p1)
         """
         self.optimize_loop(ops, expected)
@@ -2711,7 +2675,7 @@
         setfield_gc(p5s, i2, descr=adescr)
         setfield_gc(p5s, p7v, descr=bdescr)
         setarrayitem_gc(p1a, 1, p5s, descr=arraydescr2)
-        guard_true(i1, descr=fdescr) [p1a]
+        guard_true(i1) [p1a]
         p2s = new(descr=ssize)
         p3v = new_with_vtable(ConstClass(node_vtable))
         p4a = new_array(2, descr=arraydescr2)
@@ -2723,7 +2687,7 @@
         """
         expected = """
         [i1, ia, iv, pnull, i2]
-        guard_true(i1, descr=fdescr) [ia, iv, i2]
+        guard_true(i1) [ia, iv, i2]
         jump(1, 1, i2, NULL, i2)
         """
         py.test.skip("XXX")
@@ -2749,14 +2713,14 @@
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        guard_true(i3, descr=fdescr) []
+        guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         jump(p1, i2, i4)
         """
         expected = """
         [p1, i2, i3]
-        guard_true(i3, descr=fdescr) [p1, i2]
+        guard_true(i3) [p1, i2]
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         jump(p1, i2, i4)
@@ -2774,14 +2738,14 @@
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(ConstPtr(myptr), p2, descr=nextdescr)
-        guard_true(i3, descr=fdescr) []
+        guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr)
         jump(i2, i4)
         """
         expected = """
         [i2, i3]
-        guard_true(i3, descr=fdescr) [i2]
+        guard_true(i3) [i2]
         i4 = int_neg(i2)
         setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr)
         jump(i2, i4)
@@ -3096,7 +3060,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=nextdescr)
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr) [p2, p1]
+        guard_not_forced() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=nextdescr)
         jump(p0, i1)
@@ -3111,7 +3075,7 @@
         setfield_gc(p0, p2, descr=nextdescr)
         #
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr) [p2, i1]
+        guard_not_forced() [p2, i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
         p1 = new_with_vtable(ConstClass(node_vtable))
@@ -3142,7 +3106,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=refdescr)
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p2, p1]
+        guard_no_exception() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=refdescr)
         jump(p0, i1)
@@ -3151,7 +3115,7 @@
         [p0, i1]
         p3 = force_token()
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p3, i1, p0]
+        guard_no_exception() [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         jump(p0, i1)
         """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -45,7 +45,7 @@
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
-        loop = self.parse(ops)
+        loop = self.parse(ops, postprocess=self.postprocess)
         if expected != "crash!":
             expected = self.parse(expected)
         if expected_preamble:
@@ -96,41 +96,6 @@
 
 
 class OptimizeOptTest(BaseTestWithUnroll):
-    def setup_method(self, meth=None):
-        class FailDescr(compile.ResumeGuardDescr):
-            oparse = None
-            def _oparser_uses_descr_of_guard(self, oparse, fail_args):
-                # typically called 3 times: once when parsing 'ops',
-                # once when parsing 'preamble', once when parsing 'expected'.
-                self.oparse = oparse
-                self.rd_frame_info_list, self.rd_snapshot = snapshot(fail_args)
-            def _clone_if_mutable(self):
-                assert self is fdescr
-                return fdescr2
-            def __repr__(self):
-                if self is fdescr:
-                    return 'fdescr'
-                if self is fdescr2:
-                    return 'fdescr2'
-                return compile.ResumeGuardDescr.__repr__(self)
-        #
-        def snapshot(fail_args, got=[]):
-            if not got:    # only the first time, i.e. when parsing 'ops'
-                rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-                rd_snapshot = resume.Snapshot(None, fail_args)
-                got.append(rd_frame_info_list)
-                got.append(rd_snapshot)
-            return got
-        #
-        fdescr = instantiate(FailDescr)
-        self.namespace['fdescr'] = fdescr
-        fdescr2 = instantiate(FailDescr)
-        self.namespace['fdescr2'] = fdescr2
-
-    def teardown_method(self, meth):
-        self.namespace.pop('fdescr', None)
-        self.namespace.pop('fdescr2', None)
-
     def test_simple(self):
         ops = """
         []
@@ -3004,20 +2969,20 @@
     def test_merge_guard_nonnull_guard_class(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         preamble = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull_class(p1, ConstClass(node_vtable), descr=fdescr) [i0]
+        guard_nonnull_class(p1, ConstClass(node_vtable)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3)
         """
         expected = """
         [p2, i0, i1, i2]
-        guard_nonnull_class(p2, ConstClass(node_vtable), descr=fdescr2) [i0]
+        guard_nonnull_class(p2, ConstClass(node_vtable)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3)
         """
@@ -3027,20 +2992,20 @@
     def test_merge_guard_nonnull_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_value(p1, ConstPtr(myptr)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         preamble = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3)
         """
         expected = """
         [p2, i0, i1, i2]
-        guard_value(p2, ConstPtr(myptr), descr=fdescr2) [i0]
+        guard_value(p2, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         jump(ConstPtr(myptr), i0, i1, i3)
         """
@@ -3050,7 +3015,7 @@
     def test_merge_guard_nonnull_guard_class_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i2]
         i4 = int_sub(i3, 1)
@@ -3059,14 +3024,14 @@
         """
         preamble = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         i4 = int_sub(i3, 1)
         jump(p2, i0, i1, i4)
         """
         expected = """
         [p2, i0, i1, i2]
-        guard_value(p2, ConstPtr(myptr), descr=fdescr2) [i0]
+        guard_value(p2, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         i4 = int_sub(i3, 1)
         jump(ConstPtr(myptr), i0, i1, i4)
@@ -3860,7 +3825,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=nextdescr)
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr) [p2, p1]
+        guard_not_forced() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=nextdescr)
         jump(p0, i1)
@@ -3875,7 +3840,7 @@
         setfield_gc(p0, p2, descr=nextdescr)
         #
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr2) [p2, i1]
+        guard_not_forced() [p2, i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
         p1 = new_with_vtable(ConstClass(node_vtable))
@@ -3906,7 +3871,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=refdescr)
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p2, p1]
+        guard_no_exception() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=refdescr)
         escape()
@@ -3916,7 +3881,7 @@
         [p0, i1]
         p3 = force_token()
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p3, i1, p0]
+        guard_no_exception() [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         escape()
         jump(p0, i1)
@@ -3925,7 +3890,7 @@
         [p0, i1]
         p3 = force_token()
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr2) [p3, i1, p0]
+        guard_no_exception() [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         escape()
         jump(p0, i1)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -8,12 +8,13 @@
 from rpython.jit.backend.llgraph import runner
 from rpython.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr,
                                          Const, TreeLoop, AbstractDescr,
-                                         JitCellToken, TargetToken)
+                                         JitCellToken, TargetToken,
+    BasicFinalDescr)
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.codewriter.heaptracker import register_known_gctype, adr2int
-from rpython.jit.tool.oparser import parse, pure_parse
+from rpython.jit.tool.oparser import OpParser, pure_parse
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
 from rpython.jit.metainterp.jitprof import EmptyProfiler
@@ -348,42 +349,35 @@
     def store_final_boxes(self, op, boxes, metainterp_sd):
         op.setfailargs(boxes)
     def __eq__(self, other):
-        return type(self) is type(other)      # xxx obscure
-    def clone_if_mutable(self):
-        res = Storage(self.metainterp_sd, self.original_greenkey)
-        self.copy_all_attributes_into(res)
-        return res
+        return True # screw this
+        #return type(self) is type(other)      # xxx obscure
 
 def _sortboxes(boxes):
     _kind2count = {history.INT: 1, history.REF: 2, history.FLOAT: 3}
     return sorted(boxes, key=lambda box: _kind2count[box.type])
 
+final_descr = BasicFinalDescr()
+
 class BaseTest(object):
 
-    def parse(self, s, boxkinds=None, want_fail_descr=True):
-        if want_fail_descr:
-            invent_fail_descr = self.invent_fail_descr
-        else:
-            invent_fail_descr = lambda *args: None
-        return parse(s, self.cpu, self.namespace,
-                     type_system=self.type_system,
-                     boxkinds=boxkinds,
-                     invent_fail_descr=invent_fail_descr)
+    def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None):
+        self.oparse = OpParser(s, self.cpu, self.namespace, 'lltype',
+                               boxkinds,
+                               None, False, postprocess)
+        return self.oparse.parse()
+
+    def postprocess(self, op):
+        if op.is_guard():
+            op.rd_snapshot = resume.Snapshot(None, op.getfailargs())
+            op.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
 
     def add_guard_future_condition(self, res):
         # invent a GUARD_FUTURE_CONDITION to not have to change all tests
         if res.operations[-1].getopnum() == rop.JUMP:
-            guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None, descr=self.invent_fail_descr(None, -1, []))
+            guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None)
+            guard.rd_snapshot = resume.Snapshot(None, [])
             res.operations.insert(-1, guard)
 
-    def invent_fail_descr(self, model, opnum, fail_args):
-        if fail_args is None:
-            return None
-        descr = Storage()
-        descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-        descr.rd_snapshot = resume.Snapshot(None, _sortboxes(fail_args))
-        return descr
-
     def assert_equal(self, optimized, expected, text_right=None):
         from rpython.jit.metainterp.optimizeopt.util import equaloplists
         assert len(optimized.inputargs) == len(expected.inputargs)
@@ -410,7 +404,8 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        return optimize_trace(metainterp_sd, loop, self.enable_opts,
+        return optimize_trace(metainterp_sd, None, loop,
+                              self.enable_opts,
                               start_state=start_state,
                               export_state=export_state)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -15,6 +15,7 @@
 from rpython.jit.metainterp.history import TreeLoop, JitCellToken
 from rpython.jit.metainterp.optimizeopt.test.test_optimizeopt import FakeMetaInterpStaticData
 from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.metainterp import resume
 
 class BaseTestGenerateGuards(BaseTest):
 
@@ -798,14 +799,15 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        optimize_trace(metainterp_sd, bridge, self.enable_opts)
+        optimize_trace(metainterp_sd, None, bridge, self.enable_opts)
 
         
     def optimize_bridge(self, loops, bridge, expected, expected_target='Loop', **boxvalues):
         if isinstance(loops, str):
             loops = (loops, )
-        loops = [self.parse(loop) for loop in loops]
-        bridge = self.parse(bridge)
+        loops = [self.parse(loop, postprocess=self.postprocess)
+                 for loop in loops]
+        bridge = self.parse(bridge, postprocess=self.postprocess)
         self.add_guard_future_condition(bridge)
         for loop in loops:
             loop.preamble = self.unroll_and_optimize(loop)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
@@ -13,7 +13,7 @@
 
         def optimize_loop(self, ops, expected, expected_preamble=None,
                           call_pure_results=None, expected_short=None):
-            loop = self.parse(ops)
+            loop = self.parse(ops, postprocess=self.postprocess)
             if expected != "crash!":
                 expected = self.parse(expected)
             if expected_preamble:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,6 +1,5 @@
 import sys
 
-from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
 from rpython.jit.metainterp.inliner import Inliner
 from rpython.jit.metainterp.optimize import InvalidLoop
@@ -8,17 +7,18 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
         ShortBoxes, BadVirtualState, VirtualStatesCantMatch)
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp
 from rpython.jit.metainterp.resume import Snapshot
+from rpython.jit.metainterp import compile
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
 
 
 # FIXME: Introduce some VirtualOptimizer super class instead
 
-def optimize_unroll(metainterp_sd, loop, optimizations,
+def optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble=True, start_state=None,
                     export_state=True):
-    opt = UnrollOptimizer(metainterp_sd, loop, optimizations)
+    opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, loop, optimizations)
     opt.inline_short_preamble = inline_short_preamble
     return opt.propagate_all_forward(start_state, export_state)
 
@@ -52,8 +52,9 @@
 
     inline_short_preamble = True
 
-    def __init__(self, metainterp_sd, loop, optimizations):
-        self.optimizer = UnrollableOptimizer(metainterp_sd, loop, optimizations)
+    def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations):
+        self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
+                                             loop, optimizations)
         self.boxes_created_this_iteration = None
 
     def get_virtual_state(self, args):
@@ -562,8 +563,11 @@
 
             for guard in extra_guards:
                 if guard.is_guard():
-                    descr = patchguardop.getdescr().clone_if_mutable()
-                    guard.setdescr(descr)
+                    assert isinstance(patchguardop, GuardResOp)
+                    assert isinstance(guard, GuardResOp)
+                    guard.rd_snapshot = patchguardop.rd_snapshot
+                    guard.rd_frame_info_list = patchguardop.rd_frame_info_list
+                    guard.setdescr(compile.ResumeAtPositionDescr())
                 self.optimizer.send_extra_operation(guard)
 
             try:
@@ -592,8 +596,11 @@
             if newop.is_guard():
                 if not patchguardop:
                     raise InvalidLoop("would like to have short preamble, but it has a guard and there's no guard_future_condition")
-                descr = patchguardop.getdescr().clone_if_mutable()
-                newop.setdescr(descr)
+                assert isinstance(newop, GuardResOp)
+                assert isinstance(patchguardop, GuardResOp)
+                newop.rd_snapshot = patchguardop.rd_snapshot
+                newop.rd_frame_info_list = patchguardop.rd_frame_info_list
+                newop.setdescr(compile.ResumeAtPositionDescr())
             self.optimizer.send_extra_operation(newop)
             if shop.result in assumed_classes:
                 classbox = self.getvalue(newop.result).get_constant_class(self.optimizer.cpu)
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -164,8 +164,6 @@
                 assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
-        if op1.getopnum() not in (rop.JUMP, rop.LABEL):      # xxx obscure
-            assert op1.getdescr() == op2.getdescr()
         if op1.getfailargs() or op2.getfailargs():
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.jitprof import EmptyProfiler
 from rpython.jit.metainterp.logger import Logger
 from rpython.jit.metainterp.optimizeopt.util import args_dict
-from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.resoperation import rop, GuardResOp
 from rpython.rlib import nonconst, rstack
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.rlib.debug import have_debug_prints, make_sure_not_resized
@@ -1874,23 +1874,15 @@
             moreargs = [box] + extraargs
         else:
             moreargs = list(extraargs)
-        if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
-            resumedescr = compile.ResumeGuardForcedDescr(self.staticdata,
-                                                         self.jitdriver_sd)
-        elif opnum == rop.GUARD_NOT_INVALIDATED:
-            resumedescr = compile.ResumeGuardNotInvalidated()
-        elif opnum == rop.GUARD_FUTURE_CONDITION:
-            resumedescr = compile.ResumeAtPositionDescr()
-        else:
-            resumedescr = compile.ResumeGuardDescr()
-        guard_op = self.history.record(opnum, moreargs, None, descr=resumedescr)
-        self.capture_resumedata(resumedescr, resumepc)
+        guard_op = self.history.record(opnum, moreargs, None)
+        assert isinstance(guard_op, GuardResOp)
+        self.capture_resumedata(guard_op, resumepc)
         self.staticdata.profiler.count_ops(opnum, Counters.GUARDS)
         # count
         self.attach_debug_info(guard_op)
         return guard_op
 
-    def capture_resumedata(self, resumedescr, resumepc=-1):
+    def capture_resumedata(self, guard_op, resumepc=-1):
         virtualizable_boxes = None
         if (self.jitdriver_sd.virtualizable_info is not None or
             self.jitdriver_sd.greenfield_info is not None):
@@ -1902,7 +1894,7 @@
             if resumepc >= 0:
                 frame.pc = resumepc
         resume.capture_resumedata(self.framestack, virtualizable_boxes,
-                                  self.virtualref_boxes, resumedescr)
+                                  self.virtualref_boxes, guard_op)
         if self.framestack:
             self.framestack[-1].pc = saved_pc
 
@@ -2147,7 +2139,9 @@
         assert isinstance(key, compile.ResumeGuardDescr)
         # store the resumekey.wref_original_loop_token() on 'self' to make
         # sure that it stays alive as long as this MetaInterp
-        self.resumekey_original_loop_token = key.wref_original_loop_token()
+        self.resumekey_original_loop_token = key.rd_loop_token.loop_token_wref()
+        if self.resumekey_original_loop_token is None:
+            raise compile.giveup() # should be rare
         self.staticdata.try_to_free_some_loops()
         self.initialize_state_from_guard_failure(key, deadframe)
         try:
@@ -2163,13 +2157,8 @@
         self.seen_loop_header_for_jdindex = -1
         if isinstance(key, compile.ResumeAtPositionDescr):
             self.seen_loop_header_for_jdindex = self.jitdriver_sd.index
-            dont_change_position = True
-        else:
-            dont_change_position = False
         try:
-            self.prepare_resume_from_failure(key.guard_opnum,
-                                             dont_change_position,
-                                             deadframe)
+            self.prepare_resume_from_failure(key.guard_opnum, deadframe)
             if self.resumekey_original_loop_token is None:   # very rare case
                 raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
             self.interpret()
@@ -2312,12 +2301,12 @@
             else: assert 0
         self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args)
 
-    def prepare_resume_from_failure(self, opnum, dont_change_position,
-                                    deadframe):
+    def prepare_resume_from_failure(self, opnum, deadframe):
         frame = self.framestack[-1]
-        if opnum == rop.GUARD_TRUE:     # a goto_if_not that jumps only now
-            if not dont_change_position:
-                frame.pc = frame.jitcode.follow_jump(frame.pc)
+        if opnum == rop.GUARD_FUTURE_CONDITION:
+            pass
+        elif opnum == rop.GUARD_TRUE:     # a goto_if_not that jumps only now
+            frame.pc = frame.jitcode.follow_jump(frame.pc)
         elif opnum == rop.GUARD_FALSE:     # a goto_if_not that stops jumping;
             pass                  # or a switch that was in its "default" case
         elif opnum == rop.GUARD_VALUE or opnum == rop.GUARD_CLASS:
@@ -2341,12 +2330,11 @@
             pass # XXX we want to do something special in resume descr,
                  # but not now
         elif opnum == rop.GUARD_NO_OVERFLOW:   # an overflow now detected
-            if not dont_change_position:
-                self.execute_raised(OverflowError(), constant=True)
-                try:
-                    self.finishframe_exception()
-                except ChangeFrame:
-                    pass
+            self.execute_raised(OverflowError(), constant=True)
+            try:
+                self.finishframe_exception()
+            except ChangeFrame:
+                pass
         elif opnum == rop.GUARD_OVERFLOW:      # no longer overflowing
             self.clear_exception()
         else:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -91,8 +91,6 @@
     def clone(self):
         args = self.getarglist()
         descr = self.getdescr()
-        if descr is not None:
-            descr = descr.clone_if_mutable()
         op = ResOperation(self.getopnum(), args[:], self.result, descr)
         if not we_are_translated():
             op.name = self.name
@@ -217,6 +215,9 @@
 
     _fail_args = None
 
+    rd_snapshot = None
+    rd_frame_info_list = None
+
     def getfailargs(self):
         return self._fail_args
 
@@ -225,12 +226,18 @@
 
     def copy_and_change(self, opnum, args=None, result=None, descr=None):
         newop = AbstractResOp.copy_and_change(self, opnum, args, result, descr)
+        assert isinstance(newop, GuardResOp)
         newop.setfailargs(self.getfailargs())
+        newop.rd_snapshot = self.rd_snapshot
+        newop.rd_frame_info_list = self.rd_frame_info_list
         return newop
 
     def clone(self):
         newop = AbstractResOp.clone(self)
+        assert isinstance(newop, GuardResOp)
         newop.setfailargs(self.getfailargs())
+        newop.rd_snapshot = self.rd_snapshot
+        newop.rd_frame_info_list = self.rd_frame_info_list
         return newop
 
 # ============
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -55,7 +55,7 @@
                                          back.get_list_of_active_boxes(True))
 
 def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes,
-                       storage):
+                       snapshot_storage):
     n = len(framestack) - 1
     if virtualizable_boxes is not None:
         boxes = virtualref_boxes + virtualizable_boxes
@@ -66,14 +66,14 @@
         _ensure_parent_resumedata(framestack, n)
         frame_info_list = FrameInfo(top.parent_resumedata_frame_info_list,
                                     top.jitcode, top.pc)
-        storage.rd_frame_info_list = frame_info_list
+        snapshot_storage.rd_frame_info_list = frame_info_list
         snapshot = Snapshot(top.parent_resumedata_snapshot,
                             top.get_list_of_active_boxes(False))
         snapshot = Snapshot(snapshot, boxes)
-        storage.rd_snapshot = snapshot
+        snapshot_storage.rd_snapshot = snapshot
     else:
-        storage.rd_frame_info_list = None
-        storage.rd_snapshot = Snapshot(None, boxes)
+        snapshot_storage.rd_frame_info_list = None
+        snapshot_storage.rd_snapshot = Snapshot(None, boxes)
 
 #
 # The following is equivalent to the RPython-level declaration:
@@ -270,8 +270,9 @@
 
 class ResumeDataVirtualAdder(VirtualVisitor):
 
-    def __init__(self, storage, memo):
+    def __init__(self, storage, snapshot_storage, memo):
         self.storage = storage
+        self.snapshot_storage = snapshot_storage
         self.memo = memo
 
     def make_virtual_info(self, value, fieldnums):
@@ -357,13 +358,14 @@
         storage = self.storage
         # make sure that nobody attached resume data to this guard yet
         assert not storage.rd_numb
-        snapshot = storage.rd_snapshot
+        snapshot = self.snapshot_storage.rd_snapshot
         assert snapshot is not None # is that true?
         numb, liveboxes_from_env, v = self.memo.number(optimizer, snapshot)
         self.liveboxes_from_env = liveboxes_from_env
         self.liveboxes = {}
         storage.rd_numb = numb
-        storage.rd_snapshot = None
+        self.snapshot_storage.rd_snapshot = None
+        storage.rd_frame_info_list = self.snapshot_storage.rd_frame_info_list
 
         # collect liveboxes and virtuals
         n = len(liveboxes_from_env) - v
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -2764,7 +2764,8 @@
             return i
         #
         seen = []
-        def my_optimize_trace(metainterp_sd, loop, enable_opts, *args, **kwds):
+        def my_optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts,
+                              *args, **kwds):
             seen.append('unroll' in enable_opts)
             raise InvalidLoop
         old_optimize_trace = optimizeopt.optimize_trace
diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -10,6 +10,9 @@
 from rpython.jit.metainterp.optimizeopt import ALL_OPTS_DICT
 
 class FakeCPU(object):
+    class Storage:
+        pass
+    
     class tracker:
         pass
 
@@ -18,6 +21,7 @@
         self.seen = []
     def compile_loop(self, inputargs, operations, token, log=True, name='',
                      logger=None):
+        token.compiled_loop_token = self.Storage()
         self.seen.append((inputargs, operations, token))
 
 class FakeLogger(object):
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -123,7 +123,7 @@
     class FakeVirtualValue(AbstractVirtualValue):
         def visitor_dispatch_virtual_type(self, *args):
             return FakeVInfo()
-    modifier = ResumeDataVirtualAdder(None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None)
     v1 = FakeVirtualValue(None, None)
     vinfo1 = modifier.make_virtual_info(v1, [1, 2, 4])
     vinfo2 = modifier.make_virtual_info(v1, [1, 2, 4])
@@ -618,7 +618,7 @@
           FakeFrame("code2", 9, c3, b2)]
     capture_resumedata(fs, None, [], storage)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     metainterp = MyMetaInterp()
 
@@ -1022,7 +1022,7 @@
 def test_register_virtual_fields():
     b1, b2 = BoxInt(), BoxInt()
     vbox = BoxPtr()
-    modifier = ResumeDataVirtualAdder(None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1031,7 +1031,7 @@
                                   b2: UNASSIGNED}
     assert modifier.vfieldboxes == {vbox: [b1, b2]}
 
-    modifier = ResumeDataVirtualAdder(None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None)
     modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1061,7 +1061,7 @@
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())    
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     assert storage.rd_snapshot is None
     cpu = MyCPU([])
@@ -1075,14 +1075,14 @@
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.finish(FakeOptimizer({}))
     assert len(memo.consts) == 2
     assert storage.rd_consts is memo.consts
 
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)]
     storage2 = make_storage(b1s, b2s, b3s)
-    modifier2 = ResumeDataVirtualAdder(storage2, memo)
+    modifier2 = ResumeDataVirtualAdder(storage2, storage2, memo)
     modifier2.finish(FakeOptimizer({}))
     assert len(memo.consts) == 3    
     assert storage2.rd_consts is memo.consts
@@ -1137,7 +1137,7 @@
     b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     b1_2 = BoxInt()
     class FakeValue(object):
 
@@ -1169,7 +1169,7 @@
     b1s = ConstInt(111)
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())        
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     b2t, b3t = [BoxPtr(demo55o), BoxInt(33)]
     newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t)
@@ -1190,7 +1190,7 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1273,7 +1273,7 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1321,7 +1321,7 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1364,7 +1364,7 @@
     b2s, b4s = [BoxPtr(), BoxPtr()]
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1403,7 +1403,7 @@
     class Storage(object):
         pass
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, None)
+    modifier = ResumeDataVirtualAdder(storage, storage, None)
     modifier._add_pending_fields([])
     assert not storage.rd_pendingfields
     #
@@ -1411,7 +1411,7 @@
         pass
     field_a = FieldDescr()
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, None)
+    modifier = ResumeDataVirtualAdder(storage, storage, None)
     modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042),
                                    61: rffi.cast(rffi.SHORT, 1061)}
     modifier._add_pending_fields([(field_a, 42, 61, -1)])
@@ -1425,7 +1425,7 @@
     #
     array_a = FieldDescr()
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, None)
+    modifier = ResumeDataVirtualAdder(storage, storage, None)
     modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042),
                                    61: rffi.cast(rffi.SHORT, 1061),
                                    62: rffi.cast(rffi.SHORT, 1062),
@@ -1506,7 +1506,7 @@
     metainterp_sd = FakeMetaInterpStaticData()
     metainterp_sd.options = options
     memo = ResumeDataLoopMemo(metainterp_sd)
-    modifier = ResumeDataVirtualAdder(None, memo)
+    modifier = ResumeDataVirtualAdder(None, None, memo)
 
     for i in range(5):
         assert not modifier._invalidation_needed(5, i)
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -61,9 +61,10 @@
 
     def __init__(self, input, cpu, namespace, type_system, boxkinds,
                  invent_fail_descr=default_fail_descr,
-                 nonstrict=False):
+                 nonstrict=False, postproces=None):
         self.input = input
         self.vars = {}
+        self._postproces = postproces
         self.cpu = cpu
         self._consts = namespace
         self.type_system = type_system
@@ -237,8 +238,6 @@
                     fail_args.append(fail_arg)
             if descr is None and self.invent_fail_descr:
                 descr = self.invent_fail_descr(self.model, opnum, fail_args)
-            if hasattr(descr, '_oparser_uses_descr_of_guard'):
-                descr._oparser_uses_descr_of_guard(self, fail_args)
         else:
             fail_args = None
             if opnum == rop.FINISH:
@@ -250,7 +249,7 @@
 
         return opnum, args, descr, fail_args
 
-    def create_op(self, opnum, args, result, descr):
+    def create_op(self, opnum, args, result, descr, fail_args):
         if opnum == ESCAPE_OP.OPNUM:
             op = ESCAPE_OP(result)
             op.initarglist(args)
@@ -262,7 +261,12 @@
             assert descr is None
             return op
         else:
-            return ResOperation(opnum, args, result, descr)
+            res = ResOperation(opnum, args, result, descr)
+            if fail_args is not None:
+                res.setfailargs(fail_args)
+            if self._postproces:
+                self._postproces(res)
+            return res
 
     def parse_result_op(self, line):
         res, op = line.split("=", 1)
@@ -273,16 +277,12 @@
             raise ParseError("Double assign to var %s in line: %s" % (res, line))
         rvar = self.box_for_var(res)
         self.vars[res] = rvar
-        res = self.create_op(opnum, args, rvar, descr)
-        if fail_args is not None:
-            res.setfailargs(fail_args)
+        res = self.create_op(opnum, args, rvar, descr, fail_args)
         return res
 
     def parse_op_no_result(self, line):
         opnum, args, descr, fail_args = self.parse_op(line)
-        res = self.create_op(opnum, args, None, descr)
-        if fail_args is not None:
-            res.setfailargs(fail_args)
+        res = self.create_op(opnum, args, None, descr, fail_args)
         return res
 
     def parse_next_op(self, line):
@@ -377,11 +377,12 @@
 
 def parse(input, cpu=None, namespace=None, type_system='lltype',
           boxkinds=None, invent_fail_descr=default_fail_descr,
-          no_namespace=False, nonstrict=False, OpParser=OpParser):
+          no_namespace=False, nonstrict=False, OpParser=OpParser,
+          postprocess=None):
     if namespace is None and not no_namespace:
         namespace = {}
     return OpParser(input, cpu, namespace, type_system, boxkinds,
-                    invent_fail_descr, nonstrict).parse()
+                    invent_fail_descr, nonstrict, postprocess).parse()
 
 def pure_parse(*args, **kwds):
     kwds['invent_fail_descr'] = None

From noreply at buildbot.pypy.org  Thu Dec 25 21:47:18 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 25 Dec 2014 21:47:18 +0100 (CET)
Subject: [pypy-commit] pypy pypy-dont-copy-ops: fix first problems (it is
	going to be fun)
Message-ID: <20141225204718.8F1FE1C31CF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: pypy-dont-copy-ops
Changeset: r75123:eac43a44411d
Date: 2014-12-25 22:47 +0200
http://bitbucket.org/pypy/pypy/changeset/eac43a44411d/

Log:	fix first problems (it is going to be fun)

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -378,10 +378,10 @@
     def getintbound(self):
         return self.intbound
 
-    def get_last_guard(self):
+    def get_last_guard(self, optimizer):
         return None
 
-    def get_known_class(self):
+    def get_known_class(self, optimizer):
         return None
 
     def getlenbound(self):
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -317,7 +317,6 @@
                         args = [old_guard_op.getarg(0), op.getarg(1)],
                         descr = descr)
             self.optimizer.replace_guard(op, value)
-            descr.make_a_counter_per_value(op)
             # to be safe
             if isinstance(value, PtrOptValue):
                 value.last_guard_pos = -1
@@ -365,8 +364,8 @@
                             args = [old_guard_op.getarg(0), op.getarg(1)],
                             descr=descr)
                 self.optimizer.replace_guard(op, value)
+        value.make_constant_class(self.optimizer, expectedclassbox)
         self.emit_operation(op)
-        value.make_constant_class(self.optimizer, expectedclassbox)
 
     def optimize_GUARD_NONNULL_CLASS(self, op):
         value = self.getvalue(op.getarg(0))

From noreply at buildbot.pypy.org  Thu Dec 25 22:00:58 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 25 Dec 2014 22:00:58 +0100 (CET)
Subject: [pypy-commit] pypy default: test,
	fix int64 < 0 conversion on 32 bit platforms
Message-ID: <20141225210058.21ED01D288E@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r75124:0a4307698d88
Date: 2014-12-25 22:58 +0200
http://bitbucket.org/pypy/pypy/changeset/0a4307698d88/

Log:	test, fix int64 < 0 conversion on 32 bit platforms

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -287,6 +287,8 @@
         a = np.array('123', dtype='intp')
         assert a == 123
         assert a.dtype == np.intp
+        a = np.array( -(2**60))
+        assert a.dtype == np.int64
 
     def test_array_copy(self):
         from numpy import array
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -619,6 +619,9 @@
             space.int_w(w_obj)
         except OperationError, e:
             if e.match(space, space.w_OverflowError):
+                if space.r_longlong_w(w_obj) < 0:
+                    return find_binop_result_dtype(space, int64_dtype,
+                                               current_guess)
                 return find_binop_result_dtype(space, uint64_dtype,
                                                current_guess)
             raise

From noreply at buildbot.pypy.org  Fri Dec 26 07:29:12 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 26 Dec 2014 07:29:12 +0100 (CET)
Subject: [pypy-commit] pypy default: move test, fix for all cases
Message-ID: <20141226062912.CCAE51D22DE@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r75125:fb69c75eb931
Date: 2014-12-26 08:28 +0200
http://bitbucket.org/pypy/pypy/changeset/fb69c75eb931/

Log:	move test, fix for all cases

diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -665,6 +665,7 @@
 
         assert numpy.int64(9223372036854775807) == 9223372036854775807
         assert numpy.int64(9223372036854775807) == 9223372036854775807
+        assert numpy.int64(-9223372036854775807) == -9223372036854775807
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, 9223372036854775808L)
 
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -287,8 +287,6 @@
         a = np.array('123', dtype='intp')
         assert a == 123
         assert a.dtype == np.intp
-        a = np.array( -(2**60))
-        assert a.dtype == np.int64
 
     def test_array_copy(self):
         from numpy import array
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -619,7 +619,7 @@
             space.int_w(w_obj)
         except OperationError, e:
             if e.match(space, space.w_OverflowError):
-                if space.r_longlong_w(w_obj) < 0:
+                if space.is_true(space.le(w_obj, space.wrap(0))):
                     return find_binop_result_dtype(space, int64_dtype,
                                                current_guess)
                 return find_binop_result_dtype(space, uint64_dtype,

From noreply at buildbot.pypy.org  Fri Dec 26 17:28:50 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 26 Dec 2014 17:28:50 +0100 (CET)
Subject: [pypy-commit] pypy default: Update to cffi/ecf4fb44f740.
Message-ID: <20141226162850.127041D28F8@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75126:731706d14816
Date: 2014-12-26 17:28 +0100
http://bitbucket.org/pypy/pypy/changeset/731706d14816/

Log:	Update to cffi/ecf4fb44f740.

diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -62,9 +62,29 @@
 
 eptype("intptr_t",  rffi.INTPTR_T,  ctypeprim.W_CTypePrimitiveSigned)
 eptype("uintptr_t", rffi.UINTPTR_T, ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("ptrdiff_t", rffi.INTPTR_T,  ctypeprim.W_CTypePrimitiveSigned) # <-xxx
+eptype("ptrdiff_t", rffi.PTRDIFF_T, ctypeprim.W_CTypePrimitiveSigned)
 eptype("size_t",    rffi.SIZE_T,    ctypeprim.W_CTypePrimitiveUnsigned)
 eptype("ssize_t",   rffi.SSIZE_T,   ctypeprim.W_CTypePrimitiveSigned)
+eptype("intmax_t",  rffi.INTMAX_T,  ctypeprim.W_CTypePrimitiveSigned)
+eptype("uintmax_t", rffi.UINTMAX_T, ctypeprim.W_CTypePrimitiveUnsigned)
+
+eptype("int_least8_t",  rffi.INT_LEAST8_T,  ctypeprim.W_CTypePrimitiveSigned)
+eptype("int_least16_t", rffi.INT_LEAST16_T, ctypeprim.W_CTypePrimitiveSigned)
+eptype("int_least32_t", rffi.INT_LEAST32_T, ctypeprim.W_CTypePrimitiveSigned)
+eptype("int_least64_t", rffi.INT_LEAST64_T, ctypeprim.W_CTypePrimitiveSigned)
+eptype("uint_least8_t", rffi.UINT_LEAST8_T,  ctypeprim.W_CTypePrimitiveUnsigned)
+eptype("uint_least16_t",rffi.UINT_LEAST16_T, ctypeprim.W_CTypePrimitiveUnsigned)
+eptype("uint_least32_t",rffi.UINT_LEAST32_T, ctypeprim.W_CTypePrimitiveUnsigned)
+eptype("uint_least64_t",rffi.UINT_LEAST64_T, ctypeprim.W_CTypePrimitiveUnsigned)
+
+eptype("int_fast8_t",  rffi.INT_FAST8_T,  ctypeprim.W_CTypePrimitiveSigned)
+eptype("int_fast16_t", rffi.INT_FAST16_T, ctypeprim.W_CTypePrimitiveSigned)
+eptype("int_fast32_t", rffi.INT_FAST32_T, ctypeprim.W_CTypePrimitiveSigned)
+eptype("int_fast64_t", rffi.INT_FAST64_T, ctypeprim.W_CTypePrimitiveSigned)
+eptype("uint_fast8_t", rffi.UINT_FAST8_T,  ctypeprim.W_CTypePrimitiveUnsigned)
+eptype("uint_fast16_t",rffi.UINT_FAST16_T, ctypeprim.W_CTypePrimitiveUnsigned)
+eptype("uint_fast32_t",rffi.UINT_FAST32_T, ctypeprim.W_CTypePrimitiveUnsigned)
+eptype("uint_fast64_t",rffi.UINT_FAST64_T, ctypeprim.W_CTypePrimitiveUnsigned)
 
 @unwrap_spec(name=str)
 def new_primitive_type(space, name):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2718,7 +2718,16 @@
 def test_nonstandard_integer_types():
     for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t',
                      'uint32_t', 'int64_t', 'uint64_t', 'intptr_t',
-                     'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t']:
+                     'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t',
+                     'int_least8_t',  'uint_least8_t',
+                     'int_least16_t', 'uint_least16_t',
+                     'int_least32_t', 'uint_least32_t',
+                     'int_least64_t', 'uint_least64_t',
+                     'int_fast8_t',  'uint_fast8_t',
+                     'int_fast16_t', 'uint_fast16_t',
+                     'int_fast32_t', 'uint_fast32_t',
+                     'int_fast64_t', 'uint_fast64_t',
+                     'intmax_t', 'uintmax_t']:
         new_primitive_type(typename)    # works
 
 def test_cannot_convert_unicode_to_charp():
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -461,7 +461,16 @@
 TYPES += ['signed char', 'unsigned char',
           'long long', 'unsigned long long',
           'size_t', 'time_t', 'wchar_t',
-          'uintptr_t', 'intptr_t',
+          'uintptr_t', 'intptr_t', 'ptrdiff_t',
+          'int_least8_t',  'uint_least8_t',
+          'int_least16_t', 'uint_least16_t',
+          'int_least32_t', 'uint_least32_t',
+          'int_least64_t', 'uint_least64_t',
+          'int_fast8_t',  'uint_fast8_t',
+          'int_fast16_t', 'uint_fast16_t',
+          'int_fast32_t', 'uint_fast32_t',
+          'int_fast64_t', 'uint_fast64_t',
+          'intmax_t', 'uintmax_t',
           'void*']    # generic pointer type
 
 # This is a bit of a hack since we can't use rffi_platform here.
@@ -471,7 +480,6 @@
 except CompilationError:
     pass
 
-_TYPES_ARE_UNSIGNED = set(['size_t', 'uintptr_t'])   # plus "unsigned *"
 if os.name != 'nt':
     TYPES.append('mode_t')
     TYPES.append('pid_t')
@@ -489,8 +497,10 @@
         if name.startswith('unsigned'):
             name = 'u' + name[9:]
             signed = False
+        elif name == 'size_t' or name.startswith('uint'):
+            signed = False
         else:
-            signed = (name not in _TYPES_ARE_UNSIGNED)
+            signed = True
         name = name.replace(' ', '')
         names.append(name)
         populatelist.append((name.upper(), c_name, signed))
diff --git a/rpython/rtyper/tool/rfficache.py b/rpython/rtyper/tool/rfficache.py
--- a/rpython/rtyper/tool/rfficache.py
+++ b/rpython/rtyper/tool/rfficache.py
@@ -14,7 +14,7 @@
     from rpython.translator.platform import platform
     includes = ['stdlib.h', 'stdio.h', 'sys/types.h']
     if platform.name != 'msvc':
-        includes += ['inttypes.h']
+        includes += ['inttypes.h', 'stddef.h']
     include_string = "\n".join(["#include <%s>" % i for i in includes])
     c_source = py.code.Source('''
     // includes

From noreply at buildbot.pypy.org  Fri Dec 26 17:52:42 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 26 Dec 2014 17:52:42 +0100 (CET)
Subject: [pypy-commit] pypy default: Windows support
Message-ID: <20141226165242.ECF4D1C10F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75127:937033492549
Date: 2014-12-26 17:52 +0100
http://bitbucket.org/pypy/pypy/changeset/937033492549/

Log:	Windows support

diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -62,29 +62,53 @@
 
 eptype("intptr_t",  rffi.INTPTR_T,  ctypeprim.W_CTypePrimitiveSigned)
 eptype("uintptr_t", rffi.UINTPTR_T, ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("ptrdiff_t", rffi.PTRDIFF_T, ctypeprim.W_CTypePrimitiveSigned)
 eptype("size_t",    rffi.SIZE_T,    ctypeprim.W_CTypePrimitiveUnsigned)
 eptype("ssize_t",   rffi.SSIZE_T,   ctypeprim.W_CTypePrimitiveSigned)
-eptype("intmax_t",  rffi.INTMAX_T,  ctypeprim.W_CTypePrimitiveSigned)
-eptype("uintmax_t", rffi.UINTMAX_T, ctypeprim.W_CTypePrimitiveUnsigned)
 
-eptype("int_least8_t",  rffi.INT_LEAST8_T,  ctypeprim.W_CTypePrimitiveSigned)
-eptype("int_least16_t", rffi.INT_LEAST16_T, ctypeprim.W_CTypePrimitiveSigned)
-eptype("int_least32_t", rffi.INT_LEAST32_T, ctypeprim.W_CTypePrimitiveSigned)
-eptype("int_least64_t", rffi.INT_LEAST64_T, ctypeprim.W_CTypePrimitiveSigned)
-eptype("uint_least8_t", rffi.UINT_LEAST8_T,  ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("uint_least16_t",rffi.UINT_LEAST16_T, ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("uint_least32_t",rffi.UINT_LEAST32_T, ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("uint_least64_t",rffi.UINT_LEAST64_T, ctypeprim.W_CTypePrimitiveUnsigned)
+_WCTSigned = ctypeprim.W_CTypePrimitiveSigned
+_WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned
 
-eptype("int_fast8_t",  rffi.INT_FAST8_T,  ctypeprim.W_CTypePrimitiveSigned)
-eptype("int_fast16_t", rffi.INT_FAST16_T, ctypeprim.W_CTypePrimitiveSigned)
-eptype("int_fast32_t", rffi.INT_FAST32_T, ctypeprim.W_CTypePrimitiveSigned)
-eptype("int_fast64_t", rffi.INT_FAST64_T, ctypeprim.W_CTypePrimitiveSigned)
-eptype("uint_fast8_t", rffi.UINT_FAST8_T,  ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("uint_fast16_t",rffi.UINT_FAST16_T, ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("uint_fast32_t",rffi.UINT_FAST32_T, ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("uint_fast64_t",rffi.UINT_FAST64_T, ctypeprim.W_CTypePrimitiveUnsigned)
+eptype("ptrdiff_t", getattr(rffi, 'PTRDIFF_T', rffi.INTPTR_T), _WCTSigned)
+eptype("intmax_t",  getattr(rffi, 'INTMAX_T',  rffi.LONGLONG), _WCTSigned)
+eptype("uintmax_t", getattr(rffi, 'UINTMAX_T', rffi.LONGLONG), _WCTUnsign)
+
+if hasattr(rffi, 'INT_LEAST8_T'):
+    eptype("int_least8_t",  rffi.INT_LEAST8_T,  _WCTSigned)
+    eptype("int_least16_t", rffi.INT_LEAST16_T, _WCTSigned)
+    eptype("int_least32_t", rffi.INT_LEAST32_T, _WCTSigned)
+    eptype("int_least64_t", rffi.INT_LEAST64_T, _WCTSigned)
+    eptype("uint_least8_t", rffi.UINT_LEAST8_T,  _WCTUnsign)
+    eptype("uint_least16_t",rffi.UINT_LEAST16_T, _WCTUnsign)
+    eptype("uint_least32_t",rffi.UINT_LEAST32_T, _WCTUnsign)
+    eptype("uint_least64_t",rffi.UINT_LEAST64_T, _WCTUnsign)
+else:
+    eptypesize("int_least8_t",   1, _WCTSigned)
+    eptypesize("uint_least8_t",  1, _WCTUnsign)
+    eptypesize("int_least16_t",  2, _WCTSigned)
+    eptypesize("uint_least16_t", 2, _WCTUnsign)
+    eptypesize("int_least32_t",  4, _WCTSigned)
+    eptypesize("uint_least32_t", 4, _WCTUnsign)
+    eptypesize("int_least64_t",  8, _WCTSigned)
+    eptypesize("uint_least64_t", 8, _WCTUnsign)
+
+if hasattr(rffi, 'INT_FAST8_T'):
+    eptype("int_fast8_t",  rffi.INT_FAST8_T,  _WCTSigned)
+    eptype("int_fast16_t", rffi.INT_FAST16_T, _WCTSigned)
+    eptype("int_fast32_t", rffi.INT_FAST32_T, _WCTSigned)
+    eptype("int_fast64_t", rffi.INT_FAST64_T, _WCTSigned)
+    eptype("uint_fast8_t", rffi.UINT_FAST8_T,  _WCTUnsign)
+    eptype("uint_fast16_t",rffi.UINT_FAST16_T, _WCTUnsign)
+    eptype("uint_fast32_t",rffi.UINT_FAST32_T, _WCTUnsign)
+    eptype("uint_fast64_t",rffi.UINT_FAST64_T, _WCTUnsign)
+else:
+    eptypesize("int_fast8_t",   1, _WCTSigned)
+    eptypesize("uint_fast8_t",  1, _WCTUnsign)
+    eptypesize("int_fast16_t",  2, _WCTSigned)
+    eptypesize("uint_fast16_t", 2, _WCTUnsign)
+    eptypesize("int_fast32_t",  4, _WCTSigned)
+    eptypesize("uint_fast32_t", 4, _WCTUnsign)
+    eptypesize("int_fast64_t",  8, _WCTSigned)
+    eptypesize("uint_fast64_t", 8, _WCTUnsign)
 
 @unwrap_spec(name=str)
 def new_primitive_type(space, name):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -461,16 +461,7 @@
 TYPES += ['signed char', 'unsigned char',
           'long long', 'unsigned long long',
           'size_t', 'time_t', 'wchar_t',
-          'uintptr_t', 'intptr_t', 'ptrdiff_t',
-          'int_least8_t',  'uint_least8_t',
-          'int_least16_t', 'uint_least16_t',
-          'int_least32_t', 'uint_least32_t',
-          'int_least64_t', 'uint_least64_t',
-          'int_fast8_t',  'uint_fast8_t',
-          'int_fast16_t', 'uint_fast16_t',
-          'int_fast32_t', 'uint_fast32_t',
-          'int_fast64_t', 'uint_fast64_t',
-          'intmax_t', 'uintmax_t',
+          'uintptr_t', 'intptr_t',
           'void*']    # generic pointer type
 
 # This is a bit of a hack since we can't use rffi_platform here.
@@ -484,6 +475,17 @@
     TYPES.append('mode_t')
     TYPES.append('pid_t')
     TYPES.append('ssize_t')
+    # the types below are rare enough and not available on Windows
+    TYPES.extend(['ptrdiff_t',
+          'int_least8_t',  'uint_least8_t',
+          'int_least16_t', 'uint_least16_t',
+          'int_least32_t', 'uint_least32_t',
+          'int_least64_t', 'uint_least64_t',
+          'int_fast8_t',  'uint_fast8_t',
+          'int_fast16_t', 'uint_fast16_t',
+          'int_fast32_t', 'uint_fast32_t',
+          'int_fast64_t', 'uint_fast64_t',
+          'intmax_t', 'uintmax_t'])
 else:
     MODE_T = lltype.Signed
     PID_T = lltype.Signed

From noreply at buildbot.pypy.org  Fri Dec 26 17:59:05 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 26 Dec 2014 17:59:05 +0100 (CET)
Subject: [pypy-commit] pypy default: fix
Message-ID: <20141226165905.529841C10F7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75128:87009b0fd602
Date: 2014-12-26 17:59 +0100
http://bitbucket.org/pypy/pypy/changeset/87009b0fd602/

Log:	fix

diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -227,7 +227,7 @@
     '<tag key="value" />'
     >>> element.remove(subelement)    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ValueError: list.remove(x): ...
+    ValueError: list.remove(...
     >>> serialize(element) # 6
     '<tag key="value" />'
     >>> element[0:0] = [subelement, subelement, subelement]

From noreply at buildbot.pypy.org  Fri Dec 26 18:01:36 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 26 Dec 2014 18:01:36 +0100 (CET)
Subject: [pypy-commit] pypy default: trying to fix the linux64 nightly tests
Message-ID: <20141226170136.2D0A41C1347@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75129:a59b791f9299
Date: 2014-12-26 18:01 +0100
http://bitbucket.org/pypy/pypy/changeset/a59b791f9299/

Log:	trying to fix the linux64 nightly tests

diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -286,7 +286,7 @@
 
             t = thread.start_new_thread(pollster.poll, ())
             try:
-                time.sleep(0.1)
+                time.sleep(0.3)
                 for i in range(5): print '',  # to release GIL untranslated
                 # trigger ufds array reallocation
                 for fd in rfds:
@@ -297,7 +297,7 @@
             finally:
                 # and make the call to poll() from the thread return
                 os.write(w, b'spam')
-                time.sleep(0.1)
+                time.sleep(0.3)
                 for i in range(5): print '',  # to release GIL untranslated
         finally:
             os.close(r)

From noreply at buildbot.pypy.org  Fri Dec 26 18:46:32 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 26 Dec 2014 18:46:32 +0100 (CET)
Subject: [pypy-commit] pypy default: Optimize the bytecode for the case "if
	a or b:"
Message-ID: <20141226174632.EAFC91C0501@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75130:534bd32933a1
Date: 2014-12-26 18:45 +0100
http://bitbucket.org/pypy/pypy/changeset/534bd32933a1/

Log:	Optimize the bytecode for the case "if a or b:"

diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -83,17 +83,16 @@
 
 class __extend__(ast.BoolOp):
 
-    def _accept_jump_if_any_is(self, gen, condition, target):
-        self.values[0].accept_jump_if(gen, condition, target)
-        for i in range(1, len(self.values)):
+    def _accept_jump_if_any_is(self, gen, condition, target, skip_last=0):
+        for i in range(len(self.values) - skip_last):
             self.values[i].accept_jump_if(gen, condition, target)
 
     def accept_jump_if(self, gen, condition, target):
         if condition and self.op == ast.And or \
                 (not condition and self.op == ast.Or):
             end = gen.new_block()
-            self._accept_jump_if_any_is(gen, not condition, end)
-            gen.emit_jump(ops.JUMP_FORWARD, target)
+            self._accept_jump_if_any_is(gen, not condition, end, skip_last=1)
+            self.values[-1].accept_jump_if(gen, condition, target)
             gen.use_next_block(end)
         else:
             self._accept_jump_if_any_is(gen, condition, target)

From noreply at buildbot.pypy.org  Sat Dec 27 19:03:19 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 27 Dec 2014 19:03:19 +0100 (CET)
Subject: [pypy-commit] pypy default: Export in app-level buffer objects the
 method "_pypy_raw_address()"
Message-ID: <20141227180319.D103C1C1148@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75131:3899fdaff537
Date: 2014-12-27 19:03 +0100
http://bitbucket.org/pypy/pypy/changeset/3899fdaff537/

Log:	Export in app-level buffer objects the method "_pypy_raw_address()"
	mapping to the internal buffer's get_raw_address().

diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,4 +1,5 @@
 from rpython.rlib.buffer import Buffer
+from rpython.rtyper.lltypesystem import rffi
 
 # XXX not the most efficient implementation
 
@@ -20,3 +21,7 @@
     def setitem(self, index, char):
         ll_buffer = self.datainstance.ll_buffer
         ll_buffer[index] = char
+
+    def get_raw_address(self):
+        ll_buffer = self.datainstance.ll_buffer
+        return rffi.cast(rffi.CCHARP, ll_buffer)
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1144,6 +1144,15 @@
         b[3] = b'x'
         assert b[3] == b'x'
 
+    def test_pypy_raw_address(self):
+        import _rawffi
+        S = _rawffi.Structure((40, 1))
+        s = S(autofree=True)
+        addr = buffer(s)._pypy_raw_address()
+        assert type(addr) is int
+        assert buffer(s)._pypy_raw_address() == addr
+        assert buffer(s, 10)._pypy_raw_address() == addr + 10
+
     def test_union(self):
         import _rawffi
         longsize = _rawffi.sizeof('l')
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -244,6 +244,9 @@
     def getitem(self, index):
         return self.ptr[index]
 
+    def get_raw_address(self):
+        return rffi.cast(rffi.CCHARP, self.ptr)
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -5,7 +5,7 @@
 from rpython.rlib.objectmodel import compute_hash
 
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
 
@@ -123,6 +123,17 @@
         return space.wrap("<%s for 0x%s, size %d>" %
                           (info, addrstring, self.buf.getlength()))
 
+    def descr_pypy_raw_address(self, space):
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        try:
+            ptr = self.buf.get_raw_address()
+        except ValueError:
+            # report the error using the RPython-level internal repr of self.buf
+            msg = ("cannot find the underlying address of buffer that "
+                   "is internally %r" % (self.buf,))
+            raise OperationError(space.w_ValueError, space.wrap(msg))
+        return space.wrap(rffi.cast(lltype.Signed, ptr))
+
 W_Buffer.typedef = TypeDef(
     "buffer",
     __doc__ = """\
@@ -149,5 +160,6 @@
     __mul__ = interp2app(W_Buffer.descr_mul),
     __rmul__ = interp2app(W_Buffer.descr_mul),
     __repr__ = interp2app(W_Buffer.descr_repr),
+    _pypy_raw_address = interp2app(W_Buffer.descr_pypy_raw_address),
 )
 W_Buffer.typedef.acceptable_as_base_class = False
diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py
--- a/pypy/objspace/std/test/test_bufferobject.py
+++ b/pypy/objspace/std/test/test_bufferobject.py
@@ -197,3 +197,9 @@
         buf = buffer('hello world')
         raises(TypeError, "buf[MyInt(0)]")
         raises(TypeError, "buf[MyInt(0):MyInt(5)]")
+
+    def test_pypy_raw_address_base(self):
+        raises(ValueError, buffer("foobar")._pypy_raw_address)
+        raises(ValueError, buffer(u"foobar")._pypy_raw_address)
+        e = raises(ValueError, buffer(bytearray("foobar"))._pypy_raw_address)
+        assert 'BytearrayBuffer' in str(e.value)
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -115,3 +115,8 @@
             return        # otherwise, adding self.offset might make 'start'
                           # out of bounds
         self.buffer.setslice(self.offset + start, string)
+
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        ptr = self.buffer.get_raw_address()
+        return rffi.ptradd(ptr, self.offset)

From noreply at buildbot.pypy.org  Sun Dec 28 06:12:06 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 28 Dec 2014 06:12:06 +0100 (CET)
Subject: [pypy-commit] cffi default: Fight a lot the CPython
 buffer/memoryview interface until we get a
Message-ID: <20141228051206.DE79E1C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1604:cda48f686feb
Date: 2014-12-28 06:12 +0100
http://bitbucket.org/cffi/cffi/changeset/cda48f686feb/

Log:	Fight a lot the CPython buffer/memoryview interface until we get a
	hopefully not-too-buggy ffi.from_buffer(). Obviously it's only
	half-tested because 2.7 doesn't really have built-in memoryview
	objects.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -86,7 +86,7 @@
 # define PyText_FromStringAndSize PyString_FromStringAndSize
 # define PyText_InternInPlace PyString_InternInPlace
 # define PyIntOrLong_Check(op) (PyInt_Check(op) || PyLong_Check(op))
-#endif 
+#endif
 
 #if PY_MAJOR_VERSION >= 3
 # define PyInt_FromLong PyLong_FromLong
@@ -130,6 +130,7 @@
 #define CT_IS_FILE             262144
 #define CT_IS_VOID_PTR         524288
 #define CT_WITH_VAR_ARRAY     1048576
+#define CT_IS_UNSIZED_CHAR_A  2097152
 #define CT_PRIMITIVE_ANY  (CT_PRIMITIVE_SIGNED |        \
                            CT_PRIMITIVE_UNSIGNED |      \
                            CT_PRIMITIVE_CHAR |          \
@@ -224,6 +225,12 @@
 } CDataObject_own_structptr;
 
 typedef struct {
+    CDataObject head;
+    Py_ssize_t length;     /* same as CDataObject_own_length up to here */
+    Py_buffer *bufferview;
+} CDataObject_owngc_frombuf;
+
+typedef struct {
     ffi_cif cif;
     /* the following information is used when doing the call:
        - a buffer of size 'exchange_size' is malloced
@@ -1546,6 +1553,11 @@
         Py_XDECREF(args);
         cffi_closure_free(closure);
     }
+    else if (cd->c_type->ct_flags & CT_IS_UNSIZED_CHAR_A) {  /* from_buffer */
+        Py_buffer *view = ((CDataObject_owngc_frombuf *)cd)->bufferview;
+        PyBuffer_Release(view);
+        PyObject_Free(view);
+    }
     cdata_dealloc(cd);
 }
 
@@ -1560,6 +1572,10 @@
         PyObject *args = (PyObject *)(closure->user_data);
         Py_VISIT(args);
     }
+    else if (cd->c_type->ct_flags & CT_IS_UNSIZED_CHAR_A) {  /* from_buffer */
+        Py_buffer *view = ((CDataObject_owngc_frombuf *)cd)->bufferview;
+        Py_VISIT(view->obj);
+    }
     return 0;
 }
 
@@ -1577,6 +1593,10 @@
         closure->user_data = NULL;
         Py_XDECREF(args);
     }
+    else if (cd->c_type->ct_flags & CT_IS_UNSIZED_CHAR_A) {  /* from_buffer */
+        Py_buffer *view = ((CDataObject_owngc_frombuf *)cd)->bufferview;
+        PyBuffer_Release(view);
+    }
     return 0;
 }
 
@@ -1679,35 +1699,40 @@
 static PyObject *cdataowning_repr(CDataObject *cd)
 {
     Py_ssize_t size;
-    if (cd->c_type->ct_flags & CT_POINTER) {
-        if (cd->c_type->ct_flags & CT_IS_VOID_PTR)
-            goto handle_repr;
+    if (cd->c_type->ct_flags & CT_POINTER)
         size = cd->c_type->ct_itemdescr->ct_size;
-    }
     else if (cd->c_type->ct_flags & CT_ARRAY)
         size = get_array_length(cd) * cd->c_type->ct_itemdescr->ct_size;
-    else if (cd->c_type->ct_flags & CT_FUNCTIONPTR)
-        goto callback_repr;
     else
         size = cd->c_type->ct_size;
 
     return PyText_FromFormat("<cdata '%s' owning %zd bytes>",
                              cd->c_type->ct_name, size);
-
- callback_repr:
-    {
+}
+
+static PyObject *cdataowninggc_repr(CDataObject *cd)
+{
+    if (cd->c_type->ct_flags & CT_IS_VOID_PTR) {        /* a handle */
+        PyObject *x = (PyObject *)(cd->c_data + 42);
+        return _cdata_repr2(cd, "handle to", x);
+    }
+    else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) {   /* a callback */
         PyObject *args = (PyObject *)((ffi_closure *)cd->c_data)->user_data;
         if (args == NULL)
             return cdata_repr(cd);
         else
             return _cdata_repr2(cd, "calling", PyTuple_GET_ITEM(args, 1));
     }
-
- handle_repr:
-    {
-        PyObject *x = (PyObject *)(cd->c_data + 42);
-        return _cdata_repr2(cd, "handle to", x);
-    }
+    else if (cd->c_type->ct_flags & CT_IS_UNSIZED_CHAR_A) {  /* from_buffer */
+        Py_buffer *view = ((CDataObject_owngc_frombuf *)cd)->bufferview;
+        Py_ssize_t buflen = get_array_length(cd);
+        return PyText_FromFormat(
+            "<cdata '%s' buffer len %zd from '%.200s' object>",
+            cd->c_type->ct_name,
+            buflen,
+            view->obj ? Py_TYPE(view->obj)->tp_name : "(null)");
+    }
+    return cdataowning_repr(cd);
 }
 
 static int cdata_nonzero(CDataObject *cd)
@@ -2397,7 +2422,7 @@
                 ct = ((CDataObject *)obj)->c_type;
                 if (ct->ct_flags & (CT_PRIMITIVE_CHAR | CT_PRIMITIVE_UNSIGNED |
                                     CT_PRIMITIVE_SIGNED)) {
-                    if (ct->ct_size < sizeof(int)) {
+                    if (ct->ct_size < (Py_ssize_t)sizeof(int)) {
                         ct = _get_ct_int();
                         if (ct == NULL)
                             goto error;
@@ -2626,14 +2651,14 @@
 static PyTypeObject CDataOwningGC_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "_cffi_backend.CDataOwnGC",
-    sizeof(CDataObject),
+    sizeof(CDataObject_owngc_frombuf),
     0,
     (destructor)cdataowninggc_dealloc,          /* tp_dealloc */
     0,                                          /* tp_print */
     0,                                          /* tp_getattr */
     0,                                          /* tp_setattr */
     0,                                          /* tp_compare */
-    0,                                          /* tp_repr */
+    (reprfunc)cdataowninggc_repr,               /* tp_repr */
     0,                                          /* tp_as_number */
     0,                                          /* tp_as_sequence */
     0,                                          /* tp_as_mapping */
@@ -3454,11 +3479,11 @@
     td->ct_extra = ffitype;
     td->ct_flags = ptypes->flags;
     if (td->ct_flags & (CT_PRIMITIVE_SIGNED | CT_PRIMITIVE_CHAR)) {
-        if (td->ct_size <= sizeof(long))
+        if (td->ct_size <= (Py_ssize_t)sizeof(long))
             td->ct_flags |= CT_PRIMITIVE_FITS_LONG;
     }
     else if (td->ct_flags & CT_PRIMITIVE_UNSIGNED) {
-        if (td->ct_size < sizeof(long))
+        if (td->ct_size < (Py_ssize_t)sizeof(long))
             td->ct_flags |= CT_PRIMITIVE_FITS_LONG;
     }
     td->ct_name_position = strlen(td->ct_name);
@@ -3521,6 +3546,7 @@
     CTypeDescrObject *td, *ctitem;
     char extra_text[32];
     Py_ssize_t length, arraysize;
+    int flags = CT_ARRAY;
 
     if (!(ctptr->ct_flags & CT_POINTER)) {
         PyErr_SetString(PyExc_TypeError, "first arg must be a pointer ctype");
@@ -3537,6 +3563,9 @@
         sprintf(extra_text, "[]");
         length = -1;
         arraysize = -1;
+        if ((ctptr->ct_flags & CT_CAST_ANYTHING) &&
+                (ctitem->ct_flags & CT_PRIMITIVE_CHAR))
+            flags |= CT_IS_UNSIZED_CHAR_A;
     }
     else {
         length = PyNumber_AsSsize_t(lengthobj, PyExc_OverflowError);
@@ -3561,7 +3590,7 @@
     td->ct_stuff = (PyObject *)ctptr;
     td->ct_size = arraysize;
     td->ct_length = length;
-    td->ct_flags = CT_ARRAY;
+    td->ct_flags = flags;
     return (PyObject *)td;
 }
 
@@ -5081,6 +5110,96 @@
     return x;
 }
 
+static int _my_PyObject_GetContiguousBuffer(PyObject *x, Py_buffer *view)
+{
+#if PY_MAJOR_VERSION < 3
+    /* Some objects only support the buffer interface and CPython doesn't
+       translate it into the memoryview interface, mess.  Hack a very
+       minimal content for 'view'.  Don't care if the other fields are
+       uninitialized: we only call PyBuffer_Release(), which only reads
+       'view->obj'. */
+    PyBufferProcs *pb = x->ob_type->tp_as_buffer;
+    if (pb && !pb->bf_releasebuffer) {
+        /* try all three in some vaguely sensible order */
+        readbufferproc proc = (readbufferproc)pb->bf_getwritebuffer;
+        if (!proc)     proc = (readbufferproc)pb->bf_getcharbuffer;
+        if (!proc)     proc = (readbufferproc)pb->bf_getreadbuffer;
+        if (proc && pb->bf_getsegcount) {
+            if ((*pb->bf_getsegcount)(x, NULL) != 1) {
+                PyErr_SetString(PyExc_TypeError,
+                                "expected a single-segment buffer object");
+                return -1;
+            }
+            view->len = (*proc)(x, 0, &view->buf);
+            if (view->len < 0)
+                return -1;
+            view->obj = x;
+            Py_INCREF(x);
+            return 0;
+        }
+    }
+#endif
+
+    if (PyObject_GetBuffer(x, view, PyBUF_SIMPLE) < 0)
+        return -1;
+
+    if (!PyBuffer_IsContiguous(view, 'A')) {
+        PyBuffer_Release(view);
+        PyErr_SetString(PyExc_TypeError, "contiguous buffer expected");
+        return -1;
+    }
+    return 0;
+}
+
+static PyObject *b_from_buffer(PyObject *self, PyObject *args)
+{
+    CTypeDescrObject *ct;
+    CDataObject *cd;
+    PyObject *x;
+    Py_buffer *view;
+
+    if (!PyArg_ParseTuple(args, "O!O", &CTypeDescr_Type, &ct, &x))
+        return NULL;
+
+    if (!(ct->ct_flags & CT_IS_UNSIZED_CHAR_A)) {
+        PyErr_Format(PyExc_TypeError, "needs 'char[]', got '%s'", ct->ct_name);
+        return NULL;
+    }
+
+    if (PyString_Check(x) || PyUnicode_Check(x) ||
+        PyByteArray_Check(x) /* <= this one here for PyPy compatibility */ ) {
+        PyErr_SetString(PyExc_TypeError,
+                        "from_buffer() cannot return the address of the "
+                        "raw string within a "STR_OR_BYTES" or unicode or "
+                        "bytearray object");
+        return NULL;
+    }
+
+    view = PyObject_Malloc(sizeof(Py_buffer));
+    if (_my_PyObject_GetContiguousBuffer(x, view) < 0)
+        goto error1;
+
+    cd = (CDataObject *)PyObject_GC_New(CDataObject_owngc_frombuf,
+                                        &CDataOwningGC_Type);
+    if (cd == NULL)
+        goto error2;
+
+    Py_INCREF(ct);
+    cd->c_type = ct;
+    cd->c_data = view->buf;
+    cd->c_weakreflist = NULL;
+    ((CDataObject_owngc_frombuf *)cd)->length = view->len;
+    ((CDataObject_owngc_frombuf *)cd)->bufferview = view;
+    PyObject_GC_Track(cd);
+    return (PyObject *)cd;
+
+ error2:
+    PyBuffer_Release(view);
+ error1:
+    PyObject_Free(view);
+    return NULL;
+}
+
 static PyObject *b__get_types(PyObject *self, PyObject *noarg)
 {
     return PyTuple_Pack(2, (PyObject *)&CData_Type,
@@ -5326,6 +5445,7 @@
     {"set_errno", b_set_errno, METH_VARARGS},
     {"newp_handle", b_newp_handle, METH_VARARGS},
     {"from_handle", b_from_handle, METH_O},
+    {"from_buffer", b_from_buffer, METH_VARARGS},
 #ifdef MS_WIN32
     {"getwinerror", b_getwinerror, METH_VARARGS},
 #endif
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3206,6 +3206,19 @@
                              ('a2', BChar, 5)],
                    None, -1, -1, SF_PACKED)
 
+def test_from_buffer():
+    import array
+    a = array.array('H', [10000, 20000, 30000])
+    BChar = new_primitive_type("char")
+    BCharP = new_pointer_type(BChar)
+    BCharA = new_array_type(BCharP, None)
+    c = from_buffer(BCharA, a)
+    assert typeof(c) is BCharA
+    assert repr(c) == "<cdata 'char[]' buffer len 6 from 'array.array' object>"
+    p = new_pointer_type(new_primitive_type("unsigned short"))
+    cast(p, c)[1] += 500
+    assert list(a) == [10000, 20500, 30000]
+
 def test_version():
     # this test is here mostly for PyPy
     assert __version__ == "0.8.6"
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -77,6 +77,7 @@
         #
         with self._lock:
             self.BVoidP = self._get_cached_btype(model.voidp_type)
+            self.BCharA = self._get_cached_btype(model.char_array_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
@@ -264,6 +265,16 @@
         """
         return self._backend.buffer(cdata, size)
 
+    def from_buffer(self, python_buffer):
+        """Return a <cdata 'void *'> that points to the data of the
+        given object, which must support the buffer interface.  Note
+        that this is not meant to be used on the built-in types str,
+        unicode, or bytearray (you can build 'char[]' arrays explicitly)
+        but only on objects containing large quantities of raw data
+        in some other format, like 'array.array' or numpy arrays.
+        """
+        return self._backend.from_buffer(self.BCharA, python_buffer)
+
     def callback(self, cdecl, python_callable=None, error=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -235,6 +235,8 @@
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
 
+char_array_type = ArrayType(PrimitiveType('char'), None)
+
 
 class StructOrUnionOrEnum(BaseTypeByIdentity):
     _attrs_ = ('name',)
diff --git a/testing/test_cdata.py b/testing/test_cdata.py
--- a/testing/test_cdata.py
+++ b/testing/test_cdata.py
@@ -19,6 +19,8 @@
         return FakeType("void")
     def new_pointer_type(self, x):
         return FakeType('ptr-to-%r' % (x,))
+    def new_array_type(self, x, y):
+        return FakeType('array-from-%r-len-%r' % (x, y))
     def cast(self, x, y):
         return 'casted!'
     def _get_types(self):
diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py
--- a/testing/test_ffi_backend.py
+++ b/testing/test_ffi_backend.py
@@ -213,3 +213,12 @@
         code, message = ffi.getwinerror(-1)
         assert code == 2
         assert message == "The system cannot find the file specified"
+
+    def test_from_buffer(self):
+        import array
+        ffi = FFI()
+        a = array.array('H', [10000, 20000, 30000])
+        c = ffi.from_buffer(a)
+        assert ffi.typeof(c) is ffi.typeof("char[]")
+        ffi.cast("unsigned short *", c)[1] += 500
+        assert list(a) == [10000, 20500, 30000]

From noreply at buildbot.pypy.org  Sun Dec 28 06:22:04 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 28 Dec 2014 06:22:04 +0100 (CET)
Subject: [pypy-commit] cffi default: add a test
Message-ID: <20141228052204.D0ECC1C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1605:a075a6d57e9e
Date: 2014-12-28 06:22 +0100
http://bitbucket.org/cffi/cffi/changeset/a075a6d57e9e/

Log:	add a test

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3214,6 +3214,7 @@
     BCharA = new_array_type(BCharP, None)
     c = from_buffer(BCharA, a)
     assert typeof(c) is BCharA
+    assert len(c) == 6
     assert repr(c) == "<cdata 'char[]' buffer len 6 from 'array.array' object>"
     p = new_pointer_type(new_primitive_type("unsigned short"))
     cast(p, c)[1] += 500

From noreply at buildbot.pypy.org  Sun Dec 28 06:26:38 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 28 Dec 2014 06:26:38 +0100 (CET)
Subject: [pypy-commit] cffi default: Silence the new gcc warnings when
 comparing 'unsigned >= 0' or 'unsigned < 0'.
Message-ID: <20141228052638.F08C31C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1606:ec7b7d86c440
Date: 2014-12-28 06:26 +0100
http://bitbucket.org/cffi/cffi/changeset/ec7b7d86c440/

Log:	Silence the new gcc warnings when comparing 'unsigned >= 0' or
	'unsigned < 0'.

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -667,14 +667,14 @@
         prnt('static int %s(PyObject *lib)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            if enumvalue < 0:
-                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
+            if enumvalue <= 0:
+                prnt('  if ((%s) > 0 || (long)(%s) != %dL) {' % (
                     enumerator, enumerator, enumvalue))
             else:
-                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
+                prnt('  if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % (
                     enumerator, enumerator, enumvalue))
             prnt('    char buf[64];')
-            prnt('    if ((%s) < 0)' % enumerator)
+            prnt('    if ((%s) <= 0)' % enumerator)
             prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
             prnt('    else')
             prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -428,14 +428,14 @@
         prnt('int %s(char *out_error)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            if enumvalue < 0:
-                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
+            if enumvalue <= 0:
+                prnt('  if ((%s) > 0 || (long)(%s) != %dL) {' % (
                     enumerator, enumerator, enumvalue))
             else:
-                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
+                prnt('  if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % (
                     enumerator, enumerator, enumvalue))
             prnt('    char buf[64];')
-            prnt('    if ((%s) < 0)' % enumerator)
+            prnt('    if ((%s) <= 0)' % enumerator)
             prnt('        sprintf(buf, "%%ld", (long)(%s));' % enumerator)
             prnt('    else')
             prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %

From noreply at buildbot.pypy.org  Sun Dec 28 06:29:04 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 28 Dec 2014 06:29:04 +0100 (CET)
Subject: [pypy-commit] cffi default: Python 3 compat
Message-ID: <20141228052904.A596A1C13AE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1607:689abc6648a5
Date: 2014-12-28 06:29 +0100
http://bitbucket.org/cffi/cffi/changeset/689abc6648a5/

Log:	Python 3 compat

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -5166,7 +5166,7 @@
         return NULL;
     }
 
-    if (PyString_Check(x) || PyUnicode_Check(x) ||
+    if (PyBytes_Check(x) || PyText_Check(x) ||
         PyByteArray_Check(x) /* <= this one here for PyPy compatibility */ ) {
         PyErr_SetString(PyExc_TypeError,
                         "from_buffer() cannot return the address of the "

From noreply at buildbot.pypy.org  Sun Dec 28 17:36:22 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 28 Dec 2014 17:36:22 +0100 (CET)
Subject: [pypy-commit] pypy default: test, fix overflow on abs(complex),
 we have no np.seterr -> RuntimeWarning yet
Message-ID: <20141228163622.624E21D384D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r75132:035393735c13
Date: 2014-12-28 17:36 +0200
http://bitbucket.org/pypy/pypy/changeset/035393735c13/

Log:	test, fix overflow on abs(complex), we have no np.seterr ->
	RuntimeWarning yet

diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -478,6 +478,15 @@
         for i in range(4):
             assert c[i] == max(a[i], b[i])
 
+
+    def test_abs_overflow(self):
+        from numpy import array, absolute, isinf
+        a = array(complex(1.5e308,1.5e308))
+        # Prints a RuntimeWarning, but does not raise
+        b = absolute(a)
+        assert isinf(b)
+        
+
     def test_basic(self):
         import sys
         from numpy import (dtype, add, array, dtype,
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1195,7 +1195,11 @@
 
     @complex_to_real_unary_op
     def abs(self, v):
-        return rcomplex.c_abs(v[0], v[1])
+        try:
+            return rcomplex.c_abs(v[0], v[1])
+        except OverflowError:
+            # warning ...
+            return rfloat.INFINITY
 
     @raw_unary_op
     def isnan(self, v):

From noreply at buildbot.pypy.org  Sun Dec 28 17:36:24 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 28 Dec 2014 17:36:24 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: merge default into branch
Message-ID: <20141228163624.743241D384D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r75133:0e204e610bc9
Date: 2014-12-28 17:37 +0200
http://bitbucket.org/pypy/pypy/changeset/0e204e610bc9/

Log:	merge default into branch

diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -227,7 +227,7 @@
     '<tag key="value" />'
     >>> element.remove(subelement)    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ValueError: list.remove(x): ...
+    ValueError: list.remove(...
     >>> serialize(element) # 6
     '<tag key="value" />'
     >>> element[0:0] = [subelement, subelement, subelement]
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -83,17 +83,16 @@
 
 class __extend__(ast.BoolOp):
 
-    def _accept_jump_if_any_is(self, gen, condition, target):
-        self.values[0].accept_jump_if(gen, condition, target)
-        for i in range(1, len(self.values)):
+    def _accept_jump_if_any_is(self, gen, condition, target, skip_last=0):
+        for i in range(len(self.values) - skip_last):
             self.values[i].accept_jump_if(gen, condition, target)
 
     def accept_jump_if(self, gen, condition, target):
         if condition and self.op == ast.And or \
                 (not condition and self.op == ast.Or):
             end = gen.new_block()
-            self._accept_jump_if_any_is(gen, not condition, end)
-            gen.emit_jump(ops.JUMP_FORWARD, target)
+            self._accept_jump_if_any_is(gen, not condition, end, skip_last=1)
+            self.values[-1].accept_jump_if(gen, condition, target)
             gen.use_next_block(end)
         else:
             self._accept_jump_if_any_is(gen, condition, target)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -62,10 +62,54 @@
 
 eptype("intptr_t",  rffi.INTPTR_T,  ctypeprim.W_CTypePrimitiveSigned)
 eptype("uintptr_t", rffi.UINTPTR_T, ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("ptrdiff_t", rffi.INTPTR_T,  ctypeprim.W_CTypePrimitiveSigned) # <-xxx
 eptype("size_t",    rffi.SIZE_T,    ctypeprim.W_CTypePrimitiveUnsigned)
 eptype("ssize_t",   rffi.SSIZE_T,   ctypeprim.W_CTypePrimitiveSigned)
 
+_WCTSigned = ctypeprim.W_CTypePrimitiveSigned
+_WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned
+
+eptype("ptrdiff_t", getattr(rffi, 'PTRDIFF_T', rffi.INTPTR_T), _WCTSigned)
+eptype("intmax_t",  getattr(rffi, 'INTMAX_T',  rffi.LONGLONG), _WCTSigned)
+eptype("uintmax_t", getattr(rffi, 'UINTMAX_T', rffi.LONGLONG), _WCTUnsign)
+
+if hasattr(rffi, 'INT_LEAST8_T'):
+    eptype("int_least8_t",  rffi.INT_LEAST8_T,  _WCTSigned)
+    eptype("int_least16_t", rffi.INT_LEAST16_T, _WCTSigned)
+    eptype("int_least32_t", rffi.INT_LEAST32_T, _WCTSigned)
+    eptype("int_least64_t", rffi.INT_LEAST64_T, _WCTSigned)
+    eptype("uint_least8_t", rffi.UINT_LEAST8_T,  _WCTUnsign)
+    eptype("uint_least16_t",rffi.UINT_LEAST16_T, _WCTUnsign)
+    eptype("uint_least32_t",rffi.UINT_LEAST32_T, _WCTUnsign)
+    eptype("uint_least64_t",rffi.UINT_LEAST64_T, _WCTUnsign)
+else:
+    eptypesize("int_least8_t",   1, _WCTSigned)
+    eptypesize("uint_least8_t",  1, _WCTUnsign)
+    eptypesize("int_least16_t",  2, _WCTSigned)
+    eptypesize("uint_least16_t", 2, _WCTUnsign)
+    eptypesize("int_least32_t",  4, _WCTSigned)
+    eptypesize("uint_least32_t", 4, _WCTUnsign)
+    eptypesize("int_least64_t",  8, _WCTSigned)
+    eptypesize("uint_least64_t", 8, _WCTUnsign)
+
+if hasattr(rffi, 'INT_FAST8_T'):
+    eptype("int_fast8_t",  rffi.INT_FAST8_T,  _WCTSigned)
+    eptype("int_fast16_t", rffi.INT_FAST16_T, _WCTSigned)
+    eptype("int_fast32_t", rffi.INT_FAST32_T, _WCTSigned)
+    eptype("int_fast64_t", rffi.INT_FAST64_T, _WCTSigned)
+    eptype("uint_fast8_t", rffi.UINT_FAST8_T,  _WCTUnsign)
+    eptype("uint_fast16_t",rffi.UINT_FAST16_T, _WCTUnsign)
+    eptype("uint_fast32_t",rffi.UINT_FAST32_T, _WCTUnsign)
+    eptype("uint_fast64_t",rffi.UINT_FAST64_T, _WCTUnsign)
+else:
+    eptypesize("int_fast8_t",   1, _WCTSigned)
+    eptypesize("uint_fast8_t",  1, _WCTUnsign)
+    eptypesize("int_fast16_t",  2, _WCTSigned)
+    eptypesize("uint_fast16_t", 2, _WCTUnsign)
+    eptypesize("int_fast32_t",  4, _WCTSigned)
+    eptypesize("uint_fast32_t", 4, _WCTUnsign)
+    eptypesize("int_fast64_t",  8, _WCTSigned)
+    eptypesize("uint_fast64_t", 8, _WCTUnsign)
+
 @unwrap_spec(name=str)
 def new_primitive_type(space, name):
     try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2718,7 +2718,16 @@
 def test_nonstandard_integer_types():
     for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t',
                      'uint32_t', 'int64_t', 'uint64_t', 'intptr_t',
-                     'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t']:
+                     'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t',
+                     'int_least8_t',  'uint_least8_t',
+                     'int_least16_t', 'uint_least16_t',
+                     'int_least32_t', 'uint_least32_t',
+                     'int_least64_t', 'uint_least64_t',
+                     'int_fast8_t',  'uint_fast8_t',
+                     'int_fast16_t', 'uint_fast16_t',
+                     'int_fast32_t', 'uint_fast32_t',
+                     'int_fast64_t', 'uint_fast64_t',
+                     'intmax_t', 'uintmax_t']:
         new_primitive_type(typename)    # works
 
 def test_cannot_convert_unicode_to_charp():
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,4 +1,5 @@
 from rpython.rlib.buffer import Buffer
+from rpython.rtyper.lltypesystem import rffi
 
 # XXX not the most efficient implementation
 
@@ -20,3 +21,7 @@
     def setitem(self, index, char):
         ll_buffer = self.datainstance.ll_buffer
         ll_buffer[index] = char
+
+    def get_raw_address(self):
+        ll_buffer = self.datainstance.ll_buffer
+        return rffi.cast(rffi.CCHARP, ll_buffer)
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1144,6 +1144,15 @@
         b[3] = b'x'
         assert b[3] == b'x'
 
+    def test_pypy_raw_address(self):
+        import _rawffi
+        S = _rawffi.Structure((40, 1))
+        s = S(autofree=True)
+        addr = buffer(s)._pypy_raw_address()
+        assert type(addr) is int
+        assert buffer(s)._pypy_raw_address() == addr
+        assert buffer(s, 10)._pypy_raw_address() == addr + 10
+
     def test_union(self):
         import _rawffi
         longsize = _rawffi.sizeof('l')
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -244,6 +244,9 @@
     def getitem(self, index):
         return self.ptr[index]
 
+    def get_raw_address(self):
+        return rffi.cast(rffi.CCHARP, self.ptr)
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -478,6 +478,15 @@
         for i in range(4):
             assert c[i] == max(a[i], b[i])
 
+
+    def test_abs_overflow(self):
+        from numpy import array, absolute, isinf
+        a = array(complex(1.5e308,1.5e308))
+        # Prints a RuntimeWarning, but does not raise
+        b = absolute(a)
+        assert isinf(b)
+        
+
     def test_basic(self):
         import sys
         from numpy import (dtype, add, array, dtype,
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -665,6 +665,7 @@
 
         assert numpy.int64(9223372036854775807) == 9223372036854775807
         assert numpy.int64(9223372036854775807) == 9223372036854775807
+        assert numpy.int64(-9223372036854775807) == -9223372036854775807
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, 9223372036854775808L)
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1195,7 +1195,11 @@
 
     @complex_to_real_unary_op
     def abs(self, v):
-        return rcomplex.c_abs(v[0], v[1])
+        try:
+            return rcomplex.c_abs(v[0], v[1])
+        except OverflowError:
+            # warning ...
+            return rfloat.INFINITY
 
     @raw_unary_op
     def isnan(self, v):
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -1047,6 +1047,9 @@
             space.int_w(w_obj)
         except OperationError, e:
             if e.match(space, space.w_OverflowError):
+                if space.is_true(space.le(w_obj, space.wrap(0))):
+                    return find_binop_result_dtype(space, int64_dtype,
+                                               current_guess)
                 return find_binop_result_dtype(space, uint64_dtype,
                                                current_guess)
             raise
diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -286,7 +286,7 @@
 
             t = thread.start_new_thread(pollster.poll, ())
             try:
-                time.sleep(0.1)
+                time.sleep(0.3)
                 for i in range(5): print '',  # to release GIL untranslated
                 # trigger ufds array reallocation
                 for fd in rfds:
@@ -297,7 +297,7 @@
             finally:
                 # and make the call to poll() from the thread return
                 os.write(w, b'spam')
-                time.sleep(0.1)
+                time.sleep(0.3)
                 for i in range(5): print '',  # to release GIL untranslated
         finally:
             os.close(r)
diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -5,7 +5,7 @@
 from rpython.rlib.objectmodel import compute_hash
 
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
 
@@ -123,6 +123,17 @@
         return space.wrap("<%s for 0x%s, size %d>" %
                           (info, addrstring, self.buf.getlength()))
 
+    def descr_pypy_raw_address(self, space):
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        try:
+            ptr = self.buf.get_raw_address()
+        except ValueError:
+            # report the error using the RPython-level internal repr of self.buf
+            msg = ("cannot find the underlying address of buffer that "
+                   "is internally %r" % (self.buf,))
+            raise OperationError(space.w_ValueError, space.wrap(msg))
+        return space.wrap(rffi.cast(lltype.Signed, ptr))
+
 W_Buffer.typedef = TypeDef(
     "buffer",
     __doc__ = """\
@@ -149,5 +160,6 @@
     __mul__ = interp2app(W_Buffer.descr_mul),
     __rmul__ = interp2app(W_Buffer.descr_mul),
     __repr__ = interp2app(W_Buffer.descr_repr),
+    _pypy_raw_address = interp2app(W_Buffer.descr_pypy_raw_address),
 )
 W_Buffer.typedef.acceptable_as_base_class = False
diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py
--- a/pypy/objspace/std/test/test_bufferobject.py
+++ b/pypy/objspace/std/test/test_bufferobject.py
@@ -197,3 +197,9 @@
         buf = buffer('hello world')
         raises(TypeError, "buf[MyInt(0)]")
         raises(TypeError, "buf[MyInt(0):MyInt(5)]")
+
+    def test_pypy_raw_address_base(self):
+        raises(ValueError, buffer("foobar")._pypy_raw_address)
+        raises(ValueError, buffer(u"foobar")._pypy_raw_address)
+        e = raises(ValueError, buffer(bytearray("foobar"))._pypy_raw_address)
+        assert 'BytearrayBuffer' in str(e.value)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -624,7 +624,6 @@
             except KeyError:
                 pass
             else:
-                assert value.getlevel() != LEVEL_CONSTANT
                 assert cur_value.getlevel() != LEVEL_CONSTANT
                 # replacing with a different box
                 cur_value.copy_from(value)
diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py
--- a/rpython/jit/metainterp/virtualizable.py
+++ b/rpython/jit/metainterp/virtualizable.py
@@ -49,7 +49,13 @@
             ARRAYPTR = fieldType(VTYPE, name)
             ARRAY = deref(ARRAYPTR)
             assert isinstance(ARRAYPTR, lltype.Ptr)
-            assert isinstance(ARRAY, lltype.GcArray)
+            if not isinstance(ARRAY, lltype.GcArray):
+                raise Exception(
+                    "The virtualizable field '%s' is not an array (found %r)."
+                    " It usually means that you must try harder to ensure that"
+                    " the list is not resized at run-time. You can do that by"
+                    " using rpython.rlib.debug.make_sure_not_resized()." %
+                    (name, ARRAY))
             ARRAYITEMTYPES.append(arrayItem(ARRAY))
         self.array_descrs = [cpu.arraydescrof(deref(fieldType(VTYPE, name)))
                              for name in array_fields]
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -115,3 +115,8 @@
             return        # otherwise, adding self.offset might make 'start'
                           # out of bounds
         self.buffer.setslice(self.offset + start, string)
+
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        ptr = self.buffer.get_raw_address()
+        return rffi.ptradd(ptr, self.offset)
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -471,11 +471,21 @@
 except CompilationError:
     pass
 
-_TYPES_ARE_UNSIGNED = set(['size_t', 'uintptr_t'])   # plus "unsigned *"
 if os.name != 'nt':
     TYPES.append('mode_t')
     TYPES.append('pid_t')
     TYPES.append('ssize_t')
+    # the types below are rare enough and not available on Windows
+    TYPES.extend(['ptrdiff_t',
+          'int_least8_t',  'uint_least8_t',
+          'int_least16_t', 'uint_least16_t',
+          'int_least32_t', 'uint_least32_t',
+          'int_least64_t', 'uint_least64_t',
+          'int_fast8_t',  'uint_fast8_t',
+          'int_fast16_t', 'uint_fast16_t',
+          'int_fast32_t', 'uint_fast32_t',
+          'int_fast64_t', 'uint_fast64_t',
+          'intmax_t', 'uintmax_t'])
 else:
     MODE_T = lltype.Signed
     PID_T = lltype.Signed
@@ -489,8 +499,10 @@
         if name.startswith('unsigned'):
             name = 'u' + name[9:]
             signed = False
+        elif name == 'size_t' or name.startswith('uint'):
+            signed = False
         else:
-            signed = (name not in _TYPES_ARE_UNSIGNED)
+            signed = True
         name = name.replace(' ', '')
         names.append(name)
         populatelist.append((name.upper(), c_name, signed))
diff --git a/rpython/rtyper/tool/rfficache.py b/rpython/rtyper/tool/rfficache.py
--- a/rpython/rtyper/tool/rfficache.py
+++ b/rpython/rtyper/tool/rfficache.py
@@ -14,7 +14,7 @@
     from rpython.translator.platform import platform
     includes = ['stdlib.h', 'stdio.h', 'sys/types.h']
     if platform.name != 'msvc':
-        includes += ['inttypes.h']
+        includes += ['inttypes.h', 'stddef.h']
     include_string = "\n".join(["#include <%s>" % i for i in includes])
     c_source = py.code.Source('''
     // includes

From noreply at buildbot.pypy.org  Sun Dec 28 17:36:25 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 28 Dec 2014 17:36:25 +0100 (CET)
Subject: [pypy-commit] pypy ufuncapi: revise headers so pypy/numpy,
 mattip/matplotlib can build and install
Message-ID: <20141228163625.9FE3F1D384D@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: ufuncapi
Changeset: r75134:7c52ddf58e9d
Date: 2014-12-27 23:45 +0200
http://bitbucket.org/pypy/pypy/changeset/7c52ddf58e9d/

Log:	revise headers so pypy/numpy, mattip/matplotlib can build and
	install

diff --git a/pypy/module/cpyext/include/numpy/__multiarray_api.h b/pypy/module/cpyext/include/numpy/__multiarray_api.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/numpy/__multiarray_api.h
@@ -0,0 +1,10 @@
+
+
+typedef struct {
+        PyObject_HEAD
+        npy_bool obval;
+} PyBoolScalarObject;
+
+#define import_array()
+#define PyArray_New _PyArray_New
+
diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h b/pypy/module/cpyext/include/numpy/arrayobject.h
--- a/pypy/module/cpyext/include/numpy/arrayobject.h
+++ b/pypy/module/cpyext/include/numpy/arrayobject.h
@@ -11,6 +11,8 @@
 #endif
 
 #include "old_defines.h"
+#include "npy_common.h"
+#include "__multiarray_api.h"
 
 #define NPY_UNUSED(x) x
 #define PyArray_MAX(a,b) (((a)>(b))?(a):(b))
@@ -22,23 +24,6 @@
 
 PyAPI_DATA(PyTypeObject) PyArray_Type;
 
-#ifndef _NPY_COMMON_H_
-#define _NPY_COMMON_H_
-typedef unsigned char npy_bool;
-typedef unsigned char npy_uint8;
-typedef unsigned short npy_uint16;
-typedef signed short npy_int16;
-typedef signed char npy_int8;
-typedef int npy_int;
-
-typedef long npy_intp;
-#ifndef NPY_INTP_FMT
-#define NPY_INTP_FMT "ld"
-#endif
-#endif
-#ifndef import_array
-#define import_array()
-#endif
 
 #define NPY_MAXDIMS 32
 
diff --git a/pypy/module/cpyext/include/numpy/ndarraytypes.h b/pypy/module/cpyext/include/numpy/ndarraytypes.h
--- a/pypy/module/cpyext/include/numpy/ndarraytypes.h
+++ b/pypy/module/cpyext/include/numpy/ndarraytypes.h
@@ -1,38 +1,12 @@
 #ifndef NDARRAYTYPES_H
 #define NDARRAYTYPES_H
 
-//#include "npy_common.h"
+#include "numpy/npy_common.h"
 //#include "npy_endian.h"
 //#include "npy_cpu.h"
 //#include "utils.h"
 
 //for pypy - numpy has lots of typedefs
-#ifndef _NPY_COMMON_H_
-#define _NPY_COMMON_H_
-typedef Py_intptr_t npy_intp;
-typedef Py_uintptr_t npy_uintp;
-typedef unsigned char npy_bool;
-typedef long npy_int32;
-typedef unsigned long npy_uint32;
-typedef unsigned long npy_ucs4;
-typedef long npy_int64;
-typedef unsigned long npy_uint64;
-#if defined(_MSC_VER)
-        #define NPY_INLINE __inline
-#elif defined(__GNUC__)
-	#if defined(__STRICT_ANSI__)
-		#define NPY_INLINE __inline__
-	#else
-		#define NPY_INLINE inline
-	#endif
-#else
-        #define NPY_INLINE
-#endif
-#ifndef NPY_INTP_FMT
-#define NPY_INTP_FMT "ld"
-#endif
-#endif //_NPY_COMMON_H_
-
 //for pypy - make life easier, less backward support
 #define NPY_1_8_API_VERSION 0x00000008
 #define NPY_NO_DEPRECATED_API NPY_1_8_API_VERSION
diff --git a/pypy/module/cpyext/include/numpy/npy_common.h b/pypy/module/cpyext/include/numpy/npy_common.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/numpy/npy_common.h
@@ -0,0 +1,29 @@
+#ifndef _NPY_COMMON_H_
+#define _NPY_COMMON_H_
+
+typedef Py_intptr_t npy_intp;
+typedef Py_uintptr_t npy_uintp;
+typedef unsigned char npy_bool;
+typedef long npy_int32;
+typedef unsigned long npy_uint32;
+typedef unsigned long npy_ucs4;
+typedef long npy_int64;
+typedef unsigned long npy_uint64;
+typedef unsigned char npy_uint8;
+#if defined(_MSC_VER)
+        #define NPY_INLINE __inline
+#elif defined(__GNUC__)
+	#if defined(__STRICT_ANSI__)
+		#define NPY_INLINE __inline__
+	#else
+		#define NPY_INLINE inline
+	#endif
+#else
+        #define NPY_INLINE
+#endif
+#ifndef NPY_INTP_FMT
+#define NPY_INTP_FMT "ld"
+#endif
+#define NPY_API_VERSION 0x8
+#endif //_NPY_COMMON_H_
+

From noreply at buildbot.pypy.org  Mon Dec 29 09:43:28 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 29 Dec 2014 09:43:28 +0100 (CET)
Subject: [pypy-commit] cffi default: Disossiate the version encoded in the
 verifier module file names
Message-ID: <20141229084328.51D881C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1608:84b177cfda19
Date: 2014-12-29 09:43 +0100
http://bitbucket.org/cffi/cffi/changeset/84b177cfda19/

Log:	Disossiate the version encoded in the verifier module file names and
	the version of CFFI itself

diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -6,3 +6,8 @@
 
 __version__ = "0.8.6"
 __version_info__ = (0, 8, 6)
+
+# The verifier module file names are based on the CRC32 of a string that
+# contains the following version number.  It may be older than __version__
+# if nothing is clearly incompatible.
+__version_verifier_modules__ = "0.8.6"
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -1,5 +1,5 @@
 import sys, os, binascii, shutil
-from . import __version__
+from . import __version_verifier_modules__
 from . import ffiplatform
 
 if sys.version_info >= (3, 3):
@@ -32,8 +32,8 @@
             if tag:
                 raise TypeError("can't specify both 'modulename' and 'tag'")
         else:
-            key = '\x00'.join([sys.version[:3], __version__, preamble,
-                               flattened_kwds] +
+            key = '\x00'.join([sys.version[:3], __version_verifier_modules__,
+                               preamble, flattened_kwds] +
                               ffi._cdefsources)
             if sys.version_info >= (3,):
                 key = key.encode('utf-8')

From noreply at buildbot.pypy.org  Mon Dec 29 09:44:45 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 29 Dec 2014 09:44:45 +0100 (CET)
Subject: [pypy-commit] cffi default: no-op
Message-ID: <20141229084445.5F15C1C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1609:9163607c079e
Date: 2014-12-29 09:45 +0100
http://bitbucket.org/cffi/cffi/changeset/9163607c079e/

Log:	no-op

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3563,8 +3563,8 @@
         sprintf(extra_text, "[]");
         length = -1;
         arraysize = -1;
-        if ((ctptr->ct_flags & CT_CAST_ANYTHING) &&
-                (ctitem->ct_flags & CT_PRIMITIVE_CHAR))
+        if ((ctitem->ct_flags & CT_PRIMITIVE_CHAR) &&
+                ctitem->ct_size == sizeof(char))
             flags |= CT_IS_UNSIZED_CHAR_A;
     }
     else {

From noreply at buildbot.pypy.org  Mon Dec 29 11:26:25 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 29 Dec 2014 11:26:25 +0100 (CET)
Subject: [pypy-commit] pypy slim-down-resumedescr: close to be merged branch
Message-ID: <20141229102625.72AB71C030C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: slim-down-resumedescr
Changeset: r75135:10e9450c90cf
Date: 2014-12-29 12:25 +0200
http://bitbucket.org/pypy/pypy/changeset/10e9450c90cf/

Log:	close to be merged branch


From noreply at buildbot.pypy.org  Mon Dec 29 11:26:27 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 29 Dec 2014 11:26:27 +0100 (CET)
Subject: [pypy-commit] pypy default: Merge slim-down-resumedescr. It seems
 to bring down warmup times
Message-ID: <20141229102627.7F1DF1C030C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75136:dddfb36fd72f
Date: 2014-12-29 12:26 +0200
http://bitbucket.org/pypy/pypy/changeset/dddfb36fd72f/

Log:	Merge slim-down-resumedescr. It seems to bring down warmup times by
	about 5% on small benchmarks

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -901,7 +901,7 @@
             descr = tok.faildescr
             assert isinstance(descr, AbstractFailDescr)
             failure_recovery_pos = block_start + tok.pos_recovery_stub
-            descr._arm_failure_recovery_block = failure_recovery_pos
+            descr.adr_jump_offset = failure_recovery_pos
             relative_offset = tok.pos_recovery_stub - tok.offset
             guard_pos = block_start + tok.offset
             if not tok.is_guard_not_invalidated:
@@ -968,11 +968,11 @@
 
     def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
         b = InstrBuilder(self.cpu.cpuinfo.arch_version)
-        patch_addr = faildescr._arm_failure_recovery_block
+        patch_addr = faildescr.adr_jump_offset
         assert patch_addr != 0
         b.B(bridge_addr)
         b.copy_to_raw_memory(patch_addr)
-        faildescr._arm_failure_recovery_block = 0
+        faildescr.adr_jump_offset = 0
 
     # regalloc support
     def load(self, loc, value):
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -192,8 +192,6 @@
             positions[i] = rffi.cast(rffi.USHORT, position)
         # write down the positions of locs
         guardtok.faildescr.rd_locs = positions
-        # we want the descr to keep alive
-        guardtok.faildescr.rd_loop_token = self.current_clt
         return fail_descr, target
 
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -572,13 +572,13 @@
 
     def patch_pending_failure_recoveries(self, rawstart):
         # after we wrote the assembler to raw memory, set up
-        # tok.faildescr._x86_adr_jump_offset to contain the raw address of
+        # tok.faildescr.adr_jump_offset to contain the raw address of
         # the 4-byte target field in the JMP/Jcond instruction, and patch
         # the field in question to point (initially) to the recovery stub
         clt = self.current_clt
         for tok in self.pending_guard_tokens:
             addr = rawstart + tok.pos_jump_offset
-            tok.faildescr._x86_adr_jump_offset = addr
+            tok.faildescr.adr_jump_offset = addr
             relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4)
             assert rx86.fits_in_32bits(relative_target)
             #
@@ -685,7 +685,7 @@
                                    self.cpu.gc_ll_descr.gcrootmap)
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
-        adr_jump_offset = faildescr._x86_adr_jump_offset
+        adr_jump_offset = faildescr.adr_jump_offset
         assert adr_jump_offset != 0
         offset = adr_new_target - (adr_jump_offset + 4)
         # If the new target fits within a rel32 of the jump, just patch
@@ -705,7 +705,7 @@
             p = rffi.cast(rffi.INTP, adr_jump_offset)
             adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0])
             mc.copy_to_raw_memory(adr_target)
-        faildescr._x86_adr_jump_offset = 0    # means "patched"
+        faildescr.adr_jump_offset = 0    # means "patched"
 
     def fixup_target_tokens(self, rawstart):
         for targettoken in self.target_tokens_currently_compiling:
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1479,15 +1479,15 @@
             assert kind == 'v'
         return lltype.nullptr(rclass.OBJECTPTR.TO)
 
-    def _prepare_resume_from_failure(self, opnum, dont_change_position,
-                                     deadframe):
+    def _prepare_resume_from_failure(self, opnum, deadframe):
         from rpython.jit.metainterp.resoperation import rop
         #
-        if opnum == rop.GUARD_TRUE:
+        if opnum == rop.GUARD_FUTURE_CONDITION:
+            pass
+        elif opnum == rop.GUARD_TRUE:
             # Produced directly by some goto_if_not_xxx() opcode that did not
             # jump, but which must now jump.  The pc is just after the opcode.
-            if not dont_change_position:
-                self.position = self.jitcode.follow_jump(self.position)
+            self.position = self.jitcode.follow_jump(self.position)
         #
         elif opnum == rop.GUARD_FALSE:
             # Produced directly by some goto_if_not_xxx() opcode that jumped,
@@ -1517,8 +1517,7 @@
         elif opnum == rop.GUARD_NO_OVERFLOW:
             # Produced by int_xxx_ovf().  The pc is just after the opcode.
             # We get here because it did not used to overflow, but now it does.
-            if not dont_change_position:
-                return get_llexception(self.cpu, OverflowError())
+            return get_llexception(self.cpu, OverflowError())
         #
         elif opnum == rop.GUARD_OVERFLOW:
             # Produced by int_xxx_ovf().  The pc is just after the opcode.
@@ -1649,13 +1648,9 @@
         resumedescr,
         deadframe,
         all_virtuals)
-    if isinstance(resumedescr, ResumeAtPositionDescr):
-        dont_change_position = True
-    else:
-        dont_change_position = False
 
     current_exc = blackholeinterp._prepare_resume_from_failure(
-        resumedescr.guard_opnum, dont_change_position, deadframe)
+        resumedescr.guard_opnum, deadframe)
 
     _run_forever(blackholeinterp, current_exc)
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -63,14 +63,18 @@
     if metainterp_sd.warmrunnerdesc is not None:    # for tests
         assert original_jitcell_token.generation > 0     # has been registered with memmgr
     wref = weakref.ref(original_jitcell_token)
+    clt = original_jitcell_token.compiled_loop_token
+    clt.loop_token_wref = wref
     for op in loop.operations:
         descr = op.getdescr()
+        # not sure what descr.index is about
         if isinstance(descr, ResumeDescr):
-            descr.wref_original_loop_token = wref   # stick it there
-            n = descr.index
-            if n >= 0:       # we also record the resumedescr number
-                original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
-        elif isinstance(descr, JitCellToken):
+            descr.rd_loop_token = clt   # stick it there
+            #n = descr.index
+            #if n >= 0:       # we also record the resumedescr number
+            #    original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
+        #    pass
+        if isinstance(descr, JitCellToken):
             # for a CALL_ASSEMBLER: record it as a potential jump.
             if descr is not original_jitcell_token:
                 original_jitcell_token.record_jump_to(descr)
@@ -128,8 +132,8 @@
                       [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
 
     try:
-        start_state = optimize_trace(metainterp_sd, part, enable_opts,
-                                     export_state=True)
+        start_state = optimize_trace(metainterp_sd, jitdriver_sd, part,
+                                     enable_opts, export_state=True)
     except InvalidLoop:
         return None
     target_token = part.operations[0].getdescr()
@@ -156,7 +160,7 @@
         jumpargs = part.operations[-1].getarglist()
 
         try:
-            optimize_trace(metainterp_sd, part, enable_opts,
+            optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
                            start_state=start_state, export_state=False)
         except InvalidLoop:
             return None
@@ -209,7 +213,8 @@
     orignial_label = label.clone()
     assert label.getopnum() == rop.LABEL
     try:
-        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
+        optimize_trace(metainterp_sd, jitdriver_sd, part,
+                       jitdriver_sd.warmstate.enable_opts,
                        start_state=start_state, export_state=False)
     except InvalidLoop:
         # Fall back on jumping to preamble
@@ -219,7 +224,7 @@
                           [ResOperation(rop.JUMP, inputargs[:],
                                         None, descr=loop_jitcell_token)]
         try:
-            optimize_trace(metainterp_sd, part,
+            optimize_trace(metainterp_sd, jitdriver_sd, part,
                            jitdriver_sd.warmstate.enable_opts,
                            inline_short_preamble=False, start_state=start_state,
                            export_state=False)
@@ -479,22 +484,31 @@
     return d
 
 class ResumeDescr(AbstractFailDescr):
-    pass
+    _attrs_ = ()
 
 class ResumeGuardDescr(ResumeDescr):
-    # this class also gets the following attributes stored by resume.py code
-    # XXX move all of unused stuff to guard_op, now that we can have
-    #     a separate class, so it does not survive that long
-    rd_snapshot = None
-    rd_frame_info_list = None
+    _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
+               'rd_frame_info_list', 'rd_pendingfields', 'status')
+    
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
     rd_consts = None
     rd_virtuals = None
+    rd_frame_info_list = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
 
     status = r_uint(0)
 
+    def copy_all_attributes_from(self, other):
+        assert isinstance(other, ResumeGuardDescr)
+        self.rd_count = other.rd_count
+        self.rd_consts = other.rd_consts
+        self.rd_frame_info_list = other.rd_frame_info_list
+        self.rd_pendingfields = other.rd_pendingfields
+        self.rd_virtuals = other.rd_virtuals
+        self.rd_numb = other.rd_numb
+        # we don't copy status
+
     ST_BUSY_FLAG    = 0x01     # if set, busy tracing from the guard
     ST_TYPE_MASK    = 0x06     # mask for the type (TY_xxx)
     ST_SHIFT        = 3        # in "status >> ST_SHIFT" is stored:
@@ -509,31 +523,12 @@
     def store_final_boxes(self, guard_op, boxes, metainterp_sd):
         guard_op.setfailargs(boxes)
         self.rd_count = len(boxes)
-        self.guard_opnum = guard_op.getopnum()
         #
         if metainterp_sd.warmrunnerdesc is not None:   # for tests
             jitcounter = metainterp_sd.warmrunnerdesc.jitcounter
             hash = jitcounter.fetch_next_hash()
             self.status = hash & self.ST_SHIFT_MASK
 
-    def make_a_counter_per_value(self, guard_value_op):
-        assert guard_value_op.getopnum() == rop.GUARD_VALUE
-        box = guard_value_op.getarg(0)
-        try:
-            i = guard_value_op.getfailargs().index(box)
-        except ValueError:
-            return     # xxx probably very rare
-        else:
-            if box.type == history.INT:
-                ty = self.TY_INT
-            elif box.type == history.REF:
-                ty = self.TY_REF
-            elif box.type == history.FLOAT:
-                ty = self.TY_FLOAT
-            else:
-                assert 0, box.type
-            self.status = ty | (r_uint(i) << self.ST_SHIFT)
-
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
             self.start_compiling()
@@ -632,32 +627,62 @@
                                self, inputargs, new_loop.operations,
                                new_loop.original_jitcell_token)
 
-    def copy_all_attributes_into(self, res):
-        # XXX a bit ugly to have to list them all here
-        res.rd_snapshot = self.rd_snapshot
-        res.rd_frame_info_list = self.rd_frame_info_list
-        res.rd_numb = self.rd_numb
-        res.rd_consts = self.rd_consts
-        res.rd_virtuals = self.rd_virtuals
-        res.rd_pendingfields = self.rd_pendingfields
-        res.rd_count = self.rd_count
+    def make_a_counter_per_value(self, guard_value_op):
+        assert guard_value_op.getopnum() == rop.GUARD_VALUE
+        box = guard_value_op.getarg(0)
+        try:
+            i = guard_value_op.getfailargs().index(box)
+        except ValueError:
+            return     # xxx probably very rare
+        else:
+            if box.type == history.INT:
+                ty = self.TY_INT
+            elif box.type == history.REF:
+                ty = self.TY_REF
+            elif box.type == history.FLOAT:
+                ty = self.TY_FLOAT
+            else:
+                assert 0, box.type
+            self.status = ty | (r_uint(i) << self.ST_SHIFT)
 
-    def _clone_if_mutable(self):
-        res = ResumeGuardDescr()
-        self.copy_all_attributes_into(res)
-        return res
+class ResumeGuardNonnullDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NONNULL
+
+class ResumeGuardIsnullDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_ISNULL
+
+class ResumeGuardClassDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_CLASS
+
+class ResumeGuardTrueDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_TRUE
+
+class ResumeGuardFalseDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_FALSE
+
+class ResumeGuardNonnullClassDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NONNULL_CLASS
+
+class ResumeGuardExceptionDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_EXCEPTION
+
+class ResumeGuardNoExceptionDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NO_EXCEPTION
+
+class ResumeGuardOverflowDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_OVERFLOW
+
+class ResumeGuardNoOverflowDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NO_OVERFLOW
+
+class ResumeGuardValueDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_VALUE
 
 class ResumeGuardNotInvalidated(ResumeGuardDescr):
-    def _clone_if_mutable(self):
-        res = ResumeGuardNotInvalidated()
-        self.copy_all_attributes_into(res)
-        return res
+    guard_opnum = rop.GUARD_NOT_INVALIDATED
 
 class ResumeAtPositionDescr(ResumeGuardDescr):
-    def _clone_if_mutable(self):
-        res = ResumeAtPositionDescr()
-        self.copy_all_attributes_into(res)
-        return res
+    guard_opnum = rop.GUARD_FUTURE_CONDITION
 
 class AllVirtuals:
     llopaque = True
@@ -678,8 +703,10 @@
 
 
 class ResumeGuardForcedDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NOT_FORCED
 
-    def __init__(self, metainterp_sd, jitdriver_sd):
+    def _init(self, metainterp_sd, jitdriver_sd):
+        # to please the annotator
         self.metainterp_sd = metainterp_sd
         self.jitdriver_sd = jitdriver_sd
 
@@ -740,12 +767,40 @@
         hidden_all_virtuals = obj.hide(metainterp_sd.cpu)
         metainterp_sd.cpu.set_savedata_ref(deadframe, hidden_all_virtuals)
 
-    def _clone_if_mutable(self):
-        res = ResumeGuardForcedDescr(self.metainterp_sd,
-                                     self.jitdriver_sd)
-        self.copy_all_attributes_into(res)
-        return res
-
+def invent_fail_descr_for_op(op, optimizer):
+    opnum = op.getopnum()
+    if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
+        resumedescr = ResumeGuardForcedDescr()
+        resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd)
+    elif opnum == rop.GUARD_NOT_INVALIDATED:
+        resumedescr = ResumeGuardNotInvalidated()
+    elif opnum == rop.GUARD_FUTURE_CONDITION:
+        resumedescr = ResumeAtPositionDescr()
+    elif opnum == rop.GUARD_VALUE:
+        resumedescr = ResumeGuardValueDescr()
+    elif opnum == rop.GUARD_NONNULL:
+        resumedescr = ResumeGuardNonnullDescr()
+    elif opnum == rop.GUARD_ISNULL:
+        resumedescr = ResumeGuardIsnullDescr()
+    elif opnum == rop.GUARD_NONNULL_CLASS:
+        resumedescr = ResumeGuardNonnullClassDescr()
+    elif opnum == rop.GUARD_CLASS:
+        resumedescr = ResumeGuardClassDescr()
+    elif opnum == rop.GUARD_TRUE:
+        resumedescr = ResumeGuardTrueDescr()
+    elif opnum == rop.GUARD_FALSE:
+        resumedescr = ResumeGuardFalseDescr()
+    elif opnum == rop.GUARD_EXCEPTION:
+        resumedescr = ResumeGuardExceptionDescr()
+    elif opnum == rop.GUARD_NO_EXCEPTION:
+        resumedescr = ResumeGuardNoExceptionDescr()
+    elif opnum == rop.GUARD_OVERFLOW:
+        resumedescr = ResumeGuardOverflowDescr()
+    elif opnum == rop.GUARD_NO_OVERFLOW:
+        resumedescr = ResumeGuardNoOverflowDescr()
+    else:
+        assert False
+    return resumedescr
 
 class ResumeFromInterpDescr(ResumeDescr):
     def __init__(self, original_greenkey):
@@ -785,13 +840,15 @@
 
     new_trace.operations = [op.clone() for op in metainterp.history.operations]
     metainterp_sd = metainterp.staticdata
-    state = metainterp.jitdriver_sd.warmstate
+    jitdriver_sd = metainterp.jitdriver_sd
+    state = jitdriver_sd.warmstate
     if isinstance(resumekey, ResumeAtPositionDescr):
         inline_short_preamble = False
     else:
         inline_short_preamble = True
     try:
-        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts,
+        state = optimize_trace(metainterp_sd, jitdriver_sd, new_trace,
+                               state.enable_opts,
                                inline_short_preamble, export_state=True)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -134,14 +134,6 @@
     def repr_of_descr(self):
         return '%r' % (self,)
 
-    def _clone_if_mutable(self):
-        return self
-    def clone_if_mutable(self):
-        clone = self._clone_if_mutable()
-        if not we_are_translated():
-            assert clone.__class__ is self.__class__
-        return clone
-
     def hide(self, cpu):
         descr_ptr = cpu.ts.cast_instance_to_base_ref(self)
         return cpu.ts.cast_to_ref(descr_ptr)
@@ -159,6 +151,8 @@
     index = -1
     final_descr = False
 
+    _attrs_ = ('adr_jump_offset', 'rd_locs', 'rd_loop_token')
+
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         raise NotImplementedError
     def compile_and_attach(self, metainterp, new_loop):
diff --git a/rpython/jit/metainterp/inliner.py b/rpython/jit/metainterp/inliner.py
--- a/rpython/jit/metainterp/inliner.py
+++ b/rpython/jit/metainterp/inliner.py
@@ -1,5 +1,6 @@
 from rpython.jit.metainterp.history import Const
 from rpython.jit.metainterp.resume import Snapshot
+from rpython.jit.metainterp.resoperation import GuardResOp
 
 
 class Inliner(object):
@@ -26,21 +27,16 @@
                 newop.setfailargs([self.inline_arg(a) for a in args])
             else:
                 newop.setfailargs([])
+            assert isinstance(newop, GuardResOp)
+            newop.rd_snapshot = self.inline_snapshot(newop.rd_snapshot)
 
         if newop.result and not ignore_result:
             old_result = newop.result
             newop.result = newop.result.clonebox()
             self.argmap[old_result] = newop.result
 
-        self.inline_descr_inplace(newop.getdescr())
-
         return newop
 
-    def inline_descr_inplace(self, descr):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
-        if isinstance(descr, ResumeGuardDescr):
-            descr.rd_snapshot = self.inline_snapshot(descr.rd_snapshot)
-
     def inline_arg(self, arg):
         if arg is None:
             return None
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -47,7 +47,7 @@
 
     return optimizations, unroll
 
-def optimize_trace(metainterp_sd, loop, enable_opts,
+def optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts,
                    inline_short_preamble=True, start_state=None,
                    export_state=True):
     """Optimize loop.operations to remove internal overheadish operations.
@@ -59,11 +59,13 @@
                                                           loop.operations)
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
-            return optimize_unroll(metainterp_sd, loop, optimizations,
+            return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
+                                   optimizations,
                                    inline_short_preamble, start_state,
                                    export_state)
         else:
-            optimizer = Optimizer(metainterp_sd, loop, optimizations)
+            optimizer = Optimizer(metainterp_sd, jitdriver_sd, loop,
+                                  optimizations)
             optimizer.propagate_all_forward()
     finally:
         debug_stop("jit-optimize")
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -7,7 +7,8 @@
                                                      IntLowerBound, MININT,\
                                                      MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractResOp
+from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+     AbstractResOp, GuardResOp
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.tool.pairtype import extendabletype
 from rpython.rlib.debug import debug_print
@@ -501,8 +502,9 @@
 
 class Optimizer(Optimization):
 
-    def __init__(self, metainterp_sd, loop, optimizations=None):
+    def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations=None):
         self.metainterp_sd = metainterp_sd
+        self.jitdriver_sd = jitdriver_sd
         self.cpu = metainterp_sd.cpu
         self.loop = loop
         self.values = {}
@@ -734,6 +736,11 @@
         while i > 0:
             i -= 1
             if self._newoperations[i] is old_op:
+                # a bit of dance to make sure we copy all the attrs on
+                # an already emitted descr
+                newdescr = new_op.getdescr()
+                olddescr = old_op.getdescr()
+                newdescr.copy_all_attributes_from(olddescr)
                 self._newoperations[i] = new_op
                 break
         else:
@@ -741,9 +748,16 @@
 
     def store_final_boxes_in_guard(self, op, pendingfields):
         assert pendingfields is not None
-        descr = op.getdescr()
+        if op.getdescr() is not None:
+            descr = op.getdescr()
+            assert isinstance(descr, compile.ResumeAtPositionDescr)
+        else:
+            descr = compile.invent_fail_descr_for_op(op, self)
+            op.setdescr(descr)
         assert isinstance(descr, compile.ResumeGuardDescr)
-        modifier = resume.ResumeDataVirtualAdder(descr, self.resumedata_memo)
+        assert isinstance(op, GuardResOp)
+        modifier = resume.ResumeDataVirtualAdder(descr, op,
+                                                 self.resumedata_memo)
         try:
             newboxes = modifier.finish(self, pendingfields)
             if len(newboxes) > self.metainterp_sd.options.failargs_limit:
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -312,15 +312,11 @@
                 if not previous_classbox.same_constant(expected_classbox):
                     r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
                     raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r)
+            descr = compile.ResumeGuardValueDescr()
             op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+                        args = [old_guard_op.getarg(0), op.getarg(1)],
+                        descr = descr)
             self.optimizer.replaces_guard[op] = old_guard_op
-            # hack hack hack.  Change the guard_opnum on
-            # new_guard_op.getdescr() so that when resuming,
-            # the operation is not skipped by pyjitpl.py.
-            descr = op.getdescr()
-            assert isinstance(descr, compile.ResumeGuardDescr)
-            descr.guard_opnum = rop.GUARD_VALUE
             descr.make_a_counter_per_value(op)
             # to be safe
             if isinstance(value, PtrOptValue):
@@ -364,15 +360,11 @@
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
+                descr = compile.ResumeGuardNonnullClassDescr()
                 op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
-                                         args = [old_guard_op.getarg(0), op.getarg(1)])
+                            args = [old_guard_op.getarg(0), op.getarg(1)],
+                            descr=descr)
                 self.optimizer.replaces_guard[op] = old_guard_op
-                # hack hack hack.  Change the guard_opnum on
-                # new_guard_op.getdescr() so that when resuming,
-                # the operation is not skipped by pyjitpl.py.
-                descr = op.getdescr()
-                assert isinstance(descr, compile.ResumeGuardDescr)
-                descr.guard_opnum = rop.GUARD_NONNULL_CLASS
         self.emit_operation(op)
         value.make_constant_class(expectedclassbox, op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -17,16 +17,16 @@
     b0 = BoxInt()
     b1 = BoxInt()
     opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu),
-                                None)
-    fdescr = ResumeGuardDescr()
-    op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr)
+                                None, None)
+    op = ResOperation(rop.GUARD_TRUE, ['dummy'], None)
     # setup rd data
     fi0 = resume.FrameInfo(None, "code0", 11)
-    fdescr.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33)
     snapshot0 = resume.Snapshot(None, [b0])
-    fdescr.rd_snapshot = resume.Snapshot(snapshot0, [b1])
+    op.rd_snapshot = resume.Snapshot(snapshot0, [b1])
+    op.rd_frame_info_list = resume.FrameInfo(fi0, "code1", 33)
     #
     opt.store_final_boxes_in_guard(op, [])
+    fdescr = op.getdescr()
     if op.getfailargs() == [b0, b1]:
         assert list(fdescr.rd_numb.nums)      == [tag(1, TAGBOX)]
         assert list(fdescr.rd_numb.prev.nums) == [tag(0, TAGBOX)]
@@ -97,7 +97,8 @@
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
-        loop = self.parse(ops)
+                
+        loop = self.parse(ops, postprocess=self.postprocess)
         token = JitCellToken()
         loop.operations = [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \
                           loop.operations
@@ -108,42 +109,6 @@
         print '\n'.join([str(o) for o in loop.operations])
         self.assert_equal(loop, expected)
 
-    def setup_method(self, meth=None):
-        class FailDescr(compile.ResumeGuardDescr):
-            oparse = None
-            def _oparser_uses_descr_of_guard(self, oparse, fail_args):
-                # typically called 3 times: once when parsing 'ops',
-                # once when parsing 'preamble', once when parsing 'expected'.
-                self.oparse = oparse
-                self.rd_frame_info_list, self.rd_snapshot = snapshot(fail_args)
-            def _clone_if_mutable(self):
-                assert self is fdescr
-                return fdescr2
-            def __repr__(self):
-                if self is fdescr:
-                    return 'fdescr'
-                if self is fdescr2:
-                    return 'fdescr2'
-                return compile.ResumeGuardDescr.__repr__(self)
-        #
-        def snapshot(fail_args, got=[]):
-            if not got:    # only the first time, i.e. when parsing 'ops'
-                rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-                rd_snapshot = resume.Snapshot(None, fail_args)
-                got.append(rd_frame_info_list)
-                got.append(rd_snapshot)
-            return got
-        #
-        fdescr = instantiate(FailDescr)
-        self.namespace['fdescr'] = fdescr
-        fdescr2 = instantiate(FailDescr)
-        self.namespace['fdescr2'] = fdescr2
-
-    def teardown_method(self, meth):
-        self.namespace.pop('fdescr', None)
-        self.namespace.pop('fdescr2', None)
-
-
 
 class BaseTestOptimizeBasic(BaseTestBasic):
 
@@ -2065,14 +2030,14 @@
     def test_merge_guard_nonnull_guard_class(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         expected = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull_class(p1, ConstClass(node_vtable), descr=fdescr) [i0]
+        guard_nonnull_class(p1, ConstClass(node_vtable)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3, p2)
         """
@@ -2082,14 +2047,14 @@
     def test_merge_guard_nonnull_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_value(p1, ConstPtr(myptr)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         expected = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3, p2)
         """
@@ -2099,7 +2064,7 @@
     def test_merge_guard_nonnull_guard_class_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i2]
         i4 = int_sub(i3, 1)
@@ -2108,7 +2073,7 @@
         """
         expected = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         i4 = int_sub(i3, 1)
         jump(p2, i0, i1, i4, p2)
@@ -2527,7 +2492,7 @@
         reader = ResumeDataFakeReader(fdescr, fail_args,
                                       MyMetaInterp(self.cpu))
         boxes = reader.consume_boxes()
-        self._verify_fail_args(boxes, fdescr.oparse, expectedtext)
+        self._verify_fail_args(boxes, self.oparse, expectedtext)
 
     def test_expand_fail_1(self):
         ops = """
@@ -2538,12 +2503,12 @@
         i4 = getfield_gc(p1, descr=valuedescr)
         #
         i2 = int_add(10, 5)
-        guard_true(i1, descr=fdescr) [i2, i4]
+        guard_true(i1) [i2, i4]
         jump(i1, i4)
         """
         expected = """
         [i1, i3]
-        guard_true(i1, descr=fdescr) [i3]
+        guard_true(i1) [i3]
         jump(1, i3)
         """
         self.optimize_loop(ops, expected)
@@ -2555,12 +2520,12 @@
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i2, descr=valuedescr)
         setfield_gc(p1, p1, descr=nextdescr)
-        guard_true(i1, descr=fdescr) [p1]
+        guard_true(i1) [p1]
         jump(i1, i2)
         """
         expected = """
         [i1, i2]
-        guard_true(i1, descr=fdescr) [i2]
+        guard_true(i1) [i2]
         jump(1, i2)
         """
         self.optimize_loop(ops, expected)
@@ -2577,12 +2542,12 @@
         setfield_gc(p1, p2, descr=nextdescr)
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(p2, p3, descr=nextdescr)
-        guard_true(i1, descr=fdescr) [i3, p1]
+        guard_true(i1) [i3, p1]
         jump(i2, i1, i3, p3)
         """
         expected = """
         [i1, i2, i3, p3]
-        guard_true(i1, descr=fdescr) [i3, i2, p3]
+        guard_true(i1) [i3, i2, p3]
         jump(i2, 1, i3, p3)
         """
         self.optimize_loop(ops, expected)
@@ -2594,7 +2559,6 @@
     def test_expand_fail_4(self):
         for arg in ['p1', 'i2,p1', 'p1,p2', 'p2,p1',
                     'i2,p1,p2', 'i2,p2,p1']:
-            self.setup_method() # humpf
             ops = """
             [i1, i2, i3]
             p1 = new_with_vtable(ConstClass(node_vtable))
@@ -2604,12 +2568,12 @@
             setfield_gc(p1, i2, descr=valuedescr)
             setfield_gc(p1, p2, descr=nextdescr)
             setfield_gc(p2, i2, descr=valuedescr)
-            guard_true(i1, descr=fdescr) [i4, i3, %s]
+            guard_true(i1) [i4, i3, %s]
             jump(i1, i2, i3)
             """
             expected = """
             [i1, i2, i3]
-            guard_true(i1, descr=fdescr) [i3, i2]
+            guard_true(i1) [i3, i2]
             jump(1, i2, i3)
             """
             self.optimize_loop(ops % arg, expected)
@@ -2627,12 +2591,12 @@
         setfield_gc(p1, p2, descr=nextdescr)
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(p2, p1, descr=nextdescr)      # a cycle
-        guard_true(i1, descr=fdescr) [i3, i4, p1, p2]
+        guard_true(i1) [i3, i4, p1, p2]
         jump(i2, i1, i3, i4)
         """
         expected = """
         [i1, i2, i3, i4]
-        guard_true(i1, descr=fdescr) [i3, i4, i2]
+        guard_true(i1) [i3, i4, i2]
         jump(i2, 1, i3, i4)
         """
         self.optimize_loop(ops, expected)
@@ -2644,14 +2608,14 @@
     def test_expand_fail_6(self):
         ops = """
         [p0, i0, i1]
-        guard_true(i0, descr=fdescr) [p0]
+        guard_true(i0) [p0]
         p1 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p1, i1, descr=valuedescr)
         jump(p1, i1, i1)
         """
         expected = """
         [i1b, i0, i1]
-        guard_true(i0, descr=fdescr) [i1b]
+        guard_true(i0) [i1b]
         jump(i1, i1, i1)
         """
         py.test.skip("XXX")
@@ -2667,13 +2631,13 @@
         p1 = new_array(3, descr=arraydescr)
         setarrayitem_gc(p1, 1, i1, descr=arraydescr)
         setarrayitem_gc(p1, 0, 25, descr=arraydescr)
-        guard_true(i1, descr=fdescr) [p1]
+        guard_true(i1) [p1]
         i2 = getarrayitem_gc(p1, 1, descr=arraydescr)
         jump(i2)
         """
         expected = """
         [i1]
-        guard_true(i1, descr=fdescr) [i1]
+        guard_true(i1) [i1]
         jump(1)
         """
         self.optimize_loop(ops, expected)
@@ -2687,14 +2651,14 @@
         p2 = new(descr=ssize)
         setfield_gc(p2, i1, descr=adescr)
         setfield_gc(p2, p1, descr=bdescr)
-        guard_true(i1, descr=fdescr) [p2]
+        guard_true(i1) [p2]
         i3 = getfield_gc(p2, descr=adescr)
         p3 = getfield_gc(p2, descr=bdescr)
         jump(i3, p3)
         """
         expected = """
         [i1, p1]
-        guard_true(i1, descr=fdescr) [i1, p1]
+        guard_true(i1) [i1, p1]
         jump(1, p1)
         """
         self.optimize_loop(ops, expected)
@@ -2711,7 +2675,7 @@
         setfield_gc(p5s, i2, descr=adescr)
         setfield_gc(p5s, p7v, descr=bdescr)
         setarrayitem_gc(p1a, 1, p5s, descr=arraydescr2)
-        guard_true(i1, descr=fdescr) [p1a]
+        guard_true(i1) [p1a]
         p2s = new(descr=ssize)
         p3v = new_with_vtable(ConstClass(node_vtable))
         p4a = new_array(2, descr=arraydescr2)
@@ -2723,7 +2687,7 @@
         """
         expected = """
         [i1, ia, iv, pnull, i2]
-        guard_true(i1, descr=fdescr) [ia, iv, i2]
+        guard_true(i1) [ia, iv, i2]
         jump(1, 1, i2, NULL, i2)
         """
         py.test.skip("XXX")
@@ -2749,14 +2713,14 @@
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        guard_true(i3, descr=fdescr) []
+        guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         jump(p1, i2, i4)
         """
         expected = """
         [p1, i2, i3]
-        guard_true(i3, descr=fdescr) [p1, i2]
+        guard_true(i3) [p1, i2]
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         jump(p1, i2, i4)
@@ -2774,14 +2738,14 @@
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p2, i2, descr=valuedescr)
         setfield_gc(ConstPtr(myptr), p2, descr=nextdescr)
-        guard_true(i3, descr=fdescr) []
+        guard_true(i3) []
         i4 = int_neg(i2)
         setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr)
         jump(i2, i4)
         """
         expected = """
         [i2, i3]
-        guard_true(i3, descr=fdescr) [i2]
+        guard_true(i3) [i2]
         i4 = int_neg(i2)
         setfield_gc(ConstPtr(myptr), NULL, descr=nextdescr)
         jump(i2, i4)
@@ -3096,7 +3060,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=nextdescr)
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr) [p2, p1]
+        guard_not_forced() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=nextdescr)
         jump(p0, i1)
@@ -3111,7 +3075,7 @@
         setfield_gc(p0, p2, descr=nextdescr)
         #
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr) [p2, i1]
+        guard_not_forced() [p2, i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
         p1 = new_with_vtable(ConstClass(node_vtable))
@@ -3142,7 +3106,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=refdescr)
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p2, p1]
+        guard_no_exception() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=refdescr)
         jump(p0, i1)
@@ -3151,7 +3115,7 @@
         [p0, i1]
         p3 = force_token()
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p3, i1, p0]
+        guard_no_exception() [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         jump(p0, i1)
         """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -45,7 +45,7 @@
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
-        loop = self.parse(ops)
+        loop = self.parse(ops, postprocess=self.postprocess)
         if expected != "crash!":
             expected = self.parse(expected)
         if expected_preamble:
@@ -96,41 +96,6 @@
 
 
 class OptimizeOptTest(BaseTestWithUnroll):
-    def setup_method(self, meth=None):
-        class FailDescr(compile.ResumeGuardDescr):
-            oparse = None
-            def _oparser_uses_descr_of_guard(self, oparse, fail_args):
-                # typically called 3 times: once when parsing 'ops',
-                # once when parsing 'preamble', once when parsing 'expected'.
-                self.oparse = oparse
-                self.rd_frame_info_list, self.rd_snapshot = snapshot(fail_args)
-            def _clone_if_mutable(self):
-                assert self is fdescr
-                return fdescr2
-            def __repr__(self):
-                if self is fdescr:
-                    return 'fdescr'
-                if self is fdescr2:
-                    return 'fdescr2'
-                return compile.ResumeGuardDescr.__repr__(self)
-        #
-        def snapshot(fail_args, got=[]):
-            if not got:    # only the first time, i.e. when parsing 'ops'
-                rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-                rd_snapshot = resume.Snapshot(None, fail_args)
-                got.append(rd_frame_info_list)
-                got.append(rd_snapshot)
-            return got
-        #
-        fdescr = instantiate(FailDescr)
-        self.namespace['fdescr'] = fdescr
-        fdescr2 = instantiate(FailDescr)
-        self.namespace['fdescr2'] = fdescr2
-
-    def teardown_method(self, meth):
-        self.namespace.pop('fdescr', None)
-        self.namespace.pop('fdescr2', None)
-
     def test_simple(self):
         ops = """
         []
@@ -3004,20 +2969,20 @@
     def test_merge_guard_nonnull_guard_class(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         preamble = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull_class(p1, ConstClass(node_vtable), descr=fdescr) [i0]
+        guard_nonnull_class(p1, ConstClass(node_vtable)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3)
         """
         expected = """
         [p2, i0, i1, i2]
-        guard_nonnull_class(p2, ConstClass(node_vtable), descr=fdescr2) [i0]
+        guard_nonnull_class(p2, ConstClass(node_vtable)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3)
         """
@@ -3027,20 +2992,20 @@
     def test_merge_guard_nonnull_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_value(p1, ConstPtr(myptr)) [i1]
         jump(p2, i0, i1, i3, p2)
         """
         preamble = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         jump(p2, i0, i1, i3)
         """
         expected = """
         [p2, i0, i1, i2]
-        guard_value(p2, ConstPtr(myptr), descr=fdescr2) [i0]
+        guard_value(p2, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         jump(ConstPtr(myptr), i0, i1, i3)
         """
@@ -3050,7 +3015,7 @@
     def test_merge_guard_nonnull_guard_class_guard_value(self):
         ops = """
         [p1, i0, i1, i2, p2]
-        guard_nonnull(p1, descr=fdescr) [i0]
+        guard_nonnull(p1) [i0]
         i3 = int_add(i1, i2)
         guard_class(p1, ConstClass(node_vtable)) [i2]
         i4 = int_sub(i3, 1)
@@ -3059,14 +3024,14 @@
         """
         preamble = """
         [p1, i0, i1, i2, p2]
-        guard_value(p1, ConstPtr(myptr), descr=fdescr) [i0]
+        guard_value(p1, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         i4 = int_sub(i3, 1)
         jump(p2, i0, i1, i4)
         """
         expected = """
         [p2, i0, i1, i2]
-        guard_value(p2, ConstPtr(myptr), descr=fdescr2) [i0]
+        guard_value(p2, ConstPtr(myptr)) [i0]
         i3 = int_add(i1, i2)
         i4 = int_sub(i3, 1)
         jump(ConstPtr(myptr), i0, i1, i4)
@@ -3860,7 +3825,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=nextdescr)
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr) [p2, p1]
+        guard_not_forced() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=nextdescr)
         jump(p0, i1)
@@ -3875,7 +3840,7 @@
         setfield_gc(p0, p2, descr=nextdescr)
         #
         call_may_force(i1, descr=mayforcevirtdescr)
-        guard_not_forced(descr=fdescr2) [p2, i1]
+        guard_not_forced() [p2, i1]
         #
         setfield_gc(p0, NULL, descr=nextdescr)
         p1 = new_with_vtable(ConstClass(node_vtable))
@@ -3906,7 +3871,7 @@
         p2 = virtual_ref(p1, 2)
         setfield_gc(p0, p2, descr=refdescr)
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p2, p1]
+        guard_no_exception() [p2, p1]
         virtual_ref_finish(p2, p1)
         setfield_gc(p0, NULL, descr=refdescr)
         escape()
@@ -3916,7 +3881,7 @@
         [p0, i1]
         p3 = force_token()
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr) [p3, i1, p0]
+        guard_no_exception() [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         escape()
         jump(p0, i1)
@@ -3925,7 +3890,7 @@
         [p0, i1]
         p3 = force_token()
         call(i1, descr=nonwritedescr)
-        guard_no_exception(descr=fdescr2) [p3, i1, p0]
+        guard_no_exception() [p3, i1, p0]
         setfield_gc(p0, NULL, descr=refdescr)
         escape()
         jump(p0, i1)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -8,12 +8,13 @@
 from rpython.jit.backend.llgraph import runner
 from rpython.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstPtr,
                                          Const, TreeLoop, AbstractDescr,
-                                         JitCellToken, TargetToken)
+                                         JitCellToken, TargetToken,
+    BasicFinalDescr)
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.codewriter.heaptracker import register_known_gctype, adr2int
-from rpython.jit.tool.oparser import parse, pure_parse
+from rpython.jit.tool.oparser import OpParser, pure_parse
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
 from rpython.jit.metainterp.jitprof import EmptyProfiler
@@ -348,42 +349,35 @@
     def store_final_boxes(self, op, boxes, metainterp_sd):
         op.setfailargs(boxes)
     def __eq__(self, other):
-        return type(self) is type(other)      # xxx obscure
-    def clone_if_mutable(self):
-        res = Storage(self.metainterp_sd, self.original_greenkey)
-        self.copy_all_attributes_into(res)
-        return res
+        return True # screw this
+        #return type(self) is type(other)      # xxx obscure
 
 def _sortboxes(boxes):
     _kind2count = {history.INT: 1, history.REF: 2, history.FLOAT: 3}
     return sorted(boxes, key=lambda box: _kind2count[box.type])
 
+final_descr = BasicFinalDescr()
+
 class BaseTest(object):
 
-    def parse(self, s, boxkinds=None, want_fail_descr=True):
-        if want_fail_descr:
-            invent_fail_descr = self.invent_fail_descr
-        else:
-            invent_fail_descr = lambda *args: None
-        return parse(s, self.cpu, self.namespace,
-                     type_system=self.type_system,
-                     boxkinds=boxkinds,
-                     invent_fail_descr=invent_fail_descr)
+    def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None):
+        self.oparse = OpParser(s, self.cpu, self.namespace, 'lltype',
+                               boxkinds,
+                               None, False, postprocess)
+        return self.oparse.parse()
+
+    def postprocess(self, op):
+        if op.is_guard():
+            op.rd_snapshot = resume.Snapshot(None, op.getfailargs())
+            op.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
 
     def add_guard_future_condition(self, res):
         # invent a GUARD_FUTURE_CONDITION to not have to change all tests
         if res.operations[-1].getopnum() == rop.JUMP:
-            guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None, descr=self.invent_fail_descr(None, -1, []))
+            guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None)
+            guard.rd_snapshot = resume.Snapshot(None, [])
             res.operations.insert(-1, guard)
 
-    def invent_fail_descr(self, model, opnum, fail_args):
-        if fail_args is None:
-            return None
-        descr = Storage()
-        descr.rd_frame_info_list = resume.FrameInfo(None, "code", 11)
-        descr.rd_snapshot = resume.Snapshot(None, _sortboxes(fail_args))
-        return descr
-
     def assert_equal(self, optimized, expected, text_right=None):
         from rpython.jit.metainterp.optimizeopt.util import equaloplists
         assert len(optimized.inputargs) == len(expected.inputargs)
@@ -410,7 +404,8 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        return optimize_trace(metainterp_sd, loop, self.enable_opts,
+        return optimize_trace(metainterp_sd, None, loop,
+                              self.enable_opts,
                               start_state=start_state,
                               export_state=export_state)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -15,6 +15,7 @@
 from rpython.jit.metainterp.history import TreeLoop, JitCellToken
 from rpython.jit.metainterp.optimizeopt.test.test_optimizeopt import FakeMetaInterpStaticData
 from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.metainterp import resume
 
 class BaseTestGenerateGuards(BaseTest):
 
@@ -798,14 +799,15 @@
         if hasattr(self, 'callinfocollection'):
             metainterp_sd.callinfocollection = self.callinfocollection
         #
-        optimize_trace(metainterp_sd, bridge, self.enable_opts)
+        optimize_trace(metainterp_sd, None, bridge, self.enable_opts)
 
         
     def optimize_bridge(self, loops, bridge, expected, expected_target='Loop', **boxvalues):
         if isinstance(loops, str):
             loops = (loops, )
-        loops = [self.parse(loop) for loop in loops]
-        bridge = self.parse(bridge)
+        loops = [self.parse(loop, postprocess=self.postprocess)
+                 for loop in loops]
+        bridge = self.parse(bridge, postprocess=self.postprocess)
         self.add_guard_future_condition(bridge)
         for loop in loops:
             loop.preamble = self.unroll_and_optimize(loop)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py
@@ -13,7 +13,7 @@
 
         def optimize_loop(self, ops, expected, expected_preamble=None,
                           call_pure_results=None, expected_short=None):
-            loop = self.parse(ops)
+            loop = self.parse(ops, postprocess=self.postprocess)
             if expected != "crash!":
                 expected = self.parse(expected)
             if expected_preamble:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,6 +1,5 @@
 import sys
 
-from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
 from rpython.jit.metainterp.inliner import Inliner
 from rpython.jit.metainterp.optimize import InvalidLoop
@@ -8,17 +7,18 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization
 from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
         ShortBoxes, BadVirtualState, VirtualStatesCantMatch)
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp
 from rpython.jit.metainterp.resume import Snapshot
+from rpython.jit.metainterp import compile
 from rpython.rlib.debug import debug_print, debug_start, debug_stop
 
 
 # FIXME: Introduce some VirtualOptimizer super class instead
 
-def optimize_unroll(metainterp_sd, loop, optimizations,
+def optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble=True, start_state=None,
                     export_state=True):
-    opt = UnrollOptimizer(metainterp_sd, loop, optimizations)
+    opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, loop, optimizations)
     opt.inline_short_preamble = inline_short_preamble
     return opt.propagate_all_forward(start_state, export_state)
 
@@ -52,8 +52,9 @@
 
     inline_short_preamble = True
 
-    def __init__(self, metainterp_sd, loop, optimizations):
-        self.optimizer = UnrollableOptimizer(metainterp_sd, loop, optimizations)
+    def __init__(self, metainterp_sd, jitdriver_sd, loop, optimizations):
+        self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
+                                             loop, optimizations)
         self.boxes_created_this_iteration = None
 
     def get_virtual_state(self, args):
@@ -560,8 +561,11 @@
 
             for guard in extra_guards:
                 if guard.is_guard():
-                    descr = patchguardop.getdescr().clone_if_mutable()
-                    guard.setdescr(descr)
+                    assert isinstance(patchguardop, GuardResOp)
+                    assert isinstance(guard, GuardResOp)
+                    guard.rd_snapshot = patchguardop.rd_snapshot
+                    guard.rd_frame_info_list = patchguardop.rd_frame_info_list
+                    guard.setdescr(compile.ResumeAtPositionDescr())
                 self.optimizer.send_extra_operation(guard)
 
             try:
@@ -590,8 +594,11 @@
             if newop.is_guard():
                 if not patchguardop:
                     raise InvalidLoop("would like to have short preamble, but it has a guard and there's no guard_future_condition")
-                descr = patchguardop.getdescr().clone_if_mutable()
-                newop.setdescr(descr)
+                assert isinstance(newop, GuardResOp)
+                assert isinstance(patchguardop, GuardResOp)
+                newop.rd_snapshot = patchguardop.rd_snapshot
+                newop.rd_frame_info_list = patchguardop.rd_frame_info_list
+                newop.setdescr(compile.ResumeAtPositionDescr())
             self.optimizer.send_extra_operation(newop)
             if shop.result in assumed_classes:
                 classbox = self.getvalue(newop.result).get_constant_class(self.optimizer.cpu)
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -164,8 +164,6 @@
                 assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
-        if op1.getopnum() not in (rop.JUMP, rop.LABEL):      # xxx obscure
-            assert op1.getdescr() == op2.getdescr()
         if op1.getfailargs() or op2.getfailargs():
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.jitprof import EmptyProfiler
 from rpython.jit.metainterp.logger import Logger
 from rpython.jit.metainterp.optimizeopt.util import args_dict
-from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.resoperation import rop, GuardResOp
 from rpython.rlib import nonconst, rstack
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.rlib.debug import have_debug_prints, make_sure_not_resized
@@ -1874,23 +1874,15 @@
             moreargs = [box] + extraargs
         else:
             moreargs = list(extraargs)
-        if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
-            resumedescr = compile.ResumeGuardForcedDescr(self.staticdata,
-                                                         self.jitdriver_sd)
-        elif opnum == rop.GUARD_NOT_INVALIDATED:
-            resumedescr = compile.ResumeGuardNotInvalidated()
-        elif opnum == rop.GUARD_FUTURE_CONDITION:
-            resumedescr = compile.ResumeAtPositionDescr()
-        else:
-            resumedescr = compile.ResumeGuardDescr()
-        guard_op = self.history.record(opnum, moreargs, None, descr=resumedescr)
-        self.capture_resumedata(resumedescr, resumepc)
+        guard_op = self.history.record(opnum, moreargs, None)
+        assert isinstance(guard_op, GuardResOp)
+        self.capture_resumedata(guard_op, resumepc)
         self.staticdata.profiler.count_ops(opnum, Counters.GUARDS)
         # count
         self.attach_debug_info(guard_op)
         return guard_op
 
-    def capture_resumedata(self, resumedescr, resumepc=-1):
+    def capture_resumedata(self, guard_op, resumepc=-1):
         virtualizable_boxes = None
         if (self.jitdriver_sd.virtualizable_info is not None or
             self.jitdriver_sd.greenfield_info is not None):
@@ -1902,7 +1894,7 @@
             if resumepc >= 0:
                 frame.pc = resumepc
         resume.capture_resumedata(self.framestack, virtualizable_boxes,
-                                  self.virtualref_boxes, resumedescr)
+                                  self.virtualref_boxes, guard_op)
         if self.framestack:
             self.framestack[-1].pc = saved_pc
 
@@ -2147,7 +2139,9 @@
         assert isinstance(key, compile.ResumeGuardDescr)
         # store the resumekey.wref_original_loop_token() on 'self' to make
         # sure that it stays alive as long as this MetaInterp
-        self.resumekey_original_loop_token = key.wref_original_loop_token()
+        self.resumekey_original_loop_token = key.rd_loop_token.loop_token_wref()
+        if self.resumekey_original_loop_token is None:
+            raise compile.giveup() # should be rare
         self.staticdata.try_to_free_some_loops()
         self.initialize_state_from_guard_failure(key, deadframe)
         try:
@@ -2163,13 +2157,8 @@
         self.seen_loop_header_for_jdindex = -1
         if isinstance(key, compile.ResumeAtPositionDescr):
             self.seen_loop_header_for_jdindex = self.jitdriver_sd.index
-            dont_change_position = True
-        else:
-            dont_change_position = False
         try:
-            self.prepare_resume_from_failure(key.guard_opnum,
-                                             dont_change_position,
-                                             deadframe)
+            self.prepare_resume_from_failure(key.guard_opnum, deadframe)
             if self.resumekey_original_loop_token is None:   # very rare case
                 raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
             self.interpret()
@@ -2312,12 +2301,12 @@
             else: assert 0
         self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args)
 
-    def prepare_resume_from_failure(self, opnum, dont_change_position,
-                                    deadframe):
+    def prepare_resume_from_failure(self, opnum, deadframe):
         frame = self.framestack[-1]
-        if opnum == rop.GUARD_TRUE:     # a goto_if_not that jumps only now
-            if not dont_change_position:
-                frame.pc = frame.jitcode.follow_jump(frame.pc)
+        if opnum == rop.GUARD_FUTURE_CONDITION:
+            pass
+        elif opnum == rop.GUARD_TRUE:     # a goto_if_not that jumps only now
+            frame.pc = frame.jitcode.follow_jump(frame.pc)
         elif opnum == rop.GUARD_FALSE:     # a goto_if_not that stops jumping;
             pass                  # or a switch that was in its "default" case
         elif opnum == rop.GUARD_VALUE or opnum == rop.GUARD_CLASS:
@@ -2341,12 +2330,11 @@
             pass # XXX we want to do something special in resume descr,
                  # but not now
         elif opnum == rop.GUARD_NO_OVERFLOW:   # an overflow now detected
-            if not dont_change_position:
-                self.execute_raised(OverflowError(), constant=True)
-                try:
-                    self.finishframe_exception()
-                except ChangeFrame:
-                    pass
+            self.execute_raised(OverflowError(), constant=True)
+            try:
+                self.finishframe_exception()
+            except ChangeFrame:
+                pass
         elif opnum == rop.GUARD_OVERFLOW:      # no longer overflowing
             self.clear_exception()
         else:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -91,8 +91,6 @@
     def clone(self):
         args = self.getarglist()
         descr = self.getdescr()
-        if descr is not None:
-            descr = descr.clone_if_mutable()
         op = ResOperation(self.getopnum(), args[:], self.result, descr)
         if not we_are_translated():
             op.name = self.name
@@ -217,6 +215,9 @@
 
     _fail_args = None
 
+    rd_snapshot = None
+    rd_frame_info_list = None
+
     def getfailargs(self):
         return self._fail_args
 
@@ -225,12 +226,18 @@
 
     def copy_and_change(self, opnum, args=None, result=None, descr=None):
         newop = AbstractResOp.copy_and_change(self, opnum, args, result, descr)
+        assert isinstance(newop, GuardResOp)
         newop.setfailargs(self.getfailargs())
+        newop.rd_snapshot = self.rd_snapshot
+        newop.rd_frame_info_list = self.rd_frame_info_list
         return newop
 
     def clone(self):
         newop = AbstractResOp.clone(self)
+        assert isinstance(newop, GuardResOp)
         newop.setfailargs(self.getfailargs())
+        newop.rd_snapshot = self.rd_snapshot
+        newop.rd_frame_info_list = self.rd_frame_info_list
         return newop
 
 # ============
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -55,7 +55,7 @@
                                          back.get_list_of_active_boxes(True))
 
 def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes,
-                       storage):
+                       snapshot_storage):
     n = len(framestack) - 1
     if virtualizable_boxes is not None:
         boxes = virtualref_boxes + virtualizable_boxes
@@ -66,14 +66,14 @@
         _ensure_parent_resumedata(framestack, n)
         frame_info_list = FrameInfo(top.parent_resumedata_frame_info_list,
                                     top.jitcode, top.pc)
-        storage.rd_frame_info_list = frame_info_list
+        snapshot_storage.rd_frame_info_list = frame_info_list
         snapshot = Snapshot(top.parent_resumedata_snapshot,
                             top.get_list_of_active_boxes(False))
         snapshot = Snapshot(snapshot, boxes)
-        storage.rd_snapshot = snapshot
+        snapshot_storage.rd_snapshot = snapshot
     else:
-        storage.rd_frame_info_list = None
-        storage.rd_snapshot = Snapshot(None, boxes)
+        snapshot_storage.rd_frame_info_list = None
+        snapshot_storage.rd_snapshot = Snapshot(None, boxes)
 
 #
 # The following is equivalent to the RPython-level declaration:
@@ -270,8 +270,9 @@
 
 class ResumeDataVirtualAdder(VirtualVisitor):
 
-    def __init__(self, storage, memo):
+    def __init__(self, storage, snapshot_storage, memo):
         self.storage = storage
+        self.snapshot_storage = snapshot_storage
         self.memo = memo
 
     def make_virtual_info(self, value, fieldnums):
@@ -357,13 +358,14 @@
         storage = self.storage
         # make sure that nobody attached resume data to this guard yet
         assert not storage.rd_numb
-        snapshot = storage.rd_snapshot
+        snapshot = self.snapshot_storage.rd_snapshot
         assert snapshot is not None # is that true?
         numb, liveboxes_from_env, v = self.memo.number(optimizer, snapshot)
         self.liveboxes_from_env = liveboxes_from_env
         self.liveboxes = {}
         storage.rd_numb = numb
-        storage.rd_snapshot = None
+        self.snapshot_storage.rd_snapshot = None
+        storage.rd_frame_info_list = self.snapshot_storage.rd_frame_info_list
 
         # collect liveboxes and virtuals
         n = len(liveboxes_from_env) - v
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -2764,7 +2764,8 @@
             return i
         #
         seen = []
-        def my_optimize_trace(metainterp_sd, loop, enable_opts, *args, **kwds):
+        def my_optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts,
+                              *args, **kwds):
             seen.append('unroll' in enable_opts)
             raise InvalidLoop
         old_optimize_trace = optimizeopt.optimize_trace
diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -10,6 +10,9 @@
 from rpython.jit.metainterp.optimizeopt import ALL_OPTS_DICT
 
 class FakeCPU(object):
+    class Storage:
+        pass
+    
     class tracker:
         pass
 
@@ -18,6 +21,7 @@
         self.seen = []
     def compile_loop(self, inputargs, operations, token, log=True, name='',
                      logger=None):
+        token.compiled_loop_token = self.Storage()
         self.seen.append((inputargs, operations, token))
 
 class FakeLogger(object):
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -123,7 +123,7 @@
     class FakeVirtualValue(AbstractVirtualValue):
         def visitor_dispatch_virtual_type(self, *args):
             return FakeVInfo()
-    modifier = ResumeDataVirtualAdder(None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None)
     v1 = FakeVirtualValue(None, None)
     vinfo1 = modifier.make_virtual_info(v1, [1, 2, 4])
     vinfo2 = modifier.make_virtual_info(v1, [1, 2, 4])
@@ -618,7 +618,7 @@
           FakeFrame("code2", 9, c3, b2)]
     capture_resumedata(fs, None, [], storage)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     metainterp = MyMetaInterp()
 
@@ -1022,7 +1022,7 @@
 def test_register_virtual_fields():
     b1, b2 = BoxInt(), BoxInt()
     vbox = BoxPtr()
-    modifier = ResumeDataVirtualAdder(None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1031,7 +1031,7 @@
                                   b2: UNASSIGNED}
     assert modifier.vfieldboxes == {vbox: [b1, b2]}
 
-    modifier = ResumeDataVirtualAdder(None, None)
+    modifier = ResumeDataVirtualAdder(None, None, None)
     modifier.liveboxes_from_env = {vbox: tag(0, TAGVIRTUAL)}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1061,7 +1061,7 @@
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())    
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     assert storage.rd_snapshot is None
     cpu = MyCPU([])
@@ -1075,14 +1075,14 @@
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**16), ConstInt(-65)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.finish(FakeOptimizer({}))
     assert len(memo.consts) == 2
     assert storage.rd_consts is memo.consts
 
     b1s, b2s, b3s = [ConstInt(sys.maxint), ConstInt(2**17), ConstInt(-65)]
     storage2 = make_storage(b1s, b2s, b3s)
-    modifier2 = ResumeDataVirtualAdder(storage2, memo)
+    modifier2 = ResumeDataVirtualAdder(storage2, storage2, memo)
     modifier2.finish(FakeOptimizer({}))
     assert len(memo.consts) == 3    
     assert storage2.rd_consts is memo.consts
@@ -1137,7 +1137,7 @@
     b1s, b2s, b3s = [BoxInt(1), BoxInt(2), BoxInt(3)]
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     b1_2 = BoxInt()
     class FakeValue(object):
 
@@ -1169,7 +1169,7 @@
     b1s = ConstInt(111)
     storage = make_storage(b1s, b2s, b3s)
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())        
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     liveboxes = modifier.finish(FakeOptimizer({}))
     b2t, b3t = [BoxPtr(demo55o), BoxInt(33)]
     newboxes = _resume_remap(liveboxes, [b2s, b3s], b2t, b3t)
@@ -1190,7 +1190,7 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1273,7 +1273,7 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1321,7 +1321,7 @@
     c1s = ConstInt(111)
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1364,7 +1364,7 @@
     b2s, b4s = [BoxPtr(), BoxPtr()]
     storage = Storage()
     memo = ResumeDataLoopMemo(FakeMetaInterpStaticData())
-    modifier = ResumeDataVirtualAdder(storage, memo)
+    modifier = ResumeDataVirtualAdder(storage, storage, memo)
     modifier.liveboxes_from_env = {}
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
@@ -1403,7 +1403,7 @@
     class Storage(object):
         pass
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, None)
+    modifier = ResumeDataVirtualAdder(storage, storage, None)
     modifier._add_pending_fields([])
     assert not storage.rd_pendingfields
     #
@@ -1411,7 +1411,7 @@
         pass
     field_a = FieldDescr()
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, None)
+    modifier = ResumeDataVirtualAdder(storage, storage, None)
     modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042),
                                    61: rffi.cast(rffi.SHORT, 1061)}
     modifier._add_pending_fields([(field_a, 42, 61, -1)])
@@ -1425,7 +1425,7 @@
     #
     array_a = FieldDescr()
     storage = Storage()
-    modifier = ResumeDataVirtualAdder(storage, None)
+    modifier = ResumeDataVirtualAdder(storage, storage, None)
     modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042),
                                    61: rffi.cast(rffi.SHORT, 1061),
                                    62: rffi.cast(rffi.SHORT, 1062),
@@ -1506,7 +1506,7 @@
     metainterp_sd = FakeMetaInterpStaticData()
     metainterp_sd.options = options
     memo = ResumeDataLoopMemo(metainterp_sd)
-    modifier = ResumeDataVirtualAdder(None, memo)
+    modifier = ResumeDataVirtualAdder(None, None, memo)
 
     for i in range(5):
         assert not modifier._invalidation_needed(5, i)
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -61,9 +61,10 @@
 
     def __init__(self, input, cpu, namespace, type_system, boxkinds,
                  invent_fail_descr=default_fail_descr,
-                 nonstrict=False):
+                 nonstrict=False, postproces=None):
         self.input = input
         self.vars = {}
+        self._postproces = postproces
         self.cpu = cpu
         self._consts = namespace
         self.type_system = type_system
@@ -237,8 +238,6 @@
                     fail_args.append(fail_arg)
             if descr is None and self.invent_fail_descr:
                 descr = self.invent_fail_descr(self.model, opnum, fail_args)
-            if hasattr(descr, '_oparser_uses_descr_of_guard'):
-                descr._oparser_uses_descr_of_guard(self, fail_args)
         else:
             fail_args = None
             if opnum == rop.FINISH:
@@ -250,7 +249,7 @@
 
         return opnum, args, descr, fail_args
 
-    def create_op(self, opnum, args, result, descr):
+    def create_op(self, opnum, args, result, descr, fail_args):
         if opnum == ESCAPE_OP.OPNUM:
             op = ESCAPE_OP(result)
             op.initarglist(args)
@@ -262,7 +261,12 @@
             assert descr is None
             return op
         else:
-            return ResOperation(opnum, args, result, descr)
+            res = ResOperation(opnum, args, result, descr)
+            if fail_args is not None:
+                res.setfailargs(fail_args)
+            if self._postproces:
+                self._postproces(res)
+            return res
 
     def parse_result_op(self, line):
         res, op = line.split("=", 1)
@@ -273,16 +277,12 @@
             raise ParseError("Double assign to var %s in line: %s" % (res, line))
         rvar = self.box_for_var(res)
         self.vars[res] = rvar
-        res = self.create_op(opnum, args, rvar, descr)
-        if fail_args is not None:
-            res.setfailargs(fail_args)
+        res = self.create_op(opnum, args, rvar, descr, fail_args)
         return res
 
     def parse_op_no_result(self, line):
         opnum, args, descr, fail_args = self.parse_op(line)
-        res = self.create_op(opnum, args, None, descr)
-        if fail_args is not None:
-            res.setfailargs(fail_args)
+        res = self.create_op(opnum, args, None, descr, fail_args)
         return res
 
     def parse_next_op(self, line):
@@ -377,11 +377,12 @@
 
 def parse(input, cpu=None, namespace=None, type_system='lltype',
           boxkinds=None, invent_fail_descr=default_fail_descr,
-          no_namespace=False, nonstrict=False, OpParser=OpParser):
+          no_namespace=False, nonstrict=False, OpParser=OpParser,
+          postprocess=None):
     if namespace is None and not no_namespace:
         namespace = {}
     return OpParser(input, cpu, namespace, type_system, boxkinds,
-                    invent_fail_descr, nonstrict).parse()
+                    invent_fail_descr, nonstrict, postprocess).parse()
 
 def pure_parse(*args, **kwds):
     kwds['invent_fail_descr'] = None

From noreply at buildbot.pypy.org  Mon Dec 29 20:30:37 2014
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 29 Dec 2014 20:30:37 +0100 (CET)
Subject: [pypy-commit] pypy default: trivial numpy error message
	compatibility
Message-ID: <20141229193037.DD7AF1C07D2@cobra.cs.uni-duesseldorf.de>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r75137:2d5abc45da13
Date: 2014-12-29 21:30 +0200
http://bitbucket.org/pypy/pypy/changeset/2d5abc45da13/

Log:	trivial numpy error message compatibility

diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -392,7 +392,7 @@
         alignment = space.int_w(space.getitem(w_data, space.wrap(6)))
 
         if (w_names == space.w_None) != (w_fields == space.w_None):
-            raise oefmt(space.w_ValueError, "inconsistent fields and names")
+            raise oefmt(space.w_ValueError, "inconsistent fields and names in Numpy dtype unpickling")
 
         self.byteorder = endian
         self.shape = []
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -1234,7 +1234,8 @@
 
         d = np.dtype(('<f8', 2))
         exc = raises(ValueError, "d.__setstate__((3, '|', None, ('f0', 'f1'), None, 16, 1, 0))")
-        assert exc.value[0] == 'inconsistent fields and names'
+        inconsistent = 'inconsistent fields and names in Numpy dtype unpickling'
+        assert exc.value[0] == inconsistent
         assert d.fields is None
         assert d.shape == (2,)
         assert d.subdtype is not None
@@ -1242,7 +1243,7 @@
 
         d = np.dtype(('<f8', 2))
         exc = raises(ValueError, "d.__setstate__((3, '|', None, None, {'f0': (np.dtype('float64'), 0), 'f1': (np.dtype('float64'), 8)}, 16, 1, 0))")
-        assert exc.value[0] == 'inconsistent fields and names'
+        assert exc.value[0] == inconsistent
         assert d.fields is None
         assert d.shape == (2,)
         assert d.subdtype is not None
@@ -1283,7 +1284,11 @@
         from cPickle import loads, dumps
 
         d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)])
-        assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '|', None, ('x', 'y', 'z', 'value'), {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0), 'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12)}, 20, 1, 0))
+        assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '|', None, 
+                     ('x', 'y', 'z', 'value'),
+                     {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0),
+                      'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12),
+                      }, 20, 1, 0))
 
         new_d = loads(dumps(d))
 

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:28 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:28 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Restore pickle of dict iterators.
Message-ID: <20141229202828.BA5B81C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75138:3a37dfe596eb
Date: 2014-12-19 20:26 +0100
http://bitbucket.org/pypy/pypy/changeset/3a37dfe596eb/

Log:	Restore pickle of dict iterators. (The comment was wrong: we do have
	the original dict, this makes things much easier)

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -565,10 +565,10 @@
         EMPTY = None, None
 
     def next(self):
-        if self.dictimplementation is None:
+        if self.w_dict is None:
             return EMPTY
         space = self.space
-        if self.len != self.dictimplementation.length():
+        if self.len != self.w_dict.length():
             self.len = -1   # Make this error state sticky
             raise oefmt(space.w_RuntimeError,
                         "dictionary changed size during iteration")
@@ -577,7 +577,7 @@
         if self.pos < self.len:
             result = getattr(self, 'next_' + TP + '_entry')()
             self.pos += 1
-            if self.strategy is self.dictimplementation.strategy:
+            if self.strategy is self.w_dict.strategy:
                 return result      # common case
             else:
                 # waaa, obscure case: the strategy changed, but not the
@@ -587,28 +587,28 @@
                 if TP == 'key' or TP == 'value':
                     return result
                 w_key = result[0]
-                w_value = self.dictimplementation.getitem(w_key)
+                w_value = self.w_dict.getitem(w_key)
                 if w_value is None:
                     self.len = -1   # Make this error state sticky
                     raise oefmt(space.w_RuntimeError,
                                 "dictionary changed during iteration")
                 return (w_key, w_value)
         # no more entries
-        self.dictimplementation = None
+        self.w_dict = None
         return EMPTY
     return func_with_new_name(next, 'next_' + TP)
 
 
 class BaseIteratorImplementation(object):
-    def __init__(self, space, strategy, implementation):
+    def __init__(self, space, strategy, w_dict):
         self.space = space
         self.strategy = strategy
-        self.dictimplementation = implementation
-        self.len = implementation.length()
+        self.w_dict = w_dict
+        self.len = w_dict.length()
         self.pos = 0
 
     def length(self):
-        if self.dictimplementation is not None and self.len != -1:
+        if self.w_dict is not None and self.len != -1:
             return self.len - self.pos
         return 0
 
@@ -646,9 +646,9 @@
             'setitem_untyped_%s' % dictimpl.__name__)
 
     class IterClassKeys(BaseKeyIterator):
-        def __init__(self, space, strategy, impl):
-            self.iterator = strategy.getiterkeys(impl)
-            BaseIteratorImplementation.__init__(self, space, strategy, impl)
+        def __init__(self, space, strategy, w_dict):
+            self.iterator = strategy.getiterkeys(w_dict)
+            BaseIteratorImplementation.__init__(self, space, strategy, w_dict)
 
         if override_next_key is not None:
             next_key_entry = override_next_key
@@ -660,9 +660,9 @@
                     return None
 
     class IterClassValues(BaseValueIterator):
-        def __init__(self, space, strategy, impl):
-            self.iterator = strategy.getitervalues(impl)
-            BaseIteratorImplementation.__init__(self, space, strategy, impl)
+        def __init__(self, space, strategy, w_dict):
+            self.iterator = strategy.getitervalues(w_dict)
+            BaseIteratorImplementation.__init__(self, space, strategy, w_dict)
 
         if override_next_value is not None:
             next_value_entry = override_next_value
@@ -674,9 +674,9 @@
                     return None
 
     class IterClassItems(BaseItemIterator):
-        def __init__(self, space, strategy, impl):
-            self.iterator = strategy.getiteritems(impl)
-            BaseIteratorImplementation.__init__(self, space, strategy, impl)
+        def __init__(self, space, strategy, w_dict):
+            self.iterator = strategy.getiteritems(w_dict)
+            BaseIteratorImplementation.__init__(self, space, strategy, w_dict)
 
         if override_next_item is not None:
             next_item_entry = override_next_item
@@ -1167,43 +1167,27 @@
         At unpickling time, we just use that list
         and create an iterator on it.
         This is of course not the standard way.
-
-        XXX to do: remove this __reduce__ method and do
-        a registration with copyreg, instead.
         """
         w_mod    = space.getbuiltinmodule('_pickle_support')
         mod      = space.interp_w(MixedModule, w_mod)
         new_inst = mod.get('dictiter_surrogate_new')
-        w_typeobj = space.type(self)
 
-        raise oefmt(space.w_TypeError,
-                    "can't pickle dictionary-keyiterator objects")
-        # XXXXXX get that working again
+        w_dict = self.iteratorimplementation.w_dict
 
-        # we cannot call __init__ since we don't have the original dict
         if isinstance(self, W_DictMultiIterKeysObject):
-            w_clone = space.allocate_instance(W_DictMultiIterKeysObject,
-                                              w_typeobj)
+            w_clone = W_DictMultiIterKeysObject(space, w_dict.iterkeys())
         elif isinstance(self, W_DictMultiIterValuesObject):
-            w_clone = space.allocate_instance(W_DictMultiIterValuesObject,
-                                              w_typeobj)
+            w_clone = W_DictMultiIterValuesObject(space, w_dict.itervalues())
         elif isinstance(self, W_DictMultiIterItemsObject):
-            w_clone = space.allocate_instance(W_DictMultiIterItemsObject,
-                                              w_typeobj)
+            w_clone = W_DictMultiIterItemsObject(space, w_dict.iteritems())
         else:
             raise oefmt(space.w_TypeError,
                         "unsupported dictiter type '%R' during pickling", self)
-        w_clone.space = space
-        w_clone.content = self.content
-        w_clone.len = self.len
-        w_clone.pos = 0
-        w_clone.setup_iterator()
+
         # spool until we have the same pos
-        while w_clone.pos < self.pos:
-            w_clone.next_entry()
-            w_clone.pos += 1
-        stuff = [w_clone.next_entry() for i in range(w_clone.pos, w_clone.len)]
-        w_res = space.newlist(stuff)
+        for x in xrange(self.iteratorimplementation.pos):
+            w_clone.descr_next(space)
+        w_res = space.call_function(space.w_list, w_clone)
         w_ret = space.newtuple([new_inst, space.newtuple([w_res])])
         return w_ret
 
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -176,7 +176,7 @@
     strategy = self.strategy
     assert isinstance(strategy, KwargsDictStrategy)
     for i in self.iterator:
-        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        keys, values_w = strategy.unerase(self.w_dict.dstorage)
         return _wrapkey(self.space, keys[i]), values_w[i]
     else:
         return None, None
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -939,6 +939,18 @@
         helper(lambda x: x.keys())
         helper(lambda x: x.items())
 
+    def test_pickle(self):
+        d = {1: 1, 2: 2, 3: 3}
+        it = iter(d)
+        first = next(it)
+        reduced = it.__reduce__()
+        rebuild, args = reduced
+        new = rebuild(*args)
+        items = set(new)
+        assert len(items) == 2
+        items.add(first)
+        assert items == set(d)        
+
 class AppTestStrategies(object):
     def setup_class(cls):
         if cls.runappdirect:

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:29 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:29 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix a bit the stack trace formatted by
	FaultHandler
Message-ID: <20141229202829.D55B01C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75139:3d0aa9a34f0a
Date: 2014-12-19 20:30 +0100
http://bitbucket.org/pypy/pypy/changeset/3d0aa9a34f0a/

Log:	Fix a bit the stack trace formatted by FaultHandler

diff --git a/pypy/module/faulthandler/interp_faulthandler.py b/pypy/module/faulthandler/interp_faulthandler.py
--- a/pypy/module/faulthandler/interp_faulthandler.py
+++ b/pypy/module/faulthandler/interp_faulthandler.py
@@ -89,10 +89,10 @@
             code = frame.pycode
             lineno = frame.get_last_lineno()
             if code:
-                os.write(fd, "File \"%s\", line %s in %s\n" % (
+                os.write(fd, "  File \"%s\", line %s in %s\n" % (
                         code.co_filename, lineno, code.co_name))
             else:
-                os.write(fd, "File ???, line %s in ???\n" % (
+                os.write(fd, "  File ???, line %s in ???\n" % (
                         lineno,))
 
             frame = frame.f_backref()

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:31 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:31 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Add faulthandler._fatal_error()
Message-ID: <20141229202831.071B81C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75140:45ae7aa92dd8
Date: 2014-12-21 10:15 +0100
http://bitbucket.org/pypy/pypy/changeset/45ae7aa92dd8/

Log:	Add faulthandler._fatal_error()

diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py
--- a/pypy/module/faulthandler/__init__.py
+++ b/pypy/module/faulthandler/__init__.py
@@ -18,4 +18,5 @@
         '_sigabrt': 'interp_faulthandler.sigabrt',
         '_sigbus': 'interp_faulthandler.sigbus',
         '_sigill': 'interp_faulthandler.sigill',
+        '_fatal_error': 'interp_faulthandler.fatal_error',
     }
diff --git a/pypy/module/faulthandler/interp_faulthandler.py b/pypy/module/faulthandler/interp_faulthandler.py
--- a/pypy/module/faulthandler/interp_faulthandler.py
+++ b/pypy/module/faulthandler/interp_faulthandler.py
@@ -119,3 +119,9 @@
 
 def sigill():
     pypy_faulthandler_sigill()
+
+ at unwrap_spec(msg=str)
+def fatal_error(space, msg):
+    os.write(2, "Fatal Python error: %s\n" % msg);
+    dump_traceback(space, space.wrap(None), space.wrap(True))
+    raise RuntimeError(msg)
diff --git a/pypy/module/faulthandler/test/test_faulthander.py b/pypy/module/faulthandler/test/test_faulthander.py
--- a/pypy/module/faulthandler/test/test_faulthander.py
+++ b/pypy/module/faulthandler/test/test_faulthander.py
@@ -1,3 +1,9 @@
+from pypy.module.faulthandler import interp_faulthandler
+
+class TestFaultHandler:
+    def test_fatal_error(self, space):
+        raises(RuntimeError, interp_faulthandler.fatal_error, space, "Message")
+
 class AppTestFaultHandler:
     spaceconfig = {
         "usemodules": ["faulthandler"]

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:32 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:32 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: tkinter: rename TclObject to Tcl_Obj,
	and expose it
Message-ID: <20141229202832.283571C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75141:778fe234a558
Date: 2014-12-21 10:30 +0100
http://bitbucket.org/pypy/pypy/changeset/778fe234a558/

Log:	tkinter: rename TclObject to Tcl_Obj, and expose it

diff --git a/lib_pypy/_tkinter/__init__.py b/lib_pypy/_tkinter/__init__.py
--- a/lib_pypy/_tkinter/__init__.py
+++ b/lib_pypy/_tkinter/__init__.py
@@ -18,6 +18,7 @@
     raise ImportError("Tk headers and development libraries are required")
 
 from .app import TkApp
+from .tclobj import Tcl_Obj
 
 TK_VERSION = tkffi.string(tklib.get_tk_version()).decode('utf-8')
 TCL_VERSION = tkffi.string(tklib.get_tcl_version()).decode('utf-8')
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -2,7 +2,7 @@
 
 from .tklib import tklib, tkffi
 from . import TclError
-from .tclobj import TclObject, FromObj, FromTclString, AsObj, TypeCache
+from .tclobj import Tcl_Obj, FromObj, FromTclString, AsObj, TypeCache
 
 import contextlib
 import sys
@@ -24,7 +24,7 @@
 
 
 def varname_converter(input):
-    if isinstance(input, TclObject):
+    if isinstance(input, Tcl_Obj):
         return input.string
     return input.encode('utf-8')
 
@@ -339,7 +339,7 @@
             return FromTclString(result)
 
     def split(self, arg):
-        if isinstance(arg, TclObject):
+        if isinstance(arg, Tcl_Obj):
             objc = tkffi.new("int*")
             objv = tkffi.new("Tcl_Obj***")
             status = tklib.Tcl_ListObjGetElements(self.interp, arg._value, objc, objv)
@@ -358,7 +358,7 @@
         return self._split(arg)
 
     def splitlist(self, arg):
-        if isinstance(arg, TclObject):
+        if isinstance(arg, Tcl_Obj):
             objc = tkffi.new("int*")
             objv = tkffi.new("Tcl_Obj***")
             status = tklib.Tcl_ListObjGetElements(self.interp, arg._value, objc, objv)
diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -1,4 +1,4 @@
-# TclObject, conversions with Python objects
+# Tcl_Obj, conversions with Python objects
 
 from .tklib import tklib, tkffi
 
@@ -64,7 +64,7 @@
         buf = tkffi.buffer(tkffi.cast("char*", buf), length*2)[:]
         return buf.decode('utf-16')
 
-    return TclObject(value)
+    return Tcl_Obj(value)
 
 def AsObj(value):
     if isinstance(value, bytes):
@@ -88,13 +88,13 @@
         buf = tkffi.new("char[]", encoded)
         inbuf = tkffi.cast("Tcl_UniChar*", buf)
         return tklib.Tcl_NewUnicodeObj(buf, len(encoded)//2)
-    elif isinstance(value, TclObject):
+    elif isinstance(value, Tcl_Obj):
         tklib.Tcl_IncrRefCount(value._value)
         return value._value
 
     return AsObj(str(value))
 
-class TclObject(object):
+class Tcl_Obj(object):
     def __new__(cls, value):
         self = object.__new__(cls)
         tklib.Tcl_IncrRefCount(value)

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:33 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:33 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Add Tcl_Obj.__eq__, fixes many tests
Message-ID: <20141229202833.529511C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75142:baf22eaf17ee
Date: 2014-12-21 10:33 +0100
http://bitbucket.org/pypy/pypy/changeset/baf22eaf17ee/

Log:	Add Tcl_Obj.__eq__, fixes many tests

diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py
--- a/lib_pypy/_tkinter/tclobj.py
+++ b/lib_pypy/_tkinter/tclobj.py
@@ -110,6 +110,11 @@
             return self._string
         return tkffi.string(tklib.Tcl_GetString(self._value)).decode('utf-8')
 
+    def __eq__(self, other):
+        if not isinstance(other, Tcl_Obj):
+            return NotImplemented
+        return self._value == other._value
+
     @property
     def string(self):
         "the string representation of this object, either as str or bytes"

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:34 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:34 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: tkinter: Fix conversion of varnames
Message-ID: <20141229202834.89B931C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75143:a640fa296140
Date: 2014-12-21 10:38 +0100
http://bitbucket.org/pypy/pypy/changeset/a640fa296140/

Log:	tkinter: Fix conversion of varnames

diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -26,7 +26,11 @@
 def varname_converter(input):
     if isinstance(input, Tcl_Obj):
         return input.string
-    return input.encode('utf-8')
+    if isinstance(input, str):
+        input = input.encode('utf-8')
+    if b'\0' in input:
+        raise ValueError("NUL character in string")
+    return input
 
 
 def Tcl_AppInit(app):

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:35 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:35 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix most tcl tests
Message-ID: <20141229202835.AB88E1C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75144:63b83c822ed6
Date: 2014-12-21 11:16 +0100
http://bitbucket.org/pypy/pypy/changeset/63b83c822ed6/

Log:	Fix most tcl tests

diff --git a/lib-python/3/tkinter/test/test_tkinter/test_variables.py b/lib-python/3/tkinter/test/test_tkinter/test_variables.py
--- a/lib-python/3/tkinter/test/test_tkinter/test_variables.py
+++ b/lib-python/3/tkinter/test/test_tkinter/test_variables.py
@@ -1,4 +1,5 @@
 import unittest
+from test import support
 
 from tkinter import Variable, StringVar, IntVar, DoubleVar, BooleanVar, Tk
 
@@ -42,6 +43,7 @@
         v = Variable(self.root, "sample string", "varname")
         self.assertTrue(self.info_exists("varname"))
         del v
+        support.gc_collect()
         self.assertFalse(self.info_exists("varname"))
 
     def test_dont_unset_not_existing(self):
@@ -49,9 +51,11 @@
         v1 = Variable(self.root, name="name")
         v2 = Variable(self.root, name="name")
         del v1
+        support.gc_collect()
         self.assertFalse(self.info_exists("name"))
         # shouldn't raise exception
         del v2
+        support.gc_collect()
         self.assertFalse(self.info_exists("name"))
 
     def test___eq__(self):
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -359,6 +359,8 @@
             return tuple(result)
         elif isinstance(arg, tuple):
             return self._splitObj(arg)
+        if isinstance(arg, str):
+            arg = arg.encode('utf-8')
         return self._split(arg)
 
     def splitlist(self, arg):
@@ -383,7 +385,7 @@
         if res == tklib.TCL_ERROR:
             self.raiseTclError()
 
-        result = tuple(tkffi.string(argv[0][i]).decode('utf-8')
+        result = tuple(FromTclString(tkffi.string(argv[0][i]))
                        for i in range(argc[0]))
         tklib.Tcl_Free(argv[0])
         return result
@@ -427,13 +429,13 @@
             # Not a list.
             # Could be a quoted string containing funnies, e.g. {"}.
             # Return the string itself.
-            return arg.decode('utf-8')
+            return FromTclString(arg)
 
         try:
             if argc[0] == 0:
                 return ""
             elif argc[0] == 1:
-                return tkffi.string(argv[0][0])
+                return FromTclString(tkffi.string(argv[0][0]))
             else:
                 return tuple(self._split(argv[0][i])
                              for i in range(argc[0]))

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:36 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:36 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: More fixes: tcl functions take strings,
	not bytes
Message-ID: <20141229202836.E16AF1C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75145:ac735d653e72
Date: 2014-12-21 11:22 +0100
http://bitbucket.org/pypy/pypy/changeset/ac735d653e72/

Log:	More fixes: tcl functions take strings, not bytes

diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -495,9 +495,9 @@
     def getdouble(self, s):
         if isinstance(s, float):
             return s
+        if '\x00' in s:
+            raise TypeError
         s = s.encode('utf-8')
-        if b'\x00' in s:
-            raise TypeError
         v = tkffi.new("double*")
         res = tklib.Tcl_GetDouble(self.interp, s, v)
         if res == tklib.TCL_ERROR:
@@ -505,8 +505,9 @@
         return v[0]
 
     def exprboolean(self, s):
-        if b'\x00' in s:
+        if '\x00' in s:
             raise TypeError
+        s = s.encode('utf-8')
         v = tkffi.new("int*")
         res = tklib.Tcl_ExprBoolean(self.interp, s, v)
         if res == tklib.TCL_ERROR:
@@ -514,8 +515,9 @@
         return v[0]
 
     def exprlong(self, s):
-        if b'\x00' in s:
+        if '\x00' in s:
             raise TypeError
+        s = s.encode('utf-8')
         v = tkffi.new("long*")
         res = tklib.Tcl_ExprLong(self.interp, s, v)
         if res == tklib.TCL_ERROR:
@@ -523,8 +525,9 @@
         return v[0]
 
     def exprdouble(self, s):
-        if b'\x00' in s:
+        if '\x00' in s:
             raise TypeError
+        s = s.encode('utf-8')
         v = tkffi.new("double*")
         res = tklib.Tcl_ExprDouble(self.interp, s, v)
         if res == tklib.TCL_ERROR:
@@ -532,12 +535,14 @@
         return v[0]
 
     def exprstring(self, s):
-        if b'\x00' in s:
+        if '\x00' in s:
             raise TypeError
+        s = s.encode('utf-8')
         res = tklib.Tcl_ExprString(self.interp, s)
         if res == tklib.TCL_ERROR:
             self.raiseTclError()
-        return tkffi.string(tklib.Tcl_GetStringResult(self.interp))
+        return FromTclString(tkffi.string(
+            tklib.Tcl_GetStringResult(self.interp)))
 
     def mainloop(self, threshold):
         self._check_tcl_appartment()

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:38 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:38 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix the last tcl.split failure.
Message-ID: <20141229202838.1112B1C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75146:adef746e8660
Date: 2014-12-21 12:01 +0100
http://bitbucket.org/pypy/pypy/changeset/adef746e8660/

Log:	Fix the last tcl.split failure.

diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -409,6 +409,16 @@
                 result[i] = newelem
             if result is not None:
                 return tuple(result)
+        elif isinstance(arg, str):
+            argc = tkffi.new("int*")
+            argv = tkffi.new("char***")
+            list_ = arg.encode('utf-8')
+            res = tklib.Tcl_SplitList(tkffi.NULL, list_, argc, argv)
+            if res != tklib.TCL_OK:
+                return arg
+            tklib.Tcl_Free(argv[0])
+            if argc[0] > 1:
+                return self._split(list_)
         elif isinstance(arg, bytes):
             argc = tkffi.new("int*")
             argv = tkffi.new("char***")

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:39 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:39 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: test_ttk: Progress
Message-ID: <20141229202839.397BF1C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75147:e761d5274b75
Date: 2014-12-21 12:32 +0100
http://bitbucket.org/pypy/pypy/changeset/e761d5274b75/

Log:	test_ttk: Progress

diff --git a/lib-python/3/tkinter/test/test_ttk/test_extensions.py b/lib-python/3/tkinter/test/test_ttk/test_extensions.py
--- a/lib-python/3/tkinter/test/test_ttk/test_extensions.py
+++ b/lib-python/3/tkinter/test/test_ttk/test_extensions.py
@@ -2,7 +2,7 @@
 import unittest
 import tkinter
 from tkinter import ttk
-from test.support import requires, run_unittest
+from test.support import requires, run_unittest, gc_collect
 
 import tkinter.test.support as support
 
@@ -22,6 +22,7 @@
         x = ttk.LabeledScale()
         var = x._variable._name
         x.destroy()
+        gc_collect()
         self.assertRaises(tkinter.TclError, x.tk.globalgetvar, var)
 
         # manually created variable
@@ -34,6 +35,7 @@
         else:
             self.assertEqual(float(x.tk.globalgetvar(name)), myvar.get())
         del myvar
+        gc_collect()
         self.assertRaises(tkinter.TclError, x.tk.globalgetvar, name)
 
         # checking that the tracing callback is properly removed
@@ -173,6 +175,7 @@
     def test_resize(self):
         x = ttk.LabeledScale()
         x.pack(expand=True, fill='both')
+        gc_collect()
         x.wait_visibility()
         x.update()
 
@@ -209,6 +212,7 @@
         optmenu.destroy()
         self.assertEqual(optmenu.tk.globalgetvar(name), var.get())
         del var
+        gc_collect()
         self.assertRaises(tkinter.TclError, optmenu.tk.globalgetvar, name)
 
 
@@ -254,6 +258,7 @@
 
         # check that variable is updated correctly
         optmenu.pack()
+        gc_collect()
         optmenu.wait_visibility()
         optmenu['menu'].invoke(0)
         self.assertEqual(optmenu._variable.get(), items[0])
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -25,7 +25,7 @@
 
 def varname_converter(input):
     if isinstance(input, Tcl_Obj):
-        return input.string
+        input = input.string
     if isinstance(input, str):
         input = input.encode('utf-8')
     if b'\0' in input:

From noreply at buildbot.pypy.org  Mon Dec 29 21:28:40 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:28:40 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Implement the opener parameter of
	io.open()
Message-ID: <20141229202840.593781C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75148:8af0b2050bd1
Date: 2014-12-21 17:33 +0100
http://bitbucket.org/pypy/pypy/changeset/8af0b2050bd1/

Log:	Implement the opener parameter of io.open()

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import wrap_oserror, wrap_oserror2
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.rstring import StringBuilder
 from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC
@@ -131,7 +132,7 @@
         return space.wrap(self)
 
     @unwrap_spec(mode=str, closefd=int)
-    def descr_init(self, space, w_name, mode='r', closefd=True):
+    def descr_init(self, space, w_name, mode='r', closefd=True, w_opener=None):
         if space.isinstance_w(w_name, space.w_float):
             raise OperationError(space.w_TypeError, space.wrap(
                 "integer argument expected, got float"))
@@ -159,7 +160,7 @@
                     # else: pass
                 self.fd = fd
                 self.closefd = bool(closefd)
-            else:
+            elif space.is_none(w_opener):
                 self.closefd = True
                 if not closefd:
                     raise OperationError(space.w_ValueError, space.wrap(
@@ -175,6 +176,17 @@
                                         exception_name='w_IOError')
                 finally:
                     fd_is_own = True
+            else:
+                w_fd = space.call_function(w_opener, w_name, space.wrap(flags))
+                try:
+                    self.fd = space.int_w(w_fd)
+                except OperationError as e:
+                    if not e.match(space, space.w_TypeError):
+                        raise
+                    raise oefmt(space.w_TypeError,
+                                "expected integer from opener")
+                finally:
+                    fd_is_own = True
 
             self._dircheck(space, w_name)
             space.setattr(self, space.wrap("name"), w_name)
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -21,7 +21,7 @@
              encoding="str_or_None", errors="str_or_None",
              newline="str_or_None", closefd=bool)
 def open(space, w_file, mode="r", buffering=-1, encoding=None, errors=None,
-    newline=None, closefd=True):
+         newline=None, closefd=True, w_opener=None):
     from pypy.module._io.interp_bufferedio import (W_BufferedRandom,
         W_BufferedWriter, W_BufferedReader)
 
@@ -87,8 +87,8 @@
             space.wrap("binary mode doesn't take a newline argument")
         )
     w_raw = space.call_function(
-        space.gettypefor(W_FileIO), w_file, space.wrap(rawmode), space.wrap(closefd)
-    )
+        space.gettypefor(W_FileIO), w_file, space.wrap(rawmode),
+        space.wrap(closefd), w_opener)
 
     isatty = space.is_true(space.call_method(w_raw, "isatty"))
     line_buffering = buffering == 1 or (buffering < 0 and isatty)
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -229,6 +229,16 @@
                 assert g.name == f.fileno()
                 assert g.raw.name == f.fileno()
 
+    def test_opener(self):
+        import _io, os
+        with _io.open(self.tmpfile, "w") as f:
+            f.write("egg\n")
+        fd = os.open(self.tmpfile, os.O_RDONLY)
+        def opener(path, flags):
+            return fd
+        with _io.open("non-existent", "r", opener=opener) as f:
+            assert f.read() == "egg\n"
+
     def test_seek_and_tell(self):
         import _io
 

From noreply at buildbot.pypy.org  Mon Dec 29 21:39:51 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 29 Dec 2014 21:39:51 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Rework the OSError initialization,
 so that __init__ and __new__
Message-ID: <20141229203951.BDB291C030C@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75149:ad6333658219
Date: 2014-12-29 21:39 +0100
http://bitbucket.org/pypy/pypy/changeset/ad6333658219/

Log:	Rework the OSError initialization, so that __init__ and __new__ can
	be overridden and accept additional arguments (CPython issue #12555)

diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -92,7 +92,7 @@
     TypeDef, GetSetProperty, interp_attrproperty,
     descr_get_dict, descr_set_dict, descr_del_dict)
 from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.error import OperationError, setup_context
+from pypy.interpreter.error import OperationError, oefmt, setup_context
 from pypy.interpreter.pytraceback import PyTraceback, check_traceback
 from rpython.rlib import rwin32
 
@@ -467,43 +467,97 @@
         W_BaseException.__init__(self, space)
 
     @staticmethod
+    def _use_init(space, w_subtype):
+        # When __init__ is defined in a OSError subclass, we want any
+        # extraneous argument to __new__ to be ignored.  The only reasonable
+        # solution, given __new__ takes a variable number of arguments,
+        # is to defer arg parsing and initialization to __init__.
+        #
+        # But when __new__ is overriden as well, it should call our __new__
+        # with the right arguments.
+        #
+        # (see http://bugs.python.org/issue12555#msg148829 )
+        if space.is_w(w_subtype, space.w_OSError):
+            return False
+        if ((space.getattr(w_subtype, space.wrap('__init__')) !=
+             space.getattr(space.w_OSError, space.wrap('__init__'))) and
+            (space.getattr(w_subtype, space.wrap('__new__')) ==
+             space.getattr(space.w_OSError, space.wrap('__new__')))):
+            return True
+        return False
+
+    @staticmethod
+    def _parse_init_args(space, args_w):
+        if 2 <= len(args_w) <= 3:
+            w_errno = args_w[0]
+            w_strerror = args_w[1]
+            if len(args_w) == 3:
+                w_filename = args_w[2]
+            else:
+                w_filename = None
+            return w_errno, w_strerror, w_filename
+        return None, None, None
+
+    @staticmethod
     def descr_new(space, w_subtype, __args__):
-        args_w, kwds_w = __args__.unpack()  # ignore kwds
-        if space.is_w(w_subtype, space.gettypeobject(W_OSError.typedef)):
-            if len(args_w) > 0:
+        args_w, kwds_w = __args__.unpack()
+        w_errno = None
+        w_strerror = None
+        w_filename = None
+        if not W_OSError._use_init(space, w_subtype):
+            if kwds_w:
+                raise oefmt(space.w_TypeError,
+                            "OSError does not take keyword arguments")
+            (w_errno, w_strerror, w_filename,
+             ) = W_OSError._parse_init_args(space, args_w)
+        if (not space.is_none(w_errno) and
+            space.is_w(w_subtype, space.gettypeobject(W_OSError.typedef))):
+            try:
+                errno = space.int_w(w_errno)
+            except OperationError:
+                pass
+            else:
                 try:
-                    errno = space.int_w(args_w[0])
-                except OperationError:
+                    subclass = ERRNO_MAP[errno]
+                except KeyError:
                     pass
                 else:
-                    try:
-                        subclass = ERRNO_MAP[errno]
-                    except KeyError:
-                        pass
-                    else:
-                        w_subtype = space.gettypeobject(subclass.typedef)
+                    w_subtype = space.gettypeobject(subclass.typedef)
         exc = space.allocate_instance(W_OSError, w_subtype)
         W_OSError.__init__(exc, space)
-        exc.args_w = args_w
+        if not W_OSError._use_init(space, w_subtype):
+            exc._init_error(space, args_w, w_errno, w_strerror, w_filename)
         return space.wrap(exc)
 
-    def descr_init(self, space, args_w):
+    def descr_init(self, space, __args__):
+        args_w, kwds_w = __args__.unpack()
+        if not W_OSError._use_init(space, space.type(self)):
+            # Everything already done in OSError_new
+            return
+        if kwds_w:
+            raise oefmt(space.w_TypeError,
+                        "OSError does not take keyword arguments")
+        (w_errno, w_strerror, w_filename
+         ) = W_OSError._parse_init_args(space, args_w)
+        self._init_error(space, args_w, w_errno, w_strerror, w_filename)
+
+    def _init_error(self, space, args_w, w_errno, w_strerror, w_filename):
         W_BaseException.descr_init(self, space, args_w)
-        if 2 <= len(args_w) <= 3:
-            self.w_errno = args_w[0]
-            self.w_strerror = args_w[1]
-        if len(args_w) == 3:
-            # BlockingIOError's 3rd argument can be the number of
-            # characters written.
+        self.w_errno = w_errno
+        self.w_strerror = w_strerror
+
+        if not space.is_none(w_filename):
             if space.isinstance_w(
                     self, space.gettypeobject(W_BlockingIOError.typedef)):
                 try:
-                    self.written = space.int_w(args_w[2])
+                    self.written = space.int_w(w_filename)
                 except OperationError:
-                    self.w_filename = args_w[2]
+                    self.w_filename = w_filename
             else:
-                self.w_filename = args_w[2]
-            self.args_w = [args_w[0], args_w[1]]
+                self.w_filename = w_filename
+                # filename is removed from the args tuple (for compatibility
+                # purposes, see test_exceptions.py)
+                self.args_w = [w_errno, w_strerror]
 
     # since we rebind args_w, we need special reduce, grump
     def descr_reduce(self, space):
@@ -563,10 +617,10 @@
         self.w_winerror = space.w_None
         W_OSError.__init__(self, space)
 
-    def descr_init(self, space, args_w):
+    def descr_init(self, space, __args__):
         # Set errno to the POSIX errno, and winerror to the Win32
         # error code.
-        W_OSError.descr_init(self, space, args_w)
+        W_OSError.descr_init(self, space, __args__)
         try:
             errno = space.int_w(self.w_errno)
         except OperationError:
diff --git a/pypy/module/exceptions/test/test_exc.py b/pypy/module/exceptions/test/test_exc.py
--- a/pypy/module/exceptions/test/test_exc.py
+++ b/pypy/module/exceptions/test/test_exc.py
@@ -345,3 +345,33 @@
         for errcode in othercodes:
             e = OSError(errcode, "Some message")
             assert type(e) is OSError
+
+    def test_oserror_init_overriden(self):
+        class SubOSErrorWithInit(OSError):
+            def __init__(self, message, bar):
+                self.bar = bar
+                super().__init__(message)
+
+        e = SubOSErrorWithInit("some message", "baz")
+        assert e.bar == "baz"
+        assert e.args == ("some message",)
+
+        e = SubOSErrorWithInit("some message", bar="baz")
+        assert e.bar == "baz"
+        assert e.args == ("some message",)
+
+    def test_oserror_new_overriden(self):
+        class SubOSErrorWithNew(OSError):
+            def __new__(cls, message, baz):
+                self = super().__new__(cls, message)
+                self.baz = baz
+                return self
+
+        e = SubOSErrorWithNew("some message", "baz")
+        assert e.baz == "baz"
+        assert e.args == ("some message",)
+
+        e = SubOSErrorWithNew("some message", baz="baz")
+        assert e.baz == "baz"
+        assert e.args == ("some message",)
+

From noreply at buildbot.pypy.org  Mon Dec 29 22:08:08 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 29 Dec 2014 22:08:08 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Implemented load_verify_locations
 on _ssl.SSLContext
Message-ID: <20141229210808.7AE761C030C@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75150:bd472b0a403c
Date: 2014-12-29 13:08 -0800
http://bitbucket.org/pypy/pypy/changeset/bd472b0a403c/

Log:	Implemented load_verify_locations on _ssl.SSLContext

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1,10 +1,11 @@
 from rpython.rlib import rpoll, rsocket
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.ropenssl import *
+from rpython.rlib.rposix import get_errno, set_errno
 from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 from pypy.module._socket import interp_socket
@@ -924,11 +925,37 @@
                         "CERT_OPTIONAL or CERT_REQUIRED")
         self.check_hostname = check_hostname
 
+    def load_verify_locations_w(self, space, w_cafile=None, w_capath=None):
+        if space.is_none(w_cafile):
+            cafile = None
+        else:
+            cafile = space.str_w(w_cafile)
+        if space.is_none(w_capath):
+            capath = None
+        else:
+            capath = space.str_w(w_capath)
+        if cafile is None and capath is None:
+            raise OperationError(space.w_TypeError, space.wrap(
+                    "cafile and capath cannot be both omitted"))
+        set_errno(0)
+        ret = libssl_SSL_CTX_load_verify_locations(
+            self.ctx, cafile, capath)
+        if ret != 1:
+            errno = get_errno()
+            if errno:
+                libssl_ERR_clear_error()
+                raise wrap_oserror(space, OSError(errno, ''),
+                                   exception_name = 'w_IOError')
+            else:
+                raise _ssl_seterror(space, None, -1)
+
+
 _SSLContext.typedef = TypeDef(
     "_ssl._SSLContext",
     __new__=interp2app(_SSLContext.descr_new),
     _wrap_socket=interp2app(_SSLContext.descr_wrap_socket),
     set_ciphers=interp2app(_SSLContext.descr_set_ciphers),
+    load_verify_locations=interp2app(_SSLContext.load_verify_locations_w),
     set_default_verify_paths=interp2app(_SSLContext.descr_set_default_verify_paths),
 
     options=GetSetProperty(_SSLContext.descr_get_options,
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -1,3 +1,6 @@
+from rpython.tool.udir import udir
+
+
 class AppTestSSL:
     spaceconfig = dict(usemodules=('_ssl', '_socket', 'thread'))
 
@@ -236,3 +239,107 @@
         cls.space.appexec([], """():
             import socket; socket.setdefaulttimeout(1)
             """)
+
+
+class AppTestContext:
+    spaceconfig = dict(usemodules=('_ssl',))
+
+    def setup_class(cls):
+        tmpfile = udir / "tmpfile.pem"
+        tmpfile.write(SSL_CERTIFICATE + SSL_PRIVATE_KEY)
+        cls.w_keycert = cls.space.wrap(str(tmpfile))
+        tmpfile = udir / "key.pem"
+        tmpfile.write(SSL_PRIVATE_KEY)
+        cls.w_key = cls.space.wrap(str(tmpfile))
+        tmpfile = udir / "cert.pem"
+        tmpfile.write(SSL_CERTIFICATE)
+        cls.w_cert = cls.space.wrap(str(tmpfile))
+        tmpfile = udir / "badcert.pem"
+        tmpfile.write(SSL_BADCERT)
+        cls.w_badcert = cls.space.wrap(str(tmpfile))
+        tmpfile = udir / "emptycert.pem"
+        tmpfile.write(SSL_EMPTYCERT)
+        cls.w_emptycert = cls.space.wrap(str(tmpfile))
+
+    def test_load_verify_locations(self):
+        import _ssl
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx.load_verify_locations(self.keycert)
+        ctx.load_verify_locations(cafile=self.keycert, capath=None)
+
+
+SSL_CERTIFICATE = """
+-----BEGIN CERTIFICATE-----
+MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw
+MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH
+Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k
+YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
+gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7
+6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt
+pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw
+FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd
+BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G
+lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1
+CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX
+-----END CERTIFICATE-----
+"""
+
+SSL_PRIVATE_KEY = """
+-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm
+LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0
+ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP
+USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt
+CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq
+SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK
+UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y
+BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ
+ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5
+oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik
+eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F
+0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS
+x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/
+SPIXQuT8RMPDVNQ=
+-----END PRIVATE KEY-----
+"""
+SSL_BADCERT = """
+-----BEGIN RSA PRIVATE KEY-----
+MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L
+opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH
+fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB
+AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU
+D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA
+IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM
+oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0
+ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/
+loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j
+oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA
+z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq
+ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV
+q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU=
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+Just bad cert data
+-----END CERTIFICATE-----
+-----BEGIN RSA PRIVATE KEY-----
+MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L
+opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH
+fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB
+AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU
+D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA
+IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM
+oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0
+ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/
+loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j
+oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA
+z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq
+ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV
+q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU=
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+Just bad cert data
+-----END CERTIFICATE-----
+"""
+SSL_EMPTYCERT = ""

From noreply at buildbot.pypy.org  Mon Dec 29 22:14:07 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon, 29 Dec 2014 22:14:07 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: handle a base_encoding of 'utf-8' on osx
Message-ID: <20141229211407.C53DC1C030C@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.3
Changeset: r75151:24d64c593bda
Date: 2014-12-29 13:12 -0800
http://bitbucket.org/pypy/pypy/changeset/24d64c593bda/

Log:	handle a base_encoding of 'utf-8' on osx

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -168,7 +168,10 @@
         w_import = space.getattr(space.builtin, space.wrap("__import__"))
         space.call_function(w_import, space.wrap("encodings"))
         from pypy.module.sys.interp_encoding import base_encoding
-        space.call_function(w_import, space.wrap("encodings." + base_encoding))
+        # May be 'utf-8'
+        normalized_base = base_encoding.replace("-", "_").lower()
+        space.call_function(w_import, space.wrap("encodings." +
+                                                 normalized_base))
         state.codec_need_encodings = False
         if len(state.codec_search_path) == 0:
             raise OperationError(

From noreply at buildbot.pypy.org  Mon Dec 29 22:43:07 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon, 29 Dec 2014 22:43:07 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: merge py3k
Message-ID: <20141229214307.466401C0455@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.3
Changeset: r75152:c393dc8e0fc4
Date: 2014-12-29 13:15 -0800
http://bitbucket.org/pypy/pypy/changeset/c393dc8e0fc4/

Log:	merge py3k

diff too long, truncating to 2000 out of 15290 lines

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -37,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -60,7 +60,7 @@
                  skip=None):
         self.basename = basename
         self._usemodules = usemodules.split() + [
-            '_socket', 'binascii', 'rctime',
+            '_socket', 'binascii', 'time',
             'select', 'signal', 'faulthandler']
         if not sys.platform == 'win32':
             self._usemodules.extend(['_posixsubprocess', 'fcntl'])
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -1127,6 +1127,8 @@
 
     if term is None:
         term = ffi.NULL
+    elif isinstance(term, str):
+        term = term.encode('utf-8')
     err = ffi.new("int *")
     if lib.setupterm(term, fd, err) == lib.ERR:
         err = err[0]
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -72,7 +72,7 @@
         raise TypeError("expected string")
     res = libc.getgrnam(os.fsencode(name))
     if not res:
-        raise KeyError(name)
+        raise KeyError("'getgrnam(): name not found: %s'" % name)
     return _group_from_gstruct(res)
 
 @builtinify
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -30,7 +30,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_minimal_curses",
     "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -42,7 +42,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct",
     "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -67,19 +67,15 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
@@ -89,9 +85,10 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")],
 }
+if sys.platform == "win32":
+    module_suggests["cpyext"].append(("translation.shared", True))
 
 module_import_dependencies = {
     # no _rawffi if importing rpython.rlib.clibffi raises ImportError
diff --git a/pypy/doc/config/objspace.usemodules.rctime.txt b/pypy/doc/config/objspace.usemodules.rctime.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules.rctime.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Use the 'rctime' module. 
-
-'rctime' is our `rffi`_ based implementation of the builtin 'time' module.
-It supersedes the less complete :config:`objspace.usemodules.time`,
-at least for C-like targets (the C and LLVM backends).
-
-.. _`rffi`: ../rffi.html
diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt
--- a/pypy/doc/config/objspace.usemodules.time.txt
+++ b/pypy/doc/config/objspace.usemodules.time.txt
@@ -1,5 +1,1 @@
 Use the 'time' module. 
-
-Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version
-of the application-level 'time' module, at least for C-like targets (the C
-and LLVM backends).
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -205,23 +205,28 @@
 The above is true both in CPython and in PyPy.  Differences
 can occur about whether a built-in function or method will
 call an overridden method of *another* object than ``self``.
-In PyPy, they are generally always called, whereas not in
-CPython.  For example, in PyPy, ``dict1.update(dict2)``
-considers that ``dict2`` is just a general mapping object, and
-will thus call overridden ``keys()``  and ``__getitem__()``
-methods on it.  So the following code prints ``42`` on PyPy
-but ``foo`` on CPython::
+In PyPy, they are often called in cases where CPython would not.
+Two examples::
 
-    >>>> class D(dict):
-    ....     def __getitem__(self, key):
-    ....         return 42
-    ....
-    >>>>
-    >>>> d1 = {}
-    >>>> d2 = D(a='foo')
-    >>>> d1.update(d2)
-    >>>> print d1['a']
-    42
+    class D(dict):
+        def __getitem__(self, key):
+            return "%r from D" % (key,)
+
+    class A(object):
+        pass
+
+    a = A()
+    a.__dict__ = D()
+    a.foo = "a's own foo"
+    print a.foo
+    # CPython => a's own foo
+    # PyPy => 'foo' from D
+
+    glob = D(foo="base item")
+    loc = {}
+    exec "print foo" in glob, loc
+    # CPython => base item
+    # PyPy => 'foo' from D
 
 
 Mutating classes of objects which are already used as dictionary keys
@@ -292,6 +297,9 @@
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
+Notably missing from the list above are ``str`` and ``unicode``.  If your
+code relies on comparing strings with ``is``, then it might break in PyPy.
+
 
 Miscellaneous
 -------------
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -39,3 +39,15 @@
 .. branch: kill-multimethod
 
 Kill multimethod machinery, all multimethods were removed earlier.
+
+.. branch nditer-external_loop
+
+Implement `external_loop` arguement to numpy's nditer
+
+.. branch kill-rctime
+
+Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
+
+.. branch: ssa-flow
+
+Use SSA form for flow graphs inside build_flow() and part of simplify_graph()
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,15 +1,13 @@
-"""
-Python control flow graph generation and bytecode assembly.
-"""
+"""Python control flow graph generation and bytecode assembly."""
 
-from pypy.interpreter.astcompiler import ast, consts, symtable
-from pypy.interpreter import pycode
+from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import specialize, we_are_translated
+
+from pypy.interpreter.astcompiler import ast, consts, misc, symtable
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pycode import PyCode
 from pypy.tool import stdlib_opcode as ops
 
-from pypy.interpreter.error import OperationError
-from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib import rfloat
-
 
 class Instruction(object):
     """Represents a single opcode."""
@@ -21,14 +19,12 @@
         self.has_jump = False
 
     def size(self):
-        """Return the size of bytes of this instruction when it is encoded."""
+        """Return the size of bytes of this instruction when it is
+        encoded.
+        """
         if self.opcode >= ops.HAVE_ARGUMENT:
-            if self.arg > 0xFFFF:
-                return 6
-            else:
-                return 3
-        else:
-            return 1
+            return (6 if self.arg > 0xFFFF else 3)
+        return 1
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -54,9 +50,9 @@
 class Block(object):
     """A basic control flow block.
 
-    It has one entry point and several possible exit points.  Its instructions
-    may be jumps to other blocks, or if control flow reaches the end of the
-    block, it continues to next_block.
+    It has one entry point and several possible exit points.  Its
+    instructions may be jumps to other blocks, or if control flow
+    reaches the end of the block, it continues to next_block.
     """
 
     def __init__(self):
@@ -71,10 +67,10 @@
             stack.append(nextblock)
 
     def post_order(self):
-        """Return this block and its children in post order.
-        This means that the graph of blocks is first cleaned up to
-        ignore back-edges, thus turning it into a DAG.  Then the DAG
-        is linearized.  For example:
+        """Return this block and its children in post order.  This means
+        that the graph of blocks is first cleaned up to ignore
+        back-edges, thus turning it into a DAG.  Then the DAG is
+        linearized.  For example:
 
                    A --> B -\           =>     [A, D, B, C]
                      \-> D ---> C
@@ -105,7 +101,9 @@
         return resultblocks
 
     def code_size(self):
-        """Return the encoded size of all the instructions in this block."""
+        """Return the encoded size of all the instructions in this
+        block.
+        """
         i = 0
         for instr in self.instructions:
             i += instr.size()
@@ -141,6 +139,7 @@
             i += 1
     return result
 
+
 @specialize.argtype(0)
 def _iter_to_dict(iterable, offset=0):
     result = {}
@@ -302,11 +301,11 @@
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
         last_extended_arg_count = 0
-        # The reason for this loop is extended jumps.  EXTENDED_ARG extends the
-        # bytecode size, so it might invalidate the offsets we've already given.
-        # Thus we have to loop until the number of extended args is stable.  Any
-        # extended jump at all is extremely rare, so performance is not too
-        # concerning.
+        # The reason for this loop is extended jumps.  EXTENDED_ARG
+        # extends the bytecode size, so it might invalidate the offsets
+        # we've already given.  Thus we have to loop until the number of
+        # extended args is stable.  Any extended jump at all is
+        # extremely rare, so performance is not too concerning.
         while True:
             extended_arg_count = 0
             offset = 0
@@ -332,7 +331,8 @@
                                     instr.opcode = ops.JUMP_ABSOLUTE
                                     absolute = True
                                 elif target_op == ops.RETURN_VALUE:
-                                    # Replace JUMP_* to a RETURN into just a RETURN
+                                    # Replace JUMP_* to a RETURN into
+                                    # just a RETURN
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
@@ -347,7 +347,8 @@
                         instr.arg = jump_arg
                         if jump_arg > 0xFFFF:
                             extended_arg_count += 1
-            if extended_arg_count == last_extended_arg_count and not force_redo:
+            if (extended_arg_count == last_extended_arg_count and
+                not force_redo):
                 break
             else:
                 last_extended_arg_count = extended_arg_count
@@ -362,12 +363,14 @@
         while True:
             try:
                 w_key = space.next(w_iter)
-            except OperationError, e:
+            except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise
                 break
             w_index = space.getitem(w_consts, w_key)
-            consts_w[space.int_w(w_index)] = space.getitem(w_key, first)
+            w_constant = space.getitem(w_key, first)
+            w_constant = misc.intern_if_common_string(space, w_constant)
+            consts_w[space.int_w(w_index)] = w_constant
         return consts_w
 
     def _get_code_flags(self):
@@ -435,15 +438,16 @@
                         continue
                     addr = offset - current_off
                     # Python assumes that lineno always increases with
-                    # increasing bytecode address (lnotab is unsigned char).
-                    # Depending on when SET_LINENO instructions are emitted this
-                    # is not always true.  Consider the code:
+                    # increasing bytecode address (lnotab is unsigned
+                    # char).  Depending on when SET_LINENO instructions
+                    # are emitted this is not always true.  Consider the
+                    # code:
                     #     a = (1,
                     #          b)
-                    # In the bytecode stream, the assignment to "a" occurs after
-                    # the loading of "b".  This works with the C Python compiler
-                    # because it only generates a SET_LINENO instruction for the
-                    # assignment.
+                    # In the bytecode stream, the assignment to "a"
+                    # occurs after the loading of "b".  This works with
+                    # the C Python compiler because it only generates a
+                    # SET_LINENO instruction for the assignment.
                     if line or addr:
                         while addr > 255:
                             push(chr(255))
@@ -488,23 +492,23 @@
         # (Only) inherit compilerflags in PyCF_MASK
         flags |= (self.compile_info.flags & consts.PyCF_MASK)
         bytecode = ''.join([block.get_code() for block in blocks])
-        return pycode.PyCode(self.space,
-                             self.argcount,
-                             self.kwonlyargcount,
-                             len(self.var_names),
-                             stack_depth,
-                             flags,
-                             bytecode,
-                             list(consts_w),
-                             names,
-                             var_names,
-                             self.compile_info.filename,
-                             self.name,
-                             self.first_lineno,
-                             lnotab,
-                             free_names,
-                             cell_names,
-                             self.compile_info.hidden_applevel)
+        return PyCode(self.space,
+                      self.argcount,
+                      self.kwonlyargcount,
+                      len(self.var_names),
+                      stack_depth,
+                      flags,
+                      bytecode,
+                      list(consts_w),
+                      names,
+                      var_names,
+                      self.compile_info.filename,
+                      self.name,
+                      self.first_lineno,
+                      lnotab,
+                      free_names,
+                      cell_names,
+                      self.compile_info.hidden_applevel)
 
 
 def _list_from_dict(d, offset=0):
@@ -515,116 +519,116 @@
 
 
 _static_opcode_stack_effects = {
-    ops.NOP : 0,
+    ops.NOP: 0,
 
-    ops.POP_TOP : -1,
-    ops.ROT_TWO : 0,
-    ops.ROT_THREE : 0,
-    ops.DUP_TOP : 1,
-    ops.DUP_TOP_TWO : 2,
+    ops.POP_TOP: -1,
+    ops.ROT_TWO: 0,
+    ops.ROT_THREE: 0,
+    ops.DUP_TOP: 1,
+    ops.DUP_TOP_TWO: 2,
 
-    ops.UNARY_POSITIVE : 0,
-    ops.UNARY_NEGATIVE : 0,
-    ops.UNARY_NOT : 0,
-    ops.UNARY_INVERT : 0,
+    ops.UNARY_POSITIVE: 0,
+    ops.UNARY_NEGATIVE: 0,
+    ops.UNARY_NOT: 0,
+    ops.UNARY_INVERT: 0,
 
-    ops.LIST_APPEND : -1,
-    ops.SET_ADD : -1,
-    ops.MAP_ADD : -2,
-    ops.STORE_MAP : -2,
+    ops.LIST_APPEND: -1,
+    ops.SET_ADD: -1,
+    ops.MAP_ADD: -2,
+    ops.STORE_MAP: -2,
 
-    ops.BINARY_POWER : -1,
-    ops.BINARY_MULTIPLY : -1,
-    ops.BINARY_MODULO : -1,
-    ops.BINARY_ADD : -1,
-    ops.BINARY_SUBTRACT : -1,
-    ops.BINARY_SUBSCR : -1,
-    ops.BINARY_FLOOR_DIVIDE : -1,
-    ops.BINARY_TRUE_DIVIDE : -1,
-    ops.BINARY_LSHIFT : -1,
-    ops.BINARY_RSHIFT : -1,
-    ops.BINARY_AND : -1,
-    ops.BINARY_OR : -1,
-    ops.BINARY_XOR : -1,
+    ops.BINARY_POWER: -1,
+    ops.BINARY_MULTIPLY: -1,
+    ops.BINARY_MODULO: -1,
+    ops.BINARY_ADD: -1,
+    ops.BINARY_SUBTRACT: -1,
+    ops.BINARY_SUBSCR: -1,
+    ops.BINARY_FLOOR_DIVIDE: -1,
+    ops.BINARY_TRUE_DIVIDE: -1,
+    ops.BINARY_LSHIFT: -1,
+    ops.BINARY_RSHIFT: -1,
+    ops.BINARY_AND: -1,
+    ops.BINARY_OR: -1,
+    ops.BINARY_XOR: -1,
 
-    ops.INPLACE_FLOOR_DIVIDE : -1,
-    ops.INPLACE_TRUE_DIVIDE : -1,
-    ops.INPLACE_ADD : -1,
-    ops.INPLACE_SUBTRACT : -1,
-    ops.INPLACE_MULTIPLY : -1,
-    ops.INPLACE_MODULO : -1,
-    ops.INPLACE_POWER : -1,
-    ops.INPLACE_LSHIFT : -1,
-    ops.INPLACE_RSHIFT : -1,
-    ops.INPLACE_AND : -1,
-    ops.INPLACE_OR : -1,
-    ops.INPLACE_XOR : -1,
+    ops.INPLACE_FLOOR_DIVIDE: -1,
+    ops.INPLACE_TRUE_DIVIDE: -1,
+    ops.INPLACE_ADD: -1,
+    ops.INPLACE_SUBTRACT: -1,
+    ops.INPLACE_MULTIPLY: -1,
+    ops.INPLACE_MODULO: -1,
+    ops.INPLACE_POWER: -1,
+    ops.INPLACE_LSHIFT: -1,
+    ops.INPLACE_RSHIFT: -1,
+    ops.INPLACE_AND: -1,
+    ops.INPLACE_OR: -1,
+    ops.INPLACE_XOR: -1,
 
-    ops.STORE_SUBSCR : -2,
-    ops.DELETE_SUBSCR : -2,
+    ops.STORE_SUBSCR: -2,
+    ops.DELETE_SUBSCR: -2,
 
-    ops.GET_ITER : 0,
-    ops.FOR_ITER : 1,
-    ops.BREAK_LOOP : 0,
-    ops.CONTINUE_LOOP : 0,
-    ops.SETUP_LOOP : 0,
+    ops.GET_ITER: 0,
+    ops.FOR_ITER: 1,
+    ops.BREAK_LOOP: 0,
+    ops.CONTINUE_LOOP: 0,
+    ops.SETUP_LOOP: 0,
 
-    ops.PRINT_EXPR : -1,
+    ops.PRINT_EXPR: -1,
 
-    ops.WITH_CLEANUP : -1,
-    ops.LOAD_BUILD_CLASS : 1,
-    ops.STORE_LOCALS : -1,
-    ops.POP_BLOCK : 0,
-    ops.POP_EXCEPT : 0,
-    ops.END_FINALLY : -1,
-    ops.SETUP_WITH : 1,
-    ops.SETUP_FINALLY : 0,
-    ops.SETUP_EXCEPT : 4,
+    ops.WITH_CLEANUP: -1,
+    ops.LOAD_BUILD_CLASS: 1,
+    ops.STORE_LOCALS: -1,
+    ops.POP_BLOCK: 0,
+    ops.POP_EXCEPT: 0,
+    ops.END_FINALLY: -1,
+    ops.SETUP_WITH: 1,
+    ops.SETUP_FINALLY: 0,
+    ops.SETUP_EXCEPT: 4,
 
-    ops.RETURN_VALUE : -1,
-    ops.YIELD_VALUE : 0,
-    ops.YIELD_FROM : -1,
-    ops.BUILD_MAP : 1,
-    ops.BUILD_SET : 1,
-    ops.COMPARE_OP : -1,
+    ops.RETURN_VALUE: -1,
+    ops.YIELD_VALUE: 0,
+    ops.YIELD_FROM: -1,
+    ops.BUILD_MAP: 1,
+    ops.BUILD_SET: 1,
+    ops.COMPARE_OP: -1,
 
-    ops.LOOKUP_METHOD : 1,
+    ops.LOOKUP_METHOD: 1,
 
-    ops.LOAD_NAME : 1,
-    ops.STORE_NAME : -1,
-    ops.DELETE_NAME : 0,
+    ops.LOAD_NAME: 1,
+    ops.STORE_NAME: -1,
+    ops.DELETE_NAME: 0,
 
-    ops.LOAD_FAST : 1,
-    ops.STORE_FAST : -1,
-    ops.DELETE_FAST : 0,
+    ops.LOAD_FAST: 1,
+    ops.STORE_FAST: -1,
+    ops.DELETE_FAST: 0,
 
-    ops.LOAD_ATTR : 0,
-    ops.STORE_ATTR : -2,
-    ops.DELETE_ATTR : -1,
+    ops.LOAD_ATTR: 0,
+    ops.STORE_ATTR: -2,
+    ops.DELETE_ATTR: -1,
 
-    ops.LOAD_GLOBAL : 1,
-    ops.STORE_GLOBAL : -1,
-    ops.DELETE_GLOBAL : 0,
-    ops.DELETE_DEREF : 0,
-
-    ops.LOAD_CLOSURE : 1,
-    ops.LOAD_DEREF : 1,
-    ops.STORE_DEREF : -1,
+    ops.LOAD_GLOBAL: 1,
+    ops.STORE_GLOBAL: -1,
+    ops.DELETE_GLOBAL: 0,
     ops.DELETE_DEREF: 0,
 
-    ops.LOAD_CONST : 1,
+    ops.LOAD_CLOSURE: 1,
+    ops.LOAD_DEREF: 1,
+    ops.STORE_DEREF: -1,
+    ops.DELETE_DEREF: 0,
 
-    ops.IMPORT_STAR : -1,
-    ops.IMPORT_NAME : -1,
-    ops.IMPORT_FROM : 1,
+    ops.LOAD_CONST: 1,
 
-    ops.JUMP_FORWARD : 0,
-    ops.JUMP_ABSOLUTE : 0,
-    ops.JUMP_IF_TRUE_OR_POP : 0,
-    ops.JUMP_IF_FALSE_OR_POP : 0,
-    ops.POP_JUMP_IF_TRUE : -1,
-    ops.POP_JUMP_IF_FALSE : -1,
-    ops.JUMP_IF_NOT_DEBUG : 0,
+    ops.IMPORT_STAR: -1,
+    ops.IMPORT_NAME: -1,
+    ops.IMPORT_FROM: 1,
+
+    ops.JUMP_FORWARD: 0,
+    ops.JUMP_ABSOLUTE: 0,
+    ops.JUMP_IF_TRUE_OR_POP: 0,
+    ops.JUMP_IF_FALSE_OR_POP: 0,
+    ops.POP_JUMP_IF_TRUE: -1,
+    ops.POP_JUMP_IF_FALSE: -1,
+    ops.JUMP_IF_NOT_DEBUG: 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -108,6 +108,15 @@
     return "_%s%s" % (klass[i:], name)
 
 
+def intern_if_common_string(space, w_const):
+    # only intern identifier-like strings
+    from pypy.objspace.std.unicodeobject import _isidentifier
+    if (space.is_w(space.type(w_const), space.w_unicode) and
+        _isidentifier(space.unicode_w(w_const))):
+        return space.new_interned_w_str(w_const)
+    return w_const
+
+
 def new_identifier(space, name):
     # Check whether there are non-ASCII characters in the identifier; if
     # so, normalize to NFKC
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -279,6 +279,11 @@
                 if w_const is None:
                     return tup
                 consts_w[i] = w_const
+            # intern the string constants packed into the tuple here,
+            # because assemble.py will see the result as just a tuple constant
+            for i in range(len(consts_w)):
+                consts_w[i] = misc.intern_if_common_string(
+                    self.space, consts_w[i])
         else:
             consts_w = []
         w_consts = self.space.newtuple(consts_w)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -15,7 +15,7 @@
     UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
-from pypy.interpreter.miscutils import ThreadLocals
+from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
 
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
@@ -362,7 +362,7 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        self.interned_strings = {}
+        self.interned_strings = make_weak_value_dictionary(self, unicode, W_Root)
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
@@ -501,11 +501,6 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: rctime take precedence over time
-        # XXX this could probably be done as a "requires" in the config
-        if 'rctime' in modules and 'time' in modules:
-            modules.remove('time')
-
         self._builtinmodule_list = modules
         return self._builtinmodule_list
 
@@ -772,26 +767,35 @@
         else:
             return self.w_False
 
-    def new_interned_w_str(self, w_s):
-        s = self.str_w(w_s)
+    def new_interned_w_str(self, w_u):
+        assert isinstance(w_u, W_Root)   # and is not None
+        u = self.unicode_w(w_u)
+        if not we_are_translated():
+            assert type(u) is unicode
+        w_u1 = self.interned_strings.get(u)
+        if w_u1 is None:
+            w_u1 = w_u
+            self.interned_strings.set(u, w_u1)
+        return w_u1
+
+    def new_interned_str(self, s):
+        """Assumes an identifier (utf-8 encoded str)"""
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        self.interned_strings[s] = w_s
-        return w_s
+        u = s.decode('utf-8')
+        w_s1 = self.interned_strings.get(u)
+        if w_s1 is None:
+            w_s1 = self.wrap(u)
+            self.interned_strings.set(u, w_s1)
+        return w_s1
 
-    def new_interned_str(self, s):
+    def is_interned_str(self, s):
+        """Assumes an identifier (utf-8 encoded str)"""
+        # interface for marshal_impl
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        w_s = self.interned_strings[s] = self.wrap(s)
-        return w_s
+        u = s.decode('utf-8')
+        return self.interned_strings.get(u) is not None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -32,6 +32,17 @@
         self.compiler = space.createcompiler()
         self.profilefunc = None
         self.w_profilefuncarg = None
+        self.thread_disappeared = False   # might be set to True after os.fork()
+
+    @staticmethod
+    def _mark_thread_disappeared(space):
+        # Called in the child process after os.fork() by interp_posix.py.
+        # Marks all ExecutionContexts except the current one
+        # with 'thread_disappeared = True'.
+        me = space.getexecutioncontext()
+        for ec in space.threadlocals.getallvalues().values():
+            if ec is not me:
+                ec.thread_disappeared = True
 
     def gettopframe(self):
         return self.topframeref()
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -31,3 +31,19 @@
 
     def getallvalues(self):
         return {0: self._value}
+
+
+def make_weak_value_dictionary(space, keytype, valuetype):
+    "NOT_RPYTHON"
+    if space.config.translation.rweakref:
+        from rpython.rlib.rweakref import RWeakValueDictionary
+        return RWeakValueDictionary(keytype, valuetype)
+    else:
+        class FakeWeakValueDict(object):
+            def __init__(self):
+                self._dict = {}
+            def get(self, key):
+                return self._dict.get(key, None)
+            def set(self, key, value):
+                self._dict[key] = value
+        return FakeWeakValueDict()
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -131,7 +131,6 @@
         # class bodies only have CO_NEWLOCALS.
         # CO_NEWLOCALS: make a locals dict unless optimized is also set
         # CO_OPTIMIZED: no locals dict needed at all
-        # NB: this method is overridden in nestedscope.py
         flags = code.co_flags
         if not (flags & pycode.CO_OPTIMIZED):
             if flags & pycode.CO_NEWLOCALS:
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -1121,7 +1121,13 @@
             sys.stdout = out
         output = s.getvalue()
         assert "CALL_METHOD" in output
-            
+
+    def test_interned_strings(self):
+        source = """x = ('foo_bar42', 5); y = 'foo_bar42'; z = x[0]"""
+        ns = {}
+        exec(source, ns)
+        assert ns['y'] is ns['z']
+
 
 class AppTestExceptions:
     def test_indentation_error(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -378,3 +378,41 @@
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+
+    def test_interned_strings_are_weak(self):
+        import weakref, gc, random
+        space = self.space
+        assert space.config.translation.rweakref
+        w1 = space.new_interned_str("abcdef")
+        w2 = space.new_interned_str("abcdef")
+        assert w2 is w1
+        #
+        # check that 'w1' goes away if we don't hold a reference to it
+        rw1 = weakref.ref(w1)
+        del w1, w2
+        i = 10
+        while rw1() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
+        #
+        s = "foobar%r" % random.random()
+        w0 = space.wrap(s)
+        w1 = space.new_interned_w_str(w0)
+        assert w1 is w0
+        w2 = space.new_interned_w_str(w0)
+        assert w2 is w0
+        w3 = space.wrap(s)
+        assert w3 is not w0
+        w4 = space.new_interned_w_str(w3)
+        assert w4 is w0
+        #
+        # check that 'w0' goes away if we don't hold a reference to it
+        # (even if we hold a reference to 'w3')
+        rw0 = weakref.ref(w0)
+        del w0, w1, w2, w4
+        i = 10
+        while rw0() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -158,21 +158,14 @@
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
-    _attrs_            = ['value_fits_long', 'vmin', 'vrangemax']
-    _immutable_fields_ = ['value_fits_long', 'vmin', 'vrangemax']
+    _attrs_            = ['value_fits_long', 'value_smaller_than_long']
+    _immutable_fields_ = ['value_fits_long', 'value_smaller_than_long']
     is_primitive_integer = True
 
     def __init__(self, *args):
         W_CTypePrimitive.__init__(self, *args)
         self.value_fits_long = self.size <= rffi.sizeof(lltype.Signed)
-        if self.size < rffi.sizeof(lltype.Signed):
-            assert self.value_fits_long
-            sh = self.size * 8
-            self.vmin = r_uint(-1) << (sh - 1)
-            self.vrangemax = (r_uint(1) << sh) - 1
-        else:
-            self.vmin = r_uint(0)
-            self.vrangemax = r_uint(-1)
+        self.value_smaller_than_long = self.size < rffi.sizeof(lltype.Signed)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -192,8 +185,17 @@
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
             value = misc.as_long(self.space, w_ob)
-            if self.size < rffi.sizeof(lltype.Signed):
-                if r_uint(value) - self.vmin > self.vrangemax:
+            if self.value_smaller_than_long:
+                size = self.size
+                if size == 1:
+                    signextended = misc.signext(value, 1)
+                elif size == 2:
+                    signextended = misc.signext(value, 2)
+                elif size == 4:
+                    signextended = misc.signext(value, 4)
+                else:
+                    raise AssertionError("unsupported size")
+                if value != signextended:
                     self._overflow(w_ob)
             misc.write_raw_signed_data(cdata, value, self.size)
         else:
@@ -221,7 +223,7 @@
             length = w_cdata.get_array_length()
             populate_list_from_raw_array(res, buf, length)
             return res
-        elif self.value_fits_long:
+        elif self.value_smaller_than_long:
             res = [0] * w_cdata.get_array_length()
             misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
             return res
@@ -235,8 +237,8 @@
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
-                overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                overflowed = misc.pack_list_to_raw_array_bounds_signed(
+                    int_list, cdata, self.size)
                 if overflowed != 0:
                     self._overflow(self.space.wrap(overflowed))
             return True
@@ -314,8 +316,8 @@
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
-            overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
+                int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.wrap(overflowed))
             return True
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -222,6 +222,19 @@
 neg_msg = "can't convert negative number to unsigned"
 ovf_msg = "long too big to convert"
 
+ at specialize.arg(1)
+def signext(value, size):
+    # 'value' is sign-extended from 'size' bytes to a full integer.
+    # 'size' should be a constant smaller than a full integer size.
+    if size == rffi.sizeof(rffi.SIGNEDCHAR):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SIGNEDCHAR, value))
+    elif size == rffi.sizeof(rffi.SHORT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SHORT, value))
+    elif size == rffi.sizeof(rffi.INT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+    else:
+        raise AssertionError("unsupported size")
+
 # ____________________________________________________________
 
 class _NotStandardObject(Exception):
@@ -339,13 +352,26 @@
 
 # ____________________________________________________________
 
-def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+def pack_list_to_raw_array_bounds_signed(int_list, target, size):
     for TP, TPP in _prim_signed_types:
         if size == rffi.sizeof(TP):
             ptr = rffi.cast(TPP, target)
             for i in range(len(int_list)):
                 x = int_list[i]
-                if r_uint(x) - vmin > vrangemax:
+                y = rffi.cast(TP, x)
+                if x != rffi.cast(lltype.Signed, y):
+                    return x      # overflow
+                ptr[i] = y
+            return 0
+    raise NotImplementedError("bad integer size")
+
+def pack_list_to_raw_array_bounds_unsigned(int_list, target, size, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) > vrangemax:
                     return x      # overflow
                 ptr[i] = rffi.cast(TP, x)
             return 0
diff --git a/pypy/module/_codecs/locale_codec.h b/pypy/module/_codecs/locale_codec.h
--- a/pypy/module/_codecs/locale_codec.h
+++ b/pypy/module/_codecs/locale_codec.h
@@ -2,7 +2,7 @@
 #include <wchar.h>
 #include "src/precommondefs.h"
 
-RPY_EXPORTED_FOR_TESTS wchar_t* pypy_char2wchar(const char* arg, size_t *size);
-RPY_EXPORTED_FOR_TESTS void pypy_char2wchar_free(wchar_t *text);
-RPY_EXPORTED_FOR_TESTS char* pypy_wchar2char(const wchar_t *text, size_t *error_pos);
-RPY_EXPORTED_FOR_TESTS void pypy_wchar2char_free(char *bytes);
+RPY_EXTERN wchar_t* pypy_char2wchar(const char* arg, size_t *size);
+RPY_EXTERN void pypy_char2wchar_free(wchar_t *text);
+RPY_EXTERN char* pypy_wchar2char(const wchar_t *text, size_t *error_pos);
+RPY_EXTERN void pypy_wchar2char_free(char *bytes);
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -1,6 +1,6 @@
 class AppTestCProfile(object):
     spaceconfig = {
-        "usemodules": ['_lsprof', 'rctime'],
+        "usemodules": ['_lsprof', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -7,7 +7,7 @@
 
 class AppTestMD5(object):
     spaceconfig = {
-        'usemodules': ['_md5', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_md5', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
@@ -97,24 +97,24 @@
   Py_UNICODE *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_dec_s *pypy_cjk_dec_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_init(struct pypy_cjk_dec_s *d,
                              char *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_dec_free(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_chunk(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_UNICODE *pypy_cjk_dec_outbuf(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_outlen(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_remaining(struct pypy_cjk_dec_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_consumed(struct pypy_cjk_dec_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_replace_on_error(struct pypy_cjk_dec_s* d,
                                          Py_UNICODE *, Py_ssize_t, Py_ssize_t);
 
@@ -125,35 +125,35 @@
   unsigned char *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_enc_s *pypy_cjk_enc_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_init(struct pypy_cjk_enc_s *d,
                              Py_UNICODE *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_enc_free(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_chunk(struct pypy_cjk_enc_s *, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_reset(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char *pypy_cjk_enc_outbuf(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_outlen(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_remaining(struct pypy_cjk_enc_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_consumed(struct pypy_cjk_enc_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_replace_on_error(struct pypy_cjk_enc_s* d,
                                          char *, Py_ssize_t, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 const MultibyteCodec *pypy_cjk_enc_getcodec(struct pypy_cjk_enc_s *);
 
 /* list of codecs defined in the .c files */
 
 #define DEFINE_CODEC(name)                              \
-    RPY_EXPORTED_FOR_TESTS MultibyteCodec *pypy_cjkcodec_##name(void);
+    RPY_EXTERN MultibyteCodec *pypy_cjkcodec_##name(void);
 
 // _codecs_cn
 DEFINE_CODEC(gb2312)
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -254,7 +254,7 @@
         start = _GetTickCount()
 
         while True:
-            from pypy.module.rctime.interp_time import State
+            from pypy.module.time.interp_time import State
             interrupt_event = space.fromcache(State).get_interrupt_event()
             handles = [self.handle, interrupt_event]
 
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.h b/pypy/module/_posixsubprocess/_posixsubprocess.h
--- a/pypy/module/_posixsubprocess/_posixsubprocess.h
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.h
@@ -1,6 +1,6 @@
 #include "src/precommondefs.h"
 
-RPY_EXPORTED_FOR_TESTS void
+RPY_EXTERN void
 pypy_subprocess_child_exec(
            char *const exec_array[],
            char *const argv[],
@@ -17,8 +17,8 @@
            int (*preexec_fn)(void*),
            void *preexec_fn_arg);
 
-RPY_EXPORTED_FOR_TESTS int
+RPY_EXTERN int
 pypy_subprocess_cloexec_pipe(int *fds);
 
-RPY_EXPORTED_FOR_TESTS void
+RPY_EXTERN void
 pypy_subprocess_init(void);
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -1,6 +1,6 @@
 class AppTestRandom:
     spaceconfig = {
-        "usemodules": ['_random', 'rctime'],
+        "usemodules": ['_random', 'time'],
     }
 
     def test_dict(self):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -685,7 +685,8 @@
                     raise _ssl_seterror(space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return space.wrap(rffi.charpsize2str(buf_ptr[0], length))
+                    return space.wrapbytes(
+                        rffi.charpsize2str(buf_ptr[0], length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -181,6 +181,14 @@
         self.s.close()
         del ss; gc.collect()
         
+    def test_peer_certificate(self):
+        import gc, ssl
+        ss = ssl.wrap_socket(self.s)
+        assert ss.getpeercert() == {}
+        assert type(ss.getpeercert(True)) is bytes
+        self.s.close()
+        del ss; gc.collect()
+
     def test_npn_protocol(self):
         import socket, _ssl, gc
         ctx = _ssl._SSLContext()
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -24,12 +24,19 @@
 
 separate_module_source = """
 #include <openssl/crypto.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 
 static unsigned int _ssl_locks_count = 0;
 static struct RPyOpaque_ThreadLock *_ssl_locks;
 
 static unsigned long _ssl_thread_id_function(void) {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (unsigned long)GetCurrentThreadId();
+#else
+    return (unsigned long)pthread_self();
+#endif
 }
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
@@ -65,7 +72,7 @@
 eci = rthread.eci.merge(ExternalCompilationInfo(
     separate_module_sources=[separate_module_source],
     post_include_bits=[
-        "RPY_EXPORTED_FOR_TESTS int _PyPy_SSL_SetupThreads(void);"],
+        "RPY_EXTERN int _PyPy_SSL_SetupThreads(void);"],
     libraries = libraries,
 ))
 
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -54,7 +54,7 @@
     # XXX: CheckAllocation fails on py3 (seems to false positive on
     # BZ2File's RLocks)
     spaceconfig = {
-        'usemodules': ['bz2', 'binascii', 'rctime', 'struct', 'thread']
+        'usemodules': ['bz2', 'binascii', 'time', 'struct', 'thread']
     }
 
     def setup_class(cls):
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -2,6 +2,7 @@
 #define CPPYY_CAPI
 
 #include <stddef.h>
+#include "src/precommondefs.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -15,102 +16,167 @@
     typedef void* (*cppyy_methptrgetter_t)(cppyy_object_t);
 
     /* name to opaque C++ scope representation -------------------------------- */
+    RPY_EXTERN
     int cppyy_num_scopes(cppyy_scope_t parent);
+    RPY_EXTERN
     char* cppyy_scope_name(cppyy_scope_t parent, int iscope);
 
+    RPY_EXTERN
     char* cppyy_resolve_name(const char* cppitem_name);
+    RPY_EXTERN
     cppyy_scope_t cppyy_get_scope(const char* scope_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_get_template(const char* template_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj);
 
     /* memory management ------------------------------------------------------ */
+    RPY_EXTERN
     cppyy_object_t cppyy_allocate(cppyy_type_t type);
+    RPY_EXTERN
     void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self);
+    RPY_EXTERN
     void cppyy_destruct(cppyy_type_t type, cppyy_object_t self);
 
     /* method/function dispatching -------------------------------------------- */
+    RPY_EXTERN
     void   cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char   cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     short  cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     int    cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long   cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     float  cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args);
+    RPY_EXTERN
     cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type);
 
+    RPY_EXTERN
     cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx);
 
     /* handling of function argument buffer ----------------------------------- */
+    RPY_EXTERN
     void*  cppyy_allocate_function_args(int nargs);
+    RPY_EXTERN
     void   cppyy_deallocate_function_args(void* args);
+    RPY_EXTERN
     size_t cppyy_function_arg_sizeof();
+    RPY_EXTERN
     size_t cppyy_function_arg_typeoffset();
 
     /* scope reflection information ------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_namespace(cppyy_scope_t scope);
+    RPY_EXTERN
     int cppyy_is_enum(const char* type_name);
 
     /* class reflection information ------------------------------------------- */
+    RPY_EXTERN
     char* cppyy_final_name(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_scoped_final_name(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_has_complex_hierarchy(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_num_bases(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_base_name(cppyy_type_t type, int base_index);
+    RPY_EXTERN
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
 
     /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */
+    RPY_EXTERN
     ptrdiff_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
 
     /* method/function reflection information --------------------------------- */
+    RPY_EXTERN
     int cppyy_num_methods(cppyy_scope_t scope);
+    RPY_EXTERN
     cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth);
+    RPY_EXTERN
     cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name);
 
+    RPY_EXTERN
     char* cppyy_method_name(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_result_type(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_req_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_arg_type(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_arg_default(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx);
 
+    RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_template_arg_name(cppyy_scope_t scope, cppyy_index_t idx, cppyy_index_t iarg);
 
+    RPY_EXTERN
     cppyy_method_t cppyy_get_method(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     cppyy_index_t cppyy_get_global_operator(
         cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op);
 
     /* method properties ------------------------------------------------------ */
+    RPY_EXTERN
     int cppyy_is_constructor(cppyy_type_t type, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_is_staticmethod(cppyy_type_t type, cppyy_index_t idx);
 
     /* data member reflection information ------------------------------------- */
+    RPY_EXTERN
     int cppyy_num_datamembers(cppyy_scope_t scope);
+    RPY_EXTERN
     char* cppyy_datamember_name(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
 
+    RPY_EXTERN
     int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 
     /* data member properties ------------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_publicdata(cppyy_type_t type, int datamember_index);
+    RPY_EXTERN
     int cppyy_is_staticdata(cppyy_type_t type, int datamember_index);
 
     /* misc helpers ----------------------------------------------------------- */
+    RPY_EXTERN
     long long cppyy_strtoll(const char* str);
+    RPY_EXTERN
     unsigned long long cppyy_strtoull(const char* str);
+    RPY_EXTERN
     void cppyy_free(void* ptr);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_charp2stdstring(const char* str);
+    RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
 #ifdef __cplusplus
diff --git a/pypy/module/cppyy/src/dummy_backend.cxx b/pypy/module/cppyy/src/dummy_backend.cxx
--- a/pypy/module/cppyy/src/dummy_backend.cxx
+++ b/pypy/module/cppyy/src/dummy_backend.cxx
@@ -1,4 +1,3 @@
-#include "src/precommondefs.h"
 #include "cppyy.h"
 #include "capi.h"
 
@@ -349,29 +348,24 @@
 
 
 /* name to opaque C++ scope representation -------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_scopes(cppyy_scope_t handle) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_resolve_name(const char* cppitem_name) {
     return cppstring_to_cstring(cppitem_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_scope_t cppyy_get_scope(const char* scope_name) {
     return s_handles[scope_name];  // lookup failure will return 0 (== error)
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t /* obj */) {
     return klass;
 }
 
 
 /* memory management ------------------------------------------------------ */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
     if (handle == s_handles["example01"])
        delete (dummy::example01*)self;
@@ -379,7 +373,6 @@
 
 
 /* method/function dispatching -------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long idx = (long)method;
     if (idx == s_methods["static_example01::staticSetPayload_payload*_double"]) {
@@ -469,7 +462,6 @@
     }
 }
 
-RPY_EXPORTED_FOR_TESTS
 unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     unsigned char result = 0;
     const long idx = (long)method;
@@ -482,7 +474,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char result = 0;
     const long idx = (long)method;
@@ -498,7 +489,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     short result = 0;
     const long idx = (long)method; 
@@ -514,7 +504,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     int result = 0;
     const long idx = (long)method;
@@ -547,7 +536,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long result = 0;
     const long idx = (long)method;
@@ -689,7 +677,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long long result = 0;
     const long idx = (long)method;
@@ -705,7 +692,6 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     float result = 0;
     const long idx = (long)method;
@@ -718,7 +704,6 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     double result = 0.;
     const long idx = (long)method;
@@ -740,7 +725,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char* result = 0;
     const long idx = (long)method;
@@ -753,7 +737,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
     void* result = 0;
     const long idx = (long)method;
@@ -776,14 +759,12 @@
     return (cppyy_object_t)result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) {
     return (cppyy_methptrgetter_t)0;
 }
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void* cppyy_allocate_function_args(int nargs) {
     CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
     for (int i = 0; i < nargs; ++i)
@@ -793,36 +774,30 @@
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_deallocate_function_args(void* args) {
     free(args);
 }
 
-RPY_EXPORTED_FOR_TESTS
 size_t cppyy_function_arg_sizeof() {
     return sizeof(CPPYY_G__value);
 }
 
-RPY_EXPORTED_FOR_TESTS
 size_t cppyy_function_arg_typeoffset() {
     return offsetof(CPPYY_G__value, type);
 }
 
 
 /* scope reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_namespace(cppyy_scope_t /* handle */) {
     return 0;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_enum(const char* /* type_name */) {
     return 0;
 }
     
     
 /* class reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_final_name(cppyy_type_t handle) {
     for (Handles_t::iterator isp = s_handles.begin(); isp != s_handles.end(); ++isp) {
         if (isp->second == handle)
@@ -831,75 +806,61 @@
     return cppstring_to_cstring("<unknown>");
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_scoped_final_name(cppyy_type_t handle) {
     return cppyy_final_name(handle);
 }   
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_has_complex_hierarchy(cppyy_type_t /* handle */) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_bases(cppyy_type_t /*handle*/) {
    return 0;
 }
 
 
 /* method/function reflection information --------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_methods(cppyy_scope_t handle) {
     return s_scopes[handle].m_methods.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) {
     return (cppyy_index_t)imeth;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_returntype);
 }
     
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return s_scopes[handle].m_methods[method_index].m_argtypes.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppyy_method_num_args(handle, method_index);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_argtypes[arg_index]);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_arg_default(
         cppyy_scope_t /* handle */, cppyy_index_t /* method_index */, int /* arg_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_signature(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_is_template(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return 0;
 }
     
-RPY_EXPORTED_FOR_TESTS
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end()) {
         long id = s_scopes[handle].m_method_offset + (long)method_index;
@@ -911,7 +872,6 @@
 
 
 /* method properties -----------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kConstructor;
@@ -919,7 +879,6 @@
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kStatic;
@@ -929,34 +888,28 @@
 
 
 /* data member reflection information ------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_datamembers(cppyy_scope_t handle) {
     return s_scopes[handle].m_datambrs.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_datamember_name(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_datamember_type(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_type);
 }
 
-RPY_EXPORTED_FOR_TESTS
 ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_offset;
 }
 
 
 /* data member properties ------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) {
     return 1;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
@@ -964,44 +917,37 @@
 
 /* misc helpers ----------------------------------------------------------- */
 #if defined(_MSC_VER)
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_strtoll(const char* str) {
     return _strtoi64(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
 unsigned long long cppyy_strtoull(const char* str) {
     return _strtoui64(str, NULL, 0);
 }
 }
 #else
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_strtoll(const char* str) {
     return strtoll(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
 unsigned long long cppyy_strtoull(const char* str) {
     return strtoull(str, NULL, 0);
 }
 }
 #endif
 
-RPY_EXPORTED_FOR_TESTS
 void cppyy_free(void* ptr) {
     free(ptr);
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_charp2stdstring(const char* str) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(str);
     return (cppyy_object_t)arena;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(*(std::string*)ptr);
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -50,7 +50,7 @@
             eci = ExternalCompilationInfo(
                 separate_module_files=[srcpath.join('dummy_backend.cxx')],
                 include_dirs=[incpath, tstpath, cdir],
-                compile_extra=['-DRPY_EXPORTED_FOR_TESTS=RPY_EXPORTED'],
+                compile_extra=['-DRPY_EXTERN=RPY_EXPORTED'],
                 use_cpp_linker=True,
             )
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -779,8 +779,7 @@
     struct PyPyAPI {
     %(members)s
     } _pypyAPI;
-    RPY_EXPORTED_FOR_TESTS
-    struct PyPyAPI* pypyAPI = &_pypyAPI;
+    RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
     functions = generate_decls_and_callbacks(db, export_symbols)
@@ -951,7 +950,7 @@
         name_no_star = process_va_name(name)
         header = ('%s pypy_va_get_%s(va_list* vp)' %
                   (name, name_no_star))
-        pypy_decls.append('RPY_EXPORTED_FOR_TESTS ' + header + ';')
+        pypy_decls.append('RPY_EXTERN ' + header + ';')
         functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
 
     for name, (typ, expr) in GLOBALS.iteritems():
@@ -1011,7 +1010,7 @@
     if sys.platform == 'win32':
         get_pythonapi_source = '''
         #include <windows.h>
-        RPY_EXPORTED_FOR_TESTS
+        RPY_EXTERN
         HANDLE pypy_get_pythonapi_handle() {
             MEMORY_BASIC_INFORMATION  mi;
             memset(&mi, 0, sizeof(mi));
diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h
--- a/pypy/module/cpyext/include/modsupport.h
+++ b/pypy/module/cpyext/include/modsupport.h
@@ -82,11 +82,20 @@
 /*
  * This is from pyport.h.  Perhaps it belongs elsewhere.
  */
+#ifdef _WIN32
+/* explicitly export since PyAPI_FUNC is usually dllimport */
+#ifdef __cplusplus
+#define PyMODINIT_FUNC extern "C" __declspec(dllexport) void
+#else
+#define PyMODINIT_FUNC __declspec(dllexport) void
+#endif
+#else
 #ifdef __cplusplus
 #define PyMODINIT_FUNC extern "C" PyAPI_FUNC(PyObject *)
 #else
 #define PyMODINIT_FUNC PyAPI_FUNC(PyObject *)
 #endif
+#endif /* WIN32 */
 
 PyAPI_DATA(char *) _Py_PackageContext;
 
diff --git a/pypy/module/cpyext/src/pythread.c b/pypy/module/cpyext/src/pythread.c
--- a/pypy/module/cpyext/src/pythread.c
+++ b/pypy/module/cpyext/src/pythread.c
@@ -1,11 +1,18 @@
 #include <Python.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 #include "pythread.h"
 #include "src/thread.h"
 
 long
 PyThread_get_thread_ident(void)
 {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (long)GetCurrentThreadId();
+#else
+    return (long)pthread_self();
+#endif
 }
 
 PyThread_type_lock
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -7,7 +7,7 @@
     # it's necessary to run "import time" at least once before any
     # other cpyext test, otherwise the same statement will fail in
     # test_datetime.py.
-    space = gettestobjspace(usemodules=['rctime'])
+    space = gettestobjspace(usemodules=['time'])
     space.getbuiltinmodule("time")
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -102,7 +102,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
+                                   'itertools', 'time', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -12,7 +12,7 @@
 
 class AppTestFcntl:
     spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios',
-                                   'select', 'rctime'))
+                                   'select', 'time'))
 
     def setup_class(cls):
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -4,7 +4,7 @@
 
 class AppTestImpModule:
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'rctime', 'struct'],
+        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -163,7 +163,7 @@
 
 class AppTestImport(BaseImportTest):
     spaceconfig = {
-        "usemodules": ['rctime'],
+        "usemodules": ['time'],
     }
 
     def setup_class(cls):
@@ -1049,7 +1049,7 @@
 
 class AppTestImportHooks(object):
     spaceconfig = {
-        "usemodules": ['struct', 'itertools', 'rctime'],
+        "usemodules": ['struct', 'itertools', 'time'],
     }
 
     def setup_class(cls):
@@ -1266,7 +1266,7 @@
 
 
 class AppTestMultithreadedImp(object):
-    spaceconfig = dict(usemodules=['thread', 'rctime'])
+    spaceconfig = dict(usemodules=['thread', 'time'])
 
     def setup_class(cls):
         #if not conftest.option.runappdirect:
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -152,7 +152,6 @@
     atom_int(tc, int)           puts code and int
     atom_int64(tc, int64)       puts code and int64
     atom_str(tc, str)           puts code, len and string
-    atom_strlist(tc, strlist)   puts code, len and list of strings
 
     building blocks for compound types:
 
@@ -206,15 +205,6 @@
         self.atom_int(typecode, len(x))
         self.put(x)
 
-    def atom_strlist(self, typecode, tc2, x):
-        self.atom_int(typecode, len(x))
-        atom_str = self.atom_str
-        for item in x:
-            # type(str) seems to be forbidden
-            #if type(item) is not str:

From noreply at buildbot.pypy.org  Mon Dec 29 22:43:08 2014
From: noreply at buildbot.pypy.org (Arjun Naik)
Date: Mon, 29 Dec 2014 22:43:08 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: reapply 8ccff189b82f: Changed the
 _get_inttime() to raise OverflowError.
Message-ID: <20141229214308.909B41C0455@cobra.cs.uni-duesseldorf.de>

Author: Arjun Naik <arjun at arjunnaik.in>
Branch: py3.3
Changeset: r75153:9a110e6e733e
Date: 2014-07-26 15:11 +0200
http://bitbucket.org/pypy/pypy/changeset/9a110e6e733e/

Log:	reapply 8ccff189b82f: Changed the _get_inttime() to raise
	OverflowError.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -368,7 +368,7 @@
     # input doesn't fit in a time_t; call it an error.
     diff = seconds - rffi.cast(lltype.Float, t)
     if diff <= -1.0 or diff >= 1.0:
-        raise OperationError(space.w_ValueError,
+        raise OperationError(space.w_OverflowError,
                       space.wrap("timestamp out of range for platform time_t"))
     return t
 
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -42,7 +42,7 @@
         res = time.ctime(0)
         assert isinstance(res, str)
         time.ctime(time.time())
-        raises(ValueError, time.ctime, 1E200)
+        raises(OverflowError, time.ctime, 1E200)
         raises(OverflowError, time.ctime, 10**900)
         for year in [-100, 100, 1000, 2000, 10000]:
             try:
@@ -68,8 +68,8 @@
         assert 0 <= (t1 - t0) < 1.2
         t = time.time()
         assert time.gmtime(t) == time.gmtime(t)
-        raises(ValueError, time.gmtime, 2**64)
-        raises(ValueError, time.gmtime, -2**64)
+        raises(OverflowError, time.gmtime, 2**64)
+        raises(OverflowError, time.gmtime, -2**64)
 
     def test_localtime(self):
         import time

From noreply at buildbot.pypy.org  Mon Dec 29 22:43:09 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Mon, 29 Dec 2014 22:43:09 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: reapply ca139d96a0fa: fill in
	_STRUCT_TM_TIMES
Message-ID: <20141229214309.BD17C1C0455@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.3
Changeset: r75154:b60d4915a3f5
Date: 2014-07-27 15:44 -0700
http://bitbucket.org/pypy/pypy/changeset/b60d4915a3f5/

Log:	reapply ca139d96a0fa: fill in _STRUCT_TM_TIMES

diff --git a/pypy/module/time/__init__.py b/pypy/module/time/__init__.py
--- a/pypy/module/time/__init__.py
+++ b/pypy/module/time/__init__.py
@@ -17,6 +17,7 @@
         'mktime': 'interp_time.mktime',
         'strftime': 'interp_time.strftime',
         'sleep' : 'interp_time.sleep',
+        '_STRUCT_TM_ITEMS': 'space.wrap(interp_time._STRUCT_TM_ITEMS)',
     }
 
     if os.name == "posix":
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -135,6 +135,9 @@
         ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
         ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
 
+# XXX: optionally support the 2 additional tz fields
+_STRUCT_TM_ITEMS = 9
+
 class cConfig:
     pass
 
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -11,6 +11,7 @@
         assert isinstance(time.timezone, int)
         assert isinstance(time.tzname, tuple)
         assert isinstance(time.__doc__, str)
+        assert isinstance(time._STRUCT_TM_ITEMS, int)
 
     def test_sleep(self):
         import sys

From noreply at buildbot.pypy.org  Tue Dec 30 03:58:12 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Dec 2014 03:58:12 +0100 (CET)
Subject: [pypy-commit] cffi default: Must also make 'sources' relative.
Message-ID: <20141230025812.97CED1C030C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1610:419da551d501
Date: 2014-12-30 03:56 +0100
http://bitbucket.org/cffi/cffi/changeset/419da551d501/

Log:	Must also make 'sources' relative.

diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py
--- a/cffi/ffiplatform.py
+++ b/cffi/ffiplatform.py
@@ -11,7 +11,7 @@
     """
 
 
-LIST_OF_FILE_NAMES = ['include_dirs', 'library_dirs',
+LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs',
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):

From noreply at buildbot.pypy.org  Tue Dec 30 03:58:13 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Dec 2014 03:58:13 +0100 (CET)
Subject: [pypy-commit] cffi default: issue #169: offsetof('struct', 'a.b.c')
Message-ID: <20141230025813.C3CE21C030C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1611:bcdbd8bc0c38
Date: 2014-12-30 03:58 +0100
http://bitbucket.org/cffi/cffi/changeset/bcdbd8bc0c38/

Log:	issue #169: offsetof('struct', 'a.b.c')

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -192,11 +192,12 @@
 
     def offsetof(self, cdecl, fieldname):
         """Return the offset of the named field inside the given
-        structure, which must be given as a C type name.
+        structure, which must be given as a C type name.  The field
+        may be 'x.y.z' in case of nested structures.
         """
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl)
-        return self._backend.typeoffsetof(cdecl, fieldname)[1]
+        return self._typeoffsetof(cdecl, fieldname)[1]
 
     def new(self, cdecl, init=None):
         """Allocate an instance according to the specified C type and
@@ -369,12 +370,22 @@
     def addressof(self, cdata, field=None):
         """Return the address of a <cdata 'struct-or-union'>.
         If 'field' is specified, return the address of this field.
+        The field may be 'x.y.z' in case of nested structures.
         """
         ctype = self._backend.typeof(cdata)
-        ctype, offset = self._backend.typeoffsetof(ctype, field)
+        ctype, offset = self._typeoffsetof(ctype, field)
         ctypeptr = self._pointer_to(ctype)
         return self._backend.rawaddressof(ctypeptr, cdata, offset)
 
+    def _typeoffsetof(self, ctype, field):
+        if field is not None and '.' in field:
+            offset = 0
+            for field1 in field.split('.'):
+                ctype, offset1 = self._backend.typeoffsetof(ctype, field1)
+                offset += offset1
+            return ctype, offset
+        return self._backend.typeoffsetof(ctype, field)
+
     def include(self, ffi_to_include):
         """Includes the typedefs, structs, unions and enums defined
         in another FFI instance.  Usage is similar to a #include in C,
diff --git a/testing/backend_tests.py b/testing/backend_tests.py
--- a/testing/backend_tests.py
+++ b/testing/backend_tests.py
@@ -949,6 +949,15 @@
         assert ffi.offsetof("struct foo", "b") == 4
         assert ffi.offsetof("struct foo", "c") == 8
 
+    def test_offsetof_nested(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo { int a, b, c; };"
+                 "struct bar { struct foo d, e; };")
+        assert ffi.offsetof("struct bar", "e") == 12
+        assert ffi.offsetof("struct bar", "e.a") == 12
+        assert ffi.offsetof("struct bar", "e.b") == 16
+        assert ffi.offsetof("struct bar", "e.c") == 20
+
     def test_alignof(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct foo { char a; short b; char c; };")
@@ -1495,6 +1504,16 @@
         assert a == ffi.addressof(p, 'y')
         assert a != ffi.addressof(p, 'x')
 
+    def test_addressof_field_nested(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("struct foo_s { int x, y; };"
+                 "struct bar_s { struct foo_s a, b; };")
+        p = ffi.new("struct bar_s *")
+        a = ffi.addressof(p[0], 'b.y')
+        assert int(ffi.cast("uintptr_t", a)) == (
+            int(ffi.cast("uintptr_t", p)) +
+            ffi.sizeof("struct foo_s") + ffi.sizeof("int"))
+
     def test_addressof_anonymous_struct(self):
         ffi = FFI()
         ffi.cdef("typedef struct { int x; } foo_t;")

From noreply at buildbot.pypy.org  Tue Dec 30 04:21:16 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Dec 2014 04:21:16 +0100 (CET)
Subject: [pypy-commit] cffi default: Document bcdbd8bc0c38. Change the
 future version number to 0.9.
Message-ID: <20141230032116.0E3221C0455@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1612:88b30bc17082
Date: 2014-12-30 04:04 +0100
http://bitbucket.org/cffi/cffi/changeset/88b30bc17082/

Log:	Document bcdbd8bc0c38. Change the future version number to 0.9.

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -435,7 +435,7 @@
 * *New in version 0.6:* all `common Windows types`_ are defined if you run
   on Windows (``DWORD``, ``LPARAM``, etc.).
 
-* *New in version 0.8.7:* the other standard integer types from stdint.h,
+* *New in version 0.9:* the other standard integer types from stdint.h,
   as long as they map to integers of 1, 2, 4 or 8 bytes.  Larger integers
   are not supported.
 
@@ -445,7 +445,7 @@
 .. "versionadded:: 0.6": bool
 .. "versionadded:: 0.4": FILE
 .. "versionadded:: 0.6": Wintypes
-.. "versionadded:: 0.8.7": intmax_t etc.
+.. "versionadded:: 0.9": intmax_t etc.
 
 As we will see on `the verification step`_ below, the declarations can
 also contain "``...``" at various places; these are placeholders that will
@@ -672,7 +672,7 @@
    check.  Be sure to have other means of clearing the ``tmpdir``
    whenever you change your sources.
 
-.. versionadded:: 0.8.7
+.. versionadded:: 0.9
    You can give C++ source code in ``ffi.verify()``::
 
      ext = ffi.verify(r'''
@@ -681,7 +681,7 @@
          }
      ''', source_extension='.cpp', extra_compile_args=['-std=c++11'])
 
-.. versionadded:: 0.8.7
+.. versionadded:: 0.9
    The optional ``flags`` argument has been added, see ``man dlopen`` (ignored
    on Windows).  It defaults to ``ffi.RTLD_NOW``.
 
@@ -1264,6 +1264,10 @@
 
 ``ffi.offsetof("C struct type", "fieldname")``: return the offset within
 the struct of the given field.  Corresponds to ``offsetof()`` in C.
+*New in version 0.9:* ``"fieldname"`` can be ``"x.y"`` in case of nested
+structures.
+
+.. "versionadded:: 0.9" --- inlined in the previous paragraph
 
 ``ffi.getctype("C type" or <ctype>, extra="")``: return the string
 representation of the given C type.  If non-empty, the "extra" string is
@@ -1329,7 +1333,11 @@
 sure to keep it alive if it was obtained directly from ``ffi.new()``.
 *New in version 0.4.*
 
+*New in version 0.9:* ``"field"`` can be ``"x.y"`` in case of nested
+structures.
+
 .. "versionadded:: 0.4" --- inlined in the previous paragraph
+.. "versionadded:: 0.9" --- inlined in the previous paragraph
 
 
 Unimplemented features

From noreply at buildbot.pypy.org  Tue Dec 30 04:21:17 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Dec 2014 04:21:17 +0100 (CET)
Subject: [pypy-commit] cffi default: Document ffi.from_buffer()
Message-ID: <20141230032117.3129F1C0455@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1613:758533c5b7f8
Date: 2014-12-30 04:17 +0100
http://bitbucket.org/cffi/cffi/changeset/758533c5b7f8/

Log:	Document ffi.from_buffer()

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -267,9 +267,9 @@
         return self._backend.buffer(cdata, size)
 
     def from_buffer(self, python_buffer):
-        """Return a <cdata 'void *'> that points to the data of the
-        given object, which must support the buffer interface.  Note
-        that this is not meant to be used on the built-in types str,
+        """Return a <cdata 'char[]'> that points to the data of the
+        given Python object, which must support the buffer interface.
+        Note that this is not meant to be used on the built-in types str,
         unicode, or bytearray (you can build 'char[]' arrays explicitly)
         but only on objects containing large quantities of raw data
         in some other format, like 'array.array' or numpy arrays.
diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -1230,6 +1230,20 @@
    gives inconsistent results on regular byte strings).  Use ``buf[:]``
    instead.
 
+``ffi.from_buffer(python_buffer)``: return a ``<cdata 'char[]'>`` that
+points to the data of the given Python object, which must support the
+buffer interface.  This is the opposite of ``ffi.buffer()``.  It gives
+a (read-write) reference to the existing data, not a copy; for this
+reason, and for PyPy compatibility, it does not work with the built-in
+types str or unicode or bytearray.  It is meant to be used on objects
+containing large quantities of raw data, like ``array.array`` or numpy
+arrays.  It supports both the old buffer API (in Python 2.x) and the
+new memoryview API.  The original object is kept alive (and, in case
+of memoryview, locked) as long as the cdata object returned by
+``ffi.from_buffer()`` is alive.  *New in version 0.9.*
+
+.. "versionadded:: 0.9" --- inlined in the previous paragraph
+
 
 ``ffi.typeof("C type" or cdata object)``: return an object of type
 ``<ctype>`` corresponding to the parsed string, or to the C type of the
@@ -1264,10 +1278,9 @@
 
 ``ffi.offsetof("C struct type", "fieldname")``: return the offset within
 the struct of the given field.  Corresponds to ``offsetof()`` in C.
-*New in version 0.9:* ``"fieldname"`` can be ``"x.y"`` in case of nested
-structures.
 
-.. "versionadded:: 0.9" --- inlined in the previous paragraph
+.. versionchanged:: 0.9
+   ``"fieldname"`` can be ``"x.y"`` in case of nested structures.
 
 ``ffi.getctype("C type" or <ctype>, extra="")``: return the string
 representation of the given C type.  If non-empty, the "extra" string is
@@ -1333,11 +1346,10 @@
 sure to keep it alive if it was obtained directly from ``ffi.new()``.
 *New in version 0.4.*
 
-*New in version 0.9:* ``"field"`` can be ``"x.y"`` in case of nested
-structures.
+.. versionchanged:: 0.9
+   ``"field"`` can be ``"x.y"`` in case of nested structures.
 
 .. "versionadded:: 0.4" --- inlined in the previous paragraph
-.. "versionadded:: 0.9" --- inlined in the previous paragraph
 
 
 Unimplemented features

From noreply at buildbot.pypy.org  Tue Dec 30 04:21:18 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 30 Dec 2014 04:21:18 +0100 (CET)
Subject: [pypy-commit] cffi default: Make the function name at the start of
 each paragraph stand out more
Message-ID: <20141230032118.4651E1C0455@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1614:74e8d9218ff8
Date: 2014-12-30 04:21 +0100
http://bitbucket.org/cffi/cffi/changeset/74e8d9218ff8/

Log:	Make the function name at the start of each paragraph stand out more

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -1135,7 +1135,7 @@
 Misc methods on ffi
 -------------------
 
-``ffi.include(other_ffi)``: includes the typedefs, structs, unions and
+**ffi.include(other_ffi)**: includes the typedefs, structs, unions and
 enum types defined in another FFI instance.  Usage is similar to a
 ``#include`` in C, where a part of the program might include types
 defined in another part for its own usage.  Note that the include()
@@ -1151,11 +1151,11 @@
 
 .. "versionadded:: 0.5" --- inlined in the previous paragraph
 
-``ffi.errno``: the value of ``errno`` received from the most recent C call
+**ffi.errno**: the value of ``errno`` received from the most recent C call
 in this thread, and passed to the following C call, is available via
 reads and writes of the property ``ffi.errno``.
 
-``ffi.getwinerror(code=-1)``: on Windows, in addition to ``errno`` we
+**ffi.getwinerror(code=-1)**: on Windows, in addition to ``errno`` we
 also save and restore the ``GetLastError()`` value across function
 calls.  This function returns this error code as a tuple ``(code,
 message)``, adding a readable message like Python does when raising
@@ -1166,7 +1166,7 @@
 
 .. "versionadded:: 0.8" --- inlined in the previous paragraph
 
-``ffi.string(cdata, [maxlen])``: return a Python string (or unicode
+**ffi.string(cdata, [maxlen])**: return a Python string (or unicode
 string) from the 'cdata'.  *New in version 0.3.*
 
 .. "versionadded:: 0.3" --- inlined in the previous paragraph
@@ -1190,7 +1190,7 @@
   integer.
 
 
-``ffi.buffer(cdata, [size])``: return a buffer object that references
+**ffi.buffer(cdata, [size])**: return a buffer object that references
 the raw C data pointed to by the given 'cdata', of 'size' bytes.  The
 'cdata' must be a pointer or an array.  If unspecified, the size of the
 buffer is either the size of what ``cdata`` points to, or the whole size
@@ -1230,7 +1230,7 @@
    gives inconsistent results on regular byte strings).  Use ``buf[:]``
    instead.
 
-``ffi.from_buffer(python_buffer)``: return a ``<cdata 'char[]'>`` that
+**ffi.from_buffer(python_buffer)**: return a ``<cdata 'char[]'>`` that
 points to the data of the given Python object, which must support the
 buffer interface.  This is the opposite of ``ffi.buffer()``.  It gives
 a (read-write) reference to the existing data, not a copy; for this
@@ -1245,7 +1245,7 @@
 .. "versionadded:: 0.9" --- inlined in the previous paragraph
 
 
-``ffi.typeof("C type" or cdata object)``: return an object of type
+**ffi.typeof("C type" or cdata object)**: return an object of type
 ``<ctype>`` corresponding to the parsed string, or to the C type of the
 cdata instance.  Usually you don't need to call this function or to
 explicitly manipulate ``<ctype>`` objects in your code: any place that
@@ -1259,7 +1259,7 @@
         ...
 
 .. versionadded:: 0.4
-   ``ffi.CData, ffi.CType``: the Python type of the objects referred to
+   **ffi.CData, ffi.CType**: the Python type of the objects referred to
    as ``<cdata>`` and ``<ctype>`` in the rest of this document.  Note
    that some cdata objects may be actually of a subclass of
    ``ffi.CData``, and similarly with ctype, so you should check with
@@ -1269,20 +1269,20 @@
    ``item``, ``length``, ``fields``, ``args``, ``result``, ``ellipsis``,
    ``abi``, ``elements`` and ``relements``.
 
-``ffi.sizeof("C type" or cdata object)``: return the size of the
+**ffi.sizeof("C type" or cdata object)**: return the size of the
 argument in bytes.  The argument can be either a C type, or a cdata object,
 like in the equivalent ``sizeof`` operator in C.
 
-``ffi.alignof("C type")``: return the alignment of the C type.
+**ffi.alignof("C type")**: return the alignment of the C type.
 Corresponds to the ``__alignof__`` operator in GCC.
 
-``ffi.offsetof("C struct type", "fieldname")``: return the offset within
+**ffi.offsetof("C struct type", "fieldname")**: return the offset within
 the struct of the given field.  Corresponds to ``offsetof()`` in C.
 
 .. versionchanged:: 0.9
    ``"fieldname"`` can be ``"x.y"`` in case of nested structures.
 
-``ffi.getctype("C type" or <ctype>, extra="")``: return the string
+**ffi.getctype("C type" or <ctype>, extra="")**: return the string
 representation of the given C type.  If non-empty, the "extra" string is
 appended (or inserted at the right place in more complicated cases); it
 can be the name of a variable to declare, or an extra part of the type
@@ -1291,7 +1291,7 @@
 of the C type "pointer to the same type than x"; and
 ``ffi.getctype("char[80]", "a") == "char a[80]"``.
 
-``ffi.gc(cdata, destructor)``: return a new cdata object that points to the
+**ffi.gc(cdata, destructor)**: return a new cdata object that points to the
 same data.  Later, when this new cdata object is garbage-collected,
 ``destructor(old_cdata_object)`` will be called.  Example of usage:
 ``ptr = ffi.gc(lib.malloc(42), lib.free)``.  Note that like objects
@@ -1313,10 +1313,10 @@
 
 .. "versionadded:: 0.3" --- inlined in the previous paragraph
 
-``ffi.new_handle(python_object)``: return a non-NULL cdata of type
+**ffi.new_handle(python_object)**: return a non-NULL cdata of type
 ``void *`` that contains an opaque reference to ``python_object``.  You
 can pass it around to C functions or store it into C structures.  Later,
-you can use ``ffi.from_handle(p)`` to retrive the original
+you can use **ffi.from_handle(p)** to retrive the original
 ``python_object`` from a value with the same ``void *`` pointer.
 *Calling ffi.from_handle(p) is invalid and will likely crash if
 the cdata object returned by new_handle() is not kept alive!*
@@ -1333,7 +1333,7 @@
 
 .. "versionadded:: 0.7" --- inlined in the previous paragraph
 
-``ffi.addressof(cdata, field=None)``: from a cdata whose type is
+**ffi.addressof(cdata, field=None)**: from a cdata whose type is
 ``struct foo_s``, return its "address", as a cdata whose type is
 ``struct foo_s *``.  Also works on unions, but not on any other type.
 (It would be difficult because only structs and unions are internally

From noreply at buildbot.pypy.org  Tue Dec 30 08:58:15 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Dec 2014 08:58:15 +0100 (CET)
Subject: [pypy-commit] pypy pypy-dont-copy-ops: merge default
Message-ID: <20141230075815.1EF231C3354@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: pypy-dont-copy-ops
Changeset: r75155:e12313a8a8b5
Date: 2014-12-29 12:27 +0200
http://bitbucket.org/pypy/pypy/changeset/e12313a8a8b5/

Log:	merge default

diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -227,7 +227,7 @@
     '<tag key="value" />'
     >>> element.remove(subelement)    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ValueError: list.remove(x): ...
+    ValueError: list.remove(...
     >>> serialize(element) # 6
     '<tag key="value" />'
     >>> element[0:0] = [subelement, subelement, subelement]
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -83,17 +83,16 @@
 
 class __extend__(ast.BoolOp):
 
-    def _accept_jump_if_any_is(self, gen, condition, target):
-        self.values[0].accept_jump_if(gen, condition, target)
-        for i in range(1, len(self.values)):
+    def _accept_jump_if_any_is(self, gen, condition, target, skip_last=0):
+        for i in range(len(self.values) - skip_last):
             self.values[i].accept_jump_if(gen, condition, target)
 
     def accept_jump_if(self, gen, condition, target):
         if condition and self.op == ast.And or \
                 (not condition and self.op == ast.Or):
             end = gen.new_block()
-            self._accept_jump_if_any_is(gen, not condition, end)
-            gen.emit_jump(ops.JUMP_FORWARD, target)
+            self._accept_jump_if_any_is(gen, not condition, end, skip_last=1)
+            self.values[-1].accept_jump_if(gen, condition, target)
             gen.use_next_block(end)
         else:
             self._accept_jump_if_any_is(gen, condition, target)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -62,10 +62,54 @@
 
 eptype("intptr_t",  rffi.INTPTR_T,  ctypeprim.W_CTypePrimitiveSigned)
 eptype("uintptr_t", rffi.UINTPTR_T, ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("ptrdiff_t", rffi.INTPTR_T,  ctypeprim.W_CTypePrimitiveSigned) # <-xxx
 eptype("size_t",    rffi.SIZE_T,    ctypeprim.W_CTypePrimitiveUnsigned)
 eptype("ssize_t",   rffi.SSIZE_T,   ctypeprim.W_CTypePrimitiveSigned)
 
+_WCTSigned = ctypeprim.W_CTypePrimitiveSigned
+_WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned
+
+eptype("ptrdiff_t", getattr(rffi, 'PTRDIFF_T', rffi.INTPTR_T), _WCTSigned)
+eptype("intmax_t",  getattr(rffi, 'INTMAX_T',  rffi.LONGLONG), _WCTSigned)
+eptype("uintmax_t", getattr(rffi, 'UINTMAX_T', rffi.LONGLONG), _WCTUnsign)
+
+if hasattr(rffi, 'INT_LEAST8_T'):
+    eptype("int_least8_t",  rffi.INT_LEAST8_T,  _WCTSigned)
+    eptype("int_least16_t", rffi.INT_LEAST16_T, _WCTSigned)
+    eptype("int_least32_t", rffi.INT_LEAST32_T, _WCTSigned)
+    eptype("int_least64_t", rffi.INT_LEAST64_T, _WCTSigned)
+    eptype("uint_least8_t", rffi.UINT_LEAST8_T,  _WCTUnsign)
+    eptype("uint_least16_t",rffi.UINT_LEAST16_T, _WCTUnsign)
+    eptype("uint_least32_t",rffi.UINT_LEAST32_T, _WCTUnsign)
+    eptype("uint_least64_t",rffi.UINT_LEAST64_T, _WCTUnsign)
+else:
+    eptypesize("int_least8_t",   1, _WCTSigned)
+    eptypesize("uint_least8_t",  1, _WCTUnsign)
+    eptypesize("int_least16_t",  2, _WCTSigned)
+    eptypesize("uint_least16_t", 2, _WCTUnsign)
+    eptypesize("int_least32_t",  4, _WCTSigned)
+    eptypesize("uint_least32_t", 4, _WCTUnsign)
+    eptypesize("int_least64_t",  8, _WCTSigned)
+    eptypesize("uint_least64_t", 8, _WCTUnsign)
+
+if hasattr(rffi, 'INT_FAST8_T'):
+    eptype("int_fast8_t",  rffi.INT_FAST8_T,  _WCTSigned)
+    eptype("int_fast16_t", rffi.INT_FAST16_T, _WCTSigned)
+    eptype("int_fast32_t", rffi.INT_FAST32_T, _WCTSigned)
+    eptype("int_fast64_t", rffi.INT_FAST64_T, _WCTSigned)
+    eptype("uint_fast8_t", rffi.UINT_FAST8_T,  _WCTUnsign)
+    eptype("uint_fast16_t",rffi.UINT_FAST16_T, _WCTUnsign)
+    eptype("uint_fast32_t",rffi.UINT_FAST32_T, _WCTUnsign)
+    eptype("uint_fast64_t",rffi.UINT_FAST64_T, _WCTUnsign)
+else:
+    eptypesize("int_fast8_t",   1, _WCTSigned)
+    eptypesize("uint_fast8_t",  1, _WCTUnsign)
+    eptypesize("int_fast16_t",  2, _WCTSigned)
+    eptypesize("uint_fast16_t", 2, _WCTUnsign)
+    eptypesize("int_fast32_t",  4, _WCTSigned)
+    eptypesize("uint_fast32_t", 4, _WCTUnsign)
+    eptypesize("int_fast64_t",  8, _WCTSigned)
+    eptypesize("uint_fast64_t", 8, _WCTUnsign)
+
 @unwrap_spec(name=str)
 def new_primitive_type(space, name):
     try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2718,7 +2718,16 @@
 def test_nonstandard_integer_types():
     for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t',
                      'uint32_t', 'int64_t', 'uint64_t', 'intptr_t',
-                     'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t']:
+                     'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t',
+                     'int_least8_t',  'uint_least8_t',
+                     'int_least16_t', 'uint_least16_t',
+                     'int_least32_t', 'uint_least32_t',
+                     'int_least64_t', 'uint_least64_t',
+                     'int_fast8_t',  'uint_fast8_t',
+                     'int_fast16_t', 'uint_fast16_t',
+                     'int_fast32_t', 'uint_fast32_t',
+                     'int_fast64_t', 'uint_fast64_t',
+                     'intmax_t', 'uintmax_t']:
         new_primitive_type(typename)    # works
 
 def test_cannot_convert_unicode_to_charp():
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,4 +1,5 @@
 from rpython.rlib.buffer import Buffer
+from rpython.rtyper.lltypesystem import rffi
 
 # XXX not the most efficient implementation
 
@@ -20,3 +21,7 @@
     def setitem(self, index, char):
         ll_buffer = self.datainstance.ll_buffer
         ll_buffer[index] = char
+
+    def get_raw_address(self):
+        ll_buffer = self.datainstance.ll_buffer
+        return rffi.cast(rffi.CCHARP, ll_buffer)
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1144,6 +1144,15 @@
         b[3] = b'x'
         assert b[3] == b'x'
 
+    def test_pypy_raw_address(self):
+        import _rawffi
+        S = _rawffi.Structure((40, 1))
+        s = S(autofree=True)
+        addr = buffer(s)._pypy_raw_address()
+        assert type(addr) is int
+        assert buffer(s)._pypy_raw_address() == addr
+        assert buffer(s, 10)._pypy_raw_address() == addr + 10
+
     def test_union(self):
         import _rawffi
         longsize = _rawffi.sizeof('l')
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -244,6 +244,9 @@
     def getitem(self, index):
         return self.ptr[index]
 
+    def get_raw_address(self):
+        return rffi.cast(rffi.CCHARP, self.ptr)
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -478,6 +478,15 @@
         for i in range(4):
             assert c[i] == max(a[i], b[i])
 
+
+    def test_abs_overflow(self):
+        from numpy import array, absolute, isinf
+        a = array(complex(1.5e308,1.5e308))
+        # Prints a RuntimeWarning, but does not raise
+        b = absolute(a)
+        assert isinf(b)
+        
+
     def test_basic(self):
         import sys
         from numpy import (dtype, add, array, dtype,
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -665,6 +665,7 @@
 
         assert numpy.int64(9223372036854775807) == 9223372036854775807
         assert numpy.int64(9223372036854775807) == 9223372036854775807
+        assert numpy.int64(-9223372036854775807) == -9223372036854775807
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, 9223372036854775808L)
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1195,7 +1195,11 @@
 
     @complex_to_real_unary_op
     def abs(self, v):
-        return rcomplex.c_abs(v[0], v[1])
+        try:
+            return rcomplex.c_abs(v[0], v[1])
+        except OverflowError:
+            # warning ...
+            return rfloat.INFINITY
 
     @raw_unary_op
     def isnan(self, v):
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -619,6 +619,9 @@
             space.int_w(w_obj)
         except OperationError, e:
             if e.match(space, space.w_OverflowError):
+                if space.is_true(space.le(w_obj, space.wrap(0))):
+                    return find_binop_result_dtype(space, int64_dtype,
+                                               current_guess)
                 return find_binop_result_dtype(space, uint64_dtype,
                                                current_guess)
             raise
diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -286,7 +286,7 @@
 
             t = thread.start_new_thread(pollster.poll, ())
             try:
-                time.sleep(0.1)
+                time.sleep(0.3)
                 for i in range(5): print '',  # to release GIL untranslated
                 # trigger ufds array reallocation
                 for fd in rfds:
@@ -297,7 +297,7 @@
             finally:
                 # and make the call to poll() from the thread return
                 os.write(w, b'spam')
-                time.sleep(0.1)
+                time.sleep(0.3)
                 for i in range(5): print '',  # to release GIL untranslated
         finally:
             os.close(r)
diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -5,7 +5,7 @@
 from rpython.rlib.objectmodel import compute_hash
 
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
 
@@ -123,6 +123,17 @@
         return space.wrap("<%s for 0x%s, size %d>" %
                           (info, addrstring, self.buf.getlength()))
 
+    def descr_pypy_raw_address(self, space):
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        try:
+            ptr = self.buf.get_raw_address()
+        except ValueError:
+            # report the error using the RPython-level internal repr of self.buf
+            msg = ("cannot find the underlying address of buffer that "
+                   "is internally %r" % (self.buf,))
+            raise OperationError(space.w_ValueError, space.wrap(msg))
+        return space.wrap(rffi.cast(lltype.Signed, ptr))
+
 W_Buffer.typedef = TypeDef(
     "buffer",
     __doc__ = """\
@@ -149,5 +160,6 @@
     __mul__ = interp2app(W_Buffer.descr_mul),
     __rmul__ = interp2app(W_Buffer.descr_mul),
     __repr__ = interp2app(W_Buffer.descr_repr),
+    _pypy_raw_address = interp2app(W_Buffer.descr_pypy_raw_address),
 )
 W_Buffer.typedef.acceptable_as_base_class = False
diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py
--- a/pypy/objspace/std/test/test_bufferobject.py
+++ b/pypy/objspace/std/test/test_bufferobject.py
@@ -197,3 +197,9 @@
         buf = buffer('hello world')
         raises(TypeError, "buf[MyInt(0)]")
         raises(TypeError, "buf[MyInt(0):MyInt(5)]")
+
+    def test_pypy_raw_address_base(self):
+        raises(ValueError, buffer("foobar")._pypy_raw_address)
+        raises(ValueError, buffer(u"foobar")._pypy_raw_address)
+        e = raises(ValueError, buffer(bytearray("foobar"))._pypy_raw_address)
+        assert 'BytearrayBuffer' in str(e.value)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -635,7 +635,6 @@
             except KeyError:
                 pass
             else:
-                assert value.getlevel() != LEVEL_CONSTANT
                 assert cur_value.getlevel() != LEVEL_CONSTANT
                 # replacing with a different box
                 cur_value.copy_from(value)
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -115,3 +115,8 @@
             return        # otherwise, adding self.offset might make 'start'
                           # out of bounds
         self.buffer.setslice(self.offset + start, string)
+
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        ptr = self.buffer.get_raw_address()
+        return rffi.ptradd(ptr, self.offset)
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -471,11 +471,21 @@
 except CompilationError:
     pass
 
-_TYPES_ARE_UNSIGNED = set(['size_t', 'uintptr_t'])   # plus "unsigned *"
 if os.name != 'nt':
     TYPES.append('mode_t')
     TYPES.append('pid_t')
     TYPES.append('ssize_t')
+    # the types below are rare enough and not available on Windows
+    TYPES.extend(['ptrdiff_t',
+          'int_least8_t',  'uint_least8_t',
+          'int_least16_t', 'uint_least16_t',
+          'int_least32_t', 'uint_least32_t',
+          'int_least64_t', 'uint_least64_t',
+          'int_fast8_t',  'uint_fast8_t',
+          'int_fast16_t', 'uint_fast16_t',
+          'int_fast32_t', 'uint_fast32_t',
+          'int_fast64_t', 'uint_fast64_t',
+          'intmax_t', 'uintmax_t'])
 else:
     MODE_T = lltype.Signed
     PID_T = lltype.Signed
@@ -489,8 +499,10 @@
         if name.startswith('unsigned'):
             name = 'u' + name[9:]
             signed = False
+        elif name == 'size_t' or name.startswith('uint'):
+            signed = False
         else:
-            signed = (name not in _TYPES_ARE_UNSIGNED)
+            signed = True
         name = name.replace(' ', '')
         names.append(name)
         populatelist.append((name.upper(), c_name, signed))
diff --git a/rpython/rtyper/tool/rfficache.py b/rpython/rtyper/tool/rfficache.py
--- a/rpython/rtyper/tool/rfficache.py
+++ b/rpython/rtyper/tool/rfficache.py
@@ -14,7 +14,7 @@
     from rpython.translator.platform import platform
     includes = ['stdlib.h', 'stdio.h', 'sys/types.h']
     if platform.name != 'msvc':
-        includes += ['inttypes.h']
+        includes += ['inttypes.h', 'stddef.h']
     include_string = "\n".join(["#include <%s>" % i for i in includes])
     c_source = py.code.Source('''
     // includes

From noreply at buildbot.pypy.org  Tue Dec 30 08:58:16 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Dec 2014 08:58:16 +0100 (CET)
Subject: [pypy-commit] pypy default: fix tests
Message-ID: <20141230075816.4E9331C3354@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75156:15909b35f447
Date: 2014-12-30 09:58 +0200
http://bitbucket.org/pypy/pypy/changeset/15909b35f447/

Log:	fix tests

diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -27,7 +27,7 @@
     asm = None
     failargs = ()
 
-    def __init__(self, name, args, res, descr):
+    def __init__(self, name, args, res, descr, failargs=None):
         self.name = name
         self.args = args
         self.res = res
@@ -35,6 +35,7 @@
         self._is_guard = name.startswith('guard_')
         if self._is_guard:
             self.guard_no = int(self.descr[len('<Guard0x'):-1], 16)
+        self.failargs = failargs
 
     def as_json(self):
         d = {
@@ -155,8 +156,8 @@
     def box_for_var(self, res):
         return res
 
-    def create_op(self, opnum, args, res, descr):
-        return self.Op(intern(opname[opnum].lower()), args, res, descr)
+    def create_op(self, opnum, args, res, descr, fail_args):
+        return self.Op(intern(opname[opnum].lower()), args, res, descr, fail_args)
 
 
 

From noreply at buildbot.pypy.org  Tue Dec 30 11:52:28 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Dec 2014 11:52:28 +0100 (CET)
Subject: [pypy-commit] pypy default: prevent merging of those attributes
Message-ID: <20141230105228.EBE4C1C03F4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75157:8e37109904c6
Date: 2014-12-30 12:52 +0200
http://bitbucket.org/pypy/pypy/changeset/8e37109904c6/

Log:	prevent merging of those attributes

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -346,9 +346,7 @@
             if other.level == LEVEL_UNKNOWN:
                 if (box and box.nonnull() and
                         self.known_class.same_constant(cpu.ts.cls_of_box(box))):
-                    op = ResOperation(rop.GUARD_NONNULL, [box], None)
-                    extra_guards.append(op)
-                    op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
+                    op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class], None)
                     extra_guards.append(op)
                     return
                 else:

From noreply at buildbot.pypy.org  Tue Dec 30 11:55:41 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Dec 2014 11:55:41 +0100 (CET)
Subject: [pypy-commit] pypy default: fix tests
Message-ID: <20141230105541.66C2D1C03F4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75158:f5be5d01dac9
Date: 2014-12-30 12:55 +0200
http://bitbucket.org/pypy/pypy/changeset/f5be5d01dac9/

Log:	fix tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -285,8 +285,7 @@
         # knownclass unknown
         expected = """
         [p0]
-        guard_nonnull(p0) []
-        guard_class(p0, ConstClass(node_vtable)) []
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
         """
         self.guards(knownclass_info, unknown_info, unknown_val, expected)
         self.check_invalid(knownclass_info, unknown_info, unknownnull_val)
@@ -390,8 +389,7 @@
         info2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         expected = """
         [p0]
-        guard_nonnull(p0) []        
-        guard_class(p0, ConstClass(node_vtable)) []
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []        
         """
         self.guards(info1, info2, self.nodebox, expected)
         self.check_invalid(info1, info2, BoxPtr())
@@ -444,8 +442,7 @@
 
         expected = """
         [p0]
-        guard_nonnull(p0) []
-        guard_class(p0, ConstClass(node_vtable)) []
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
         """
         state = vstate1.generate_guards(vstate2, [value, value], self.cpu)
         self.compare(state.extra_guards, expected, [self.nodebox])
@@ -480,8 +477,7 @@
 
         expected = """
         [p0]
-        guard_nonnull(p0) []
-        guard_class(p0, ConstClass(node_vtable)) []
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
         """
         self.guards(info1, info2, value1, expected, [self.nodebox])
 
@@ -505,8 +501,7 @@
 
         expected = """
         [p0]
-        guard_nonnull(p0) []
-        guard_class(p0, ConstClass(node_vtable)) []
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
         """
         self.guards(info1, info2, value1, expected, [self.nodebox])
 
@@ -532,8 +527,7 @@
 
         expected = """
         [p0]
-        guard_nonnull(p0) []
-        guard_class(p0, ConstClass(node_vtable)) []
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
         """
         self.guards(info1, info2, value1, expected, [self.nodebox])
 
@@ -560,8 +554,7 @@
 
         expected = """
         [p0]
-        guard_nonnull(p0) []
-        guard_class(p0, ConstClass(node_vtable)) []
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
         """
         self.guards(info1, info2, value1, expected, [self.nodebox])
 

From noreply at buildbot.pypy.org  Tue Dec 30 13:43:09 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Dec 2014 13:43:09 +0100 (CET)
Subject: [pypy-commit] pypy pypy-dont-copy-ops: fixes
Message-ID: <20141230124309.565561C03F4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: pypy-dont-copy-ops
Changeset: r75159:9053d96eece4
Date: 2014-12-30 14:42 +0200
http://bitbucket.org/pypy/pypy/changeset/9053d96eece4/

Log:	fixes

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -130,7 +130,7 @@
     label = ResOperation(rop.LABEL, inputargs, None,
                          descr=TargetToken(jitcell_token))
     end_label = ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)
-    part.operations = [label] + h_ops + [end_label]
+    part.operations = [label] + h_ops[start:] + [end_label]
 
     try:
         start_state = optimize_trace(metainterp_sd, jitdriver_sd, part,
@@ -208,7 +208,7 @@
     h_ops = history.operations
 
     part.operations = [partial_trace.operations[-1]] + \
-                      h_ops + \
+                      h_ops[start:] + \
                       [ResOperation(rop.JUMP, jumpargs, None, descr=loop_jitcell_token)]
     label = part.operations[0]
     orignial_label = label.clone()
@@ -768,8 +768,7 @@
         hidden_all_virtuals = obj.hide(metainterp_sd.cpu)
         metainterp_sd.cpu.set_savedata_ref(deadframe, hidden_all_virtuals)
 
-def invent_fail_descr_for_op(op, optimizer):
-    opnum = op.getopnum()
+def invent_fail_descr_for_op(opnum, optimizer):
     if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
         resumedescr = ResumeGuardForcedDescr()
         resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd)
@@ -837,7 +836,6 @@
     # it does not work -- i.e. none of the existing old_loop_tokens match.
     new_trace = create_empty_loop(metainterp)
     new_trace.inputargs = metainterp.history.inputargs[:]
-    # clone ops, as optimize_bridge can mutate the ops
 
     new_trace.operations = metainterp.history.operations[:]
     metainterp_sd = metainterp.staticdata
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -94,10 +94,12 @@
         else:
             return self._cached_fields.get(structvalue, None)
 
-    def remember_field_value(self, structvalue, fieldvalue, getfield_op=None):
+    def remember_field_value(self, structvalue, fieldvalue, op=None,
+                             optimizer=None):
         assert self._lazy_setfield is None
         self._cached_fields[structvalue] = fieldvalue
-        self._cached_fields_getfield_op[structvalue] = getfield_op
+        op = optimizer.get_op_replacement(op)
+        self._cached_fields_getfield_op[structvalue] = op
 
     def force_lazy_setfield(self, optheap, can_cache=True):
         op = self._lazy_setfield
@@ -121,7 +123,8 @@
             # field.
             structvalue = optheap.getvalue(op.getarg(0))
             fieldvalue = optheap.getvalue(op.getarglist()[-1])
-            self.remember_field_value(structvalue, fieldvalue, op)
+            self.remember_field_value(structvalue, fieldvalue, op,
+                                      optheap.optimizer)
         elif not can_cache:
             self.clear()
 
@@ -446,7 +449,7 @@
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
-        cf.remember_field_value(structvalue, fieldvalue, op)
+        cf.remember_field_value(structvalue, fieldvalue, op, self.optimizer)
 
     def optimize_GETFIELD_GC_PURE(self, op):
         structvalue = self.getvalue(op.getarg(0))
@@ -490,7 +493,7 @@
         # the remember the result of reading the array item
         if cf is not None:
             fieldvalue = self.getvalue(op.result)
-            cf.remember_field_value(arrayvalue, fieldvalue, op)
+            cf.remember_field_value(arrayvalue, fieldvalue, op, self.optimizer)
 
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -239,14 +239,17 @@
     def make_nonnull(self, optimizer):
         assert self.getlevel() < LEVEL_NONNULL
         self.setlevel(LEVEL_NONNULL)
-        self.last_guard_pos = len(optimizer._newoperations)
+        if optimizer is not None:
+            self.last_guard_pos = len(optimizer._newoperations) - 1
+            assert self.get_last_guard(optimizer).is_guard()
 
     def make_constant_class(self, optimizer, classbox):
         assert self.getlevel() < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.setlevel(LEVEL_KNOWNCLASS)
         if optimizer is not None:
-            self.last_guard_pos = len(optimizer._newoperations)
+            self.last_guard_pos = len(optimizer._newoperations) - 1
+            assert self.get_last_guard(optimizer).is_guard()
 
     def import_from(self, other, optimizer):
         OptValue.import_from(self, other, optimizer)
@@ -381,7 +384,7 @@
     def get_last_guard(self, optimizer):
         return None
 
-    def get_known_class(self, optimizer):
+    def get_known_class(self):
         return None
 
     def getlenbound(self):
@@ -602,6 +605,13 @@
         self.ensure_imported(value)
         return value
 
+    def get_box_replacement(self, box):
+        try:
+            v = self.values[box]
+        except KeyError:
+            return box
+        return v.get_key_box()
+
     def ensure_imported(self, value):
         pass
 
@@ -726,9 +736,10 @@
             else:
                 self.ensure_imported(value)
                 newbox = value.force_box(self)
-                if not changed and arg is not newbox:
-                    op = op.clone()
-                    changed = True
+                if arg is not newbox:
+                    if not changed:
+                        op = op.clone()
+                        changed = True
                     op.setarg(i, newbox)
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():
@@ -744,8 +755,25 @@
                 op = self.store_final_boxes_in_guard(guard_op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
+        self._last_emitted_op = orig_op
         self._newoperations.append(op)
 
+    def get_op_replacement(self, op):
+        changed = False
+        for i, arg in enumerate(op.getarglist()):
+            try:
+                v = self.values[arg]
+            except KeyError:
+                pass
+            else:
+                box = v.get_key_box()
+                if box is not arg:
+                    if not changed:
+                        changed = True
+                        op = op.clone()
+                    op.setarg(i, box)
+        return op
+
     def replace_op(self, old_op_pos, new_op):
         old_op = self._newoperations[old_op_pos]
         assert old_op.is_guard()
@@ -760,7 +788,7 @@
             descr = op.getdescr()
             assert isinstance(descr, compile.ResumeAtPositionDescr)
         else:
-            descr = compile.invent_fail_descr_for_op(op, self)
+            descr = compile.invent_fail_descr_for_op(op.getopnum(), self)
             op.setdescr(descr)
         assert isinstance(descr, compile.ResumeGuardDescr)
         assert isinstance(op, GuardResOp)
@@ -799,13 +827,7 @@
         n = op.numargs()
         args = [None] * (n + 2)
         for i in range(n):
-            arg = op.getarg(i)
-            try:
-                value = self.values[arg]
-            except KeyError:
-                pass
-            else:
-                arg = value.get_key_box()
+            arg = self.get_box_replacement(op.getarg(i))
             args[i] = arg
         args[n] = ConstInt(op.getopnum())
         args[n + 1] = op.getdescr()
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -26,6 +26,7 @@
             nextop = None
 
         args = None
+        remember = None
         if canfold:
             for i in range(op.numargs()):
                 if self.get_constant_box(op.getarg(i)) is None:
@@ -45,7 +46,7 @@
                 self.optimizer.make_equal_to(op.result, oldvalue, True)
                 return
             else:
-                self.remember_emitting_pure(op)
+                remember = op
 
         # otherwise, the operation remains
         self.emit_operation(op)
@@ -55,6 +56,8 @@
             self.emit_operation(nextop)
         if args is not None:
             self.pure_operations[args] = self.getvalue(op.result)
+        if remember:
+            self.remember_emitting_pure(remember)
 
     def optimize_CALL_PURE(self, op):
         # Step 1: check if all arguments are constant
@@ -77,12 +80,12 @@
             return
         else:
             self.pure_operations[args] = self.getvalue(op.result)
-            self.remember_emitting_pure(op)
 
         # replace CALL_PURE with just CALL
         args = op.getarglist()
         self.emit_operation(ResOperation(rop.CALL, args, op.result,
                                          op.getdescr()))
+        self.remember_emitting_pure(op)
 
     def optimize_GUARD_NO_EXCEPTION(self, op):
         if self.last_emitted_operation is REMOVED:
@@ -112,6 +115,7 @@
         return self.pure_operations.get(key, None)
 
     def remember_emitting_pure(self, op):
+        op = self.optimizer.get_op_replacement(op)
         self.emitted_pure_operations[op] = True
 
     def produce_potential_short_preamble_ops(self, sb):
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -317,6 +317,7 @@
                         args = [old_guard_op.getarg(0), op.getarg(1)],
                         descr = descr)
             self.optimizer.replace_guard(op, value)
+            descr.make_a_counter_per_value(op)
             # to be safe
             if isinstance(value, PtrOptValue):
                 value.last_guard_pos = -1
@@ -364,8 +365,13 @@
                             args = [old_guard_op.getarg(0), op.getarg(1)],
                             descr=descr)
                 self.optimizer.replace_guard(op, value)
+                # not emitting the guard, so we have to pass None to
+                # make_constant_class, so last_guard_pos is not updated
+                self.emit_operation(op)
+                value.make_constant_class(None, expectedclassbox)
+                return
+        self.emit_operation(op)
         value.make_constant_class(self.optimizer, expectedclassbox)
-        self.emit_operation(op)
 
     def optimize_GUARD_NONNULL_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -125,12 +125,12 @@
 
         ptr = PtrOptValue(BoxPtr())
         nonnull = PtrOptValue(BoxPtr())
-        nonnull.make_nonnull(0)
+        nonnull.make_nonnull(None)
         knownclass = PtrOptValue(BoxPtr())
         clsbox = self.cpu.ts.cls_of_box(BoxPtr(self.myptr))
-        knownclass.make_constant_class(clsbox, 0)
+        knownclass.make_constant_class(None, clsbox)
         const = PtrOptValue(BoxPtr)
-        const.make_constant_class(clsbox, 0)
+        const.make_constant_class(None, clsbox)
         const.make_constant(ConstPtr(self.myptr))
         inorder = [ptr, nonnull, knownclass, const]
         for i in range(len(inorder)):
@@ -181,18 +181,18 @@
 
     def test_known_class_generalization(self):
         knownclass1 = PtrOptValue(BoxPtr())
-        knownclass1.make_constant_class(ConstPtr(self.myptr), 0)
+        knownclass1.make_constant_class(None, ConstPtr(self.myptr))
         info1 = NotVirtualStateInfo(knownclass1)
         info1.position = 0
         knownclass2 = PtrOptValue(BoxPtr())
-        knownclass2.make_constant_class(ConstPtr(self.myptr), 0)
+        knownclass2.make_constant_class(None, ConstPtr(self.myptr))
         info2 = NotVirtualStateInfo(knownclass2)
         info2.position = 0
         self.check_no_guards(info1, info2)
         self.check_no_guards(info2, info1)
 
         knownclass3 = PtrOptValue(BoxPtr())
-        knownclass3.make_constant_class(ConstPtr(self.myptr2), 0)
+        knownclass3.make_constant_class(None, ConstPtr(self.myptr2))
         info3 = NotVirtualStateInfo(knownclass3)
         info3.position = 0
         self.check_invalid(info1, info3)
@@ -221,11 +221,11 @@
 
         knownclass_val = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        knownclass_val.make_constant_class(classbox, -1)
+        knownclass_val.make_constant_class(None, classbox,)
         knownclass_info = NotVirtualStateInfo(knownclass_val)
         knownclass2_val = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        knownclass2_val.make_constant_class(classbox, -1)
+        knownclass2_val.make_constant_class(None, classbox)
         knownclass2_info = NotVirtualStateInfo(knownclass2_val)
 
         constant_val = IntOptValue(BoxInt())
@@ -385,7 +385,7 @@
     def test_known_class(self):
         value1 = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value1.make_constant_class(classbox, -1)
+        value1.make_constant_class(None, classbox)
         info1 = NotVirtualStateInfo(value1)
         info2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         expected = """
@@ -399,7 +399,7 @@
     def test_known_class_value(self):
         value1 = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value1.make_constant_class(classbox, -1)
+        value1.make_constant_class(None, classbox)
         box = self.nodebox
         guards = value1.make_guards(box)
         expected = """
@@ -423,7 +423,7 @@
     def test_equal_inputargs(self):
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         vstate1 = VirtualState([knownclass_info, knownclass_info])
         assert vstate1.generalization_of(vstate1)
@@ -461,7 +461,7 @@
     def test_generate_guards_on_virtual_fields_matches_array(self):
         innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(constclassbox, -1)
+        innervalue1.make_constant_class(None, constclassbox)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
         innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
@@ -488,7 +488,7 @@
     def test_generate_guards_on_virtual_fields_matches_instance(self):
         innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(constclassbox, -1)
+        innervalue1.make_constant_class(None, constclassbox)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
         innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
@@ -513,7 +513,7 @@
     def test_generate_guards_on_virtual_fields_matches_struct(self):
         innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(constclassbox, -1)
+        innervalue1.make_constant_class(None, constclassbox)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
         innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
@@ -540,7 +540,7 @@
     def test_generate_guards_on_virtual_fields_matches_arraystruct(self):
         innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(constclassbox, -1)
+        innervalue1.make_constant_class(None, constclassbox)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
         innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
@@ -572,7 +572,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -600,7 +600,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -609,7 +609,7 @@
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
@@ -622,7 +622,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [10, 20])
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -631,7 +631,7 @@
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
@@ -644,7 +644,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -653,7 +653,7 @@
         info2 = VirtualStateInfo(ConstInt(7), [1, 2])
         value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
@@ -666,7 +666,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -683,7 +683,7 @@
         info1 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -692,7 +692,7 @@
         info2 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
@@ -705,7 +705,7 @@
         info1 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -714,7 +714,7 @@
         info2 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info]
         vstate2 = VirtualState([info2])
@@ -727,7 +727,7 @@
         info1 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -736,7 +736,7 @@
         info2 = VArrayStateInfo(7)
         value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
@@ -749,7 +749,7 @@
         info1 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -766,7 +766,7 @@
     def test_crash_varay_clear(self):
         innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(constclassbox, -1)
+        innervalue1.make_constant_class(None, constclassbox)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
         innerinfo1.position_in_notvirtuals = 0

From noreply at buildbot.pypy.org  Tue Dec 30 13:43:10 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Dec 2014 13:43:10 +0100 (CET)
Subject: [pypy-commit] pypy pypy-dont-copy-ops: close to be merged branch
Message-ID: <20141230124310.689781C03F4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: pypy-dont-copy-ops
Changeset: r75160:dd59dc9171c8
Date: 2014-12-30 14:42 +0200
http://bitbucket.org/pypy/pypy/changeset/dd59dc9171c8/

Log:	close to be merged branch


From noreply at buildbot.pypy.org  Tue Dec 30 13:43:11 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 30 Dec 2014 13:43:11 +0100 (CET)
Subject: [pypy-commit] pypy default: Merge dont-copy-ops - don't make clones
 of ops before sending them to
Message-ID: <20141230124311.93FBD1C03F4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75161:70e5803eaf3c
Date: 2014-12-30 14:42 +0200
http://bitbucket.org/pypy/pypy/changeset/70e5803eaf3c/

Log:	Merge dont-copy-ops - don't make clones of ops before sending them
	to optimizer

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -127,9 +127,10 @@
     part = create_empty_loop(metainterp)
     part.inputargs = inputargs[:]
     h_ops = history.operations
-    part.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(jitcell_token))] + \
-                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
-                      [ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)]
+    label = ResOperation(rop.LABEL, inputargs, None,
+                         descr=TargetToken(jitcell_token))
+    end_label = ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)
+    part.operations = [label] + h_ops[start:] + [end_label]
 
     try:
         start_state = optimize_trace(metainterp_sd, jitdriver_sd, part,
@@ -207,7 +208,7 @@
     h_ops = history.operations
 
     part.operations = [partial_trace.operations[-1]] + \
-                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
+                      h_ops[start:] + \
                       [ResOperation(rop.JUMP, jumpargs, None, descr=loop_jitcell_token)]
     label = part.operations[0]
     orignial_label = label.clone()
@@ -767,8 +768,7 @@
         hidden_all_virtuals = obj.hide(metainterp_sd.cpu)
         metainterp_sd.cpu.set_savedata_ref(deadframe, hidden_all_virtuals)
 
-def invent_fail_descr_for_op(op, optimizer):
-    opnum = op.getopnum()
+def invent_fail_descr_for_op(opnum, optimizer):
     if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
         resumedescr = ResumeGuardForcedDescr()
         resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd)
@@ -836,9 +836,8 @@
     # it does not work -- i.e. none of the existing old_loop_tokens match.
     new_trace = create_empty_loop(metainterp)
     new_trace.inputargs = metainterp.history.inputargs[:]
-    # clone ops, as optimize_bridge can mutate the ops
 
-    new_trace.operations = [op.clone() for op in metainterp.history.operations]
+    new_trace.operations = metainterp.history.operations[:]
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     state = jitdriver_sd.warmstate
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -94,10 +94,12 @@
         else:
             return self._cached_fields.get(structvalue, None)
 
-    def remember_field_value(self, structvalue, fieldvalue, getfield_op=None):
+    def remember_field_value(self, structvalue, fieldvalue, op=None,
+                             optimizer=None):
         assert self._lazy_setfield is None
         self._cached_fields[structvalue] = fieldvalue
-        self._cached_fields_getfield_op[structvalue] = getfield_op
+        op = optimizer.get_op_replacement(op)
+        self._cached_fields_getfield_op[structvalue] = op
 
     def force_lazy_setfield(self, optheap, can_cache=True):
         op = self._lazy_setfield
@@ -121,7 +123,8 @@
             # field.
             structvalue = optheap.getvalue(op.getarg(0))
             fieldvalue = optheap.getvalue(op.getarglist()[-1])
-            self.remember_field_value(structvalue, fieldvalue, op)
+            self.remember_field_value(structvalue, fieldvalue, op,
+                                      optheap.optimizer)
         elif not can_cache:
             self.clear()
 
@@ -446,7 +449,7 @@
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
-        cf.remember_field_value(structvalue, fieldvalue, op)
+        cf.remember_field_value(structvalue, fieldvalue, op, self.optimizer)
 
     def optimize_GETFIELD_GC_PURE(self, op):
         structvalue = self.getvalue(op.getarg(0))
@@ -490,7 +493,7 @@
         # the remember the result of reading the array item
         if cf is not None:
             fieldvalue = self.getvalue(op.result)
-            cf.remember_field_value(arrayvalue, fieldvalue, op)
+            cf.remember_field_value(arrayvalue, fieldvalue, op, self.optimizer)
 
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -75,7 +75,7 @@
         if other.getlevel() == LEVEL_CONSTANT:
             self.make_constant(other.get_key_box())
         elif other.getlevel() == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.get_known_class(), None)
+            self.make_constant_class(None, other.get_known_class())
         else:
             if other.getlevel() == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -193,7 +193,7 @@
         self.box = constbox
         self.setlevel(LEVEL_CONSTANT)
 
-    def get_last_guard(self):
+    def get_last_guard(self, optimizer):
         return None
 
     def get_known_class(self):
@@ -209,10 +209,10 @@
         return None
 
 class PtrOptValue(OptValue):
-    _attrs_ = ('known_class', 'last_guard', 'lenbound')
+    _attrs_ = ('known_class', 'last_guard_pos', 'lenbound')
 
     known_class = None
-    last_guard = None
+    last_guard_pos = -1
     lenbound = None
 
     def __init__(self, box, level=None, known_class=None, intbound=None):
@@ -225,7 +225,7 @@
         self.box = other_value.box
         self.known_class = other_value.known_class
         self._tag = other_value._tag
-        self.last_guard = other_value.last_guard
+        self.last_guard_pos = other_value.last_guard_pos
         self.lenbound = other_value.lenbound
 
     def make_len_gt(self, mode, descr, val):
@@ -236,16 +236,20 @@
         else:
             self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))
 
-    def make_nonnull(self, guardop):
+    def make_nonnull(self, optimizer):
         assert self.getlevel() < LEVEL_NONNULL
         self.setlevel(LEVEL_NONNULL)
-        self.last_guard = guardop
+        if optimizer is not None:
+            self.last_guard_pos = len(optimizer._newoperations) - 1
+            assert self.get_last_guard(optimizer).is_guard()
 
-    def make_constant_class(self, classbox, guardop):
+    def make_constant_class(self, optimizer, classbox):
         assert self.getlevel() < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.setlevel(LEVEL_KNOWNCLASS)
-        self.last_guard = guardop
+        if optimizer is not None:
+            self.last_guard_pos = len(optimizer._newoperations) - 1
+            assert self.get_last_guard(optimizer).is_guard()
 
     def import_from(self, other, optimizer):
         OptValue.import_from(self, other, optimizer)
@@ -300,8 +304,10 @@
     def getlenbound(self):
         return self.lenbound
 
-    def get_last_guard(self):
-        return self.last_guard
+    def get_last_guard(self, optimizer):
+        if self.last_guard_pos == -1:
+            return None
+        return optimizer._newoperations[self.last_guard_pos]
 
     def get_known_class(self):
         return self.known_class
@@ -346,7 +352,7 @@
                 return True
         return False
 
-    def make_nonnull(self, guardop):
+    def make_nonnull(self, optimizer):
         assert self.getlevel() < LEVEL_NONNULL
         self.setlevel(LEVEL_NONNULL)
 
@@ -375,7 +381,7 @@
     def getintbound(self):
         return self.intbound
 
-    def get_last_guard(self):
+    def get_last_guard(self, optimizer):
         return None
 
     def get_known_class(self):
@@ -547,6 +553,12 @@
 
         self.optimizations  = optimizations
 
+    def replace_guard(self, op, value):
+        assert isinstance(value, PtrOptValue)
+        if value.last_guard_pos == -1:
+            return
+        self.replaces_guard[op] = value.last_guard_pos
+
     def force_at_end_of_preamble(self):
         for o in self.optimizations:
             o.force_at_end_of_preamble()
@@ -593,6 +605,13 @@
         self.ensure_imported(value)
         return value
 
+    def get_box_replacement(self, box):
+        try:
+            v = self.values[box]
+        except KeyError:
+            return box
+        return v.get_key_box()
+
     def ensure_imported(self, value):
         pass
 
@@ -706,6 +725,8 @@
     @specialize.argtype(0)
     def _emit_operation(self, op):
         assert op.getopnum() != rop.CALL_PURE
+        changed = False
+        orig_op = op
         for i in range(op.numargs()):
             arg = op.getarg(i)
             try:
@@ -714,37 +735,52 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box(self))
+                newbox = value.force_box(self)
+                if arg is not newbox:
+                    if not changed:
+                        op = op.clone()
+                        changed = True
+                    op.setarg(i, newbox)
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.Counters.OPT_GUARDS)
             pendingfields = self.pendingfields
             self.pendingfields = None
-            if self.replaces_guard and op in self.replaces_guard:
-                self.replace_op(self.replaces_guard[op], op)
+            if self.replaces_guard and orig_op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[orig_op], op)
                 del self.replaces_guard[op]
                 return
             else:
-                op = self.store_final_boxes_in_guard(op, pendingfields)
+                guard_op = op.clone()
+                op = self.store_final_boxes_in_guard(guard_op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
+        self._last_emitted_op = orig_op
         self._newoperations.append(op)
 
-    def replace_op(self, old_op, new_op):
-        # XXX: Do we want to cache indexes to prevent search?
-        i = len(self._newoperations)
-        while i > 0:
-            i -= 1
-            if self._newoperations[i] is old_op:
-                # a bit of dance to make sure we copy all the attrs on
-                # an already emitted descr
-                newdescr = new_op.getdescr()
-                olddescr = old_op.getdescr()
-                newdescr.copy_all_attributes_from(olddescr)
-                self._newoperations[i] = new_op
-                break
-        else:
-            assert False
+    def get_op_replacement(self, op):
+        changed = False
+        for i, arg in enumerate(op.getarglist()):
+            try:
+                v = self.values[arg]
+            except KeyError:
+                pass
+            else:
+                box = v.get_key_box()
+                if box is not arg:
+                    if not changed:
+                        changed = True
+                        op = op.clone()
+                    op.setarg(i, box)
+        return op
+
+    def replace_op(self, old_op_pos, new_op):
+        old_op = self._newoperations[old_op_pos]
+        assert old_op.is_guard()
+        old_descr = old_op.getdescr()
+        new_descr = new_op.getdescr()
+        new_descr.copy_all_attributes_from(old_descr)
+        self._newoperations[old_op_pos] = new_op
 
     def store_final_boxes_in_guard(self, op, pendingfields):
         assert pendingfields is not None
@@ -752,7 +788,7 @@
             descr = op.getdescr()
             assert isinstance(descr, compile.ResumeAtPositionDescr)
         else:
-            descr = compile.invent_fail_descr_for_op(op, self)
+            descr = compile.invent_fail_descr_for_op(op.getopnum(), self)
             op.setdescr(descr)
         assert isinstance(descr, compile.ResumeGuardDescr)
         assert isinstance(op, GuardResOp)
@@ -791,13 +827,7 @@
         n = op.numargs()
         args = [None] * (n + 2)
         for i in range(n):
-            arg = op.getarg(i)
-            try:
-                value = self.values[arg]
-            except KeyError:
-                pass
-            else:
-                arg = value.get_key_box()
+            arg = self.get_box_replacement(op.getarg(i))
             args[i] = arg
         args[n] = ConstInt(op.getopnum())
         args[n + 1] = op.getdescr()
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -26,6 +26,7 @@
             nextop = None
 
         args = None
+        remember = None
         if canfold:
             for i in range(op.numargs()):
                 if self.get_constant_box(op.getarg(i)) is None:
@@ -45,7 +46,7 @@
                 self.optimizer.make_equal_to(op.result, oldvalue, True)
                 return
             else:
-                self.remember_emitting_pure(op)
+                remember = op
 
         # otherwise, the operation remains
         self.emit_operation(op)
@@ -55,6 +56,8 @@
             self.emit_operation(nextop)
         if args is not None:
             self.pure_operations[args] = self.getvalue(op.result)
+        if remember:
+            self.remember_emitting_pure(remember)
 
     def optimize_CALL_PURE(self, op):
         # Step 1: check if all arguments are constant
@@ -77,12 +80,12 @@
             return
         else:
             self.pure_operations[args] = self.getvalue(op.result)
-            self.remember_emitting_pure(op)
 
         # replace CALL_PURE with just CALL
         args = op.getarglist()
         self.emit_operation(ResOperation(rop.CALL, args, op.result,
                                          op.getdescr()))
+        self.remember_emitting_pure(op)
 
     def optimize_GUARD_NO_EXCEPTION(self, op):
         if self.last_emitted_operation is REMOVED:
@@ -112,6 +115,7 @@
         return self.pure_operations.get(key, None)
 
     def remember_emitting_pure(self, op):
+        op = self.optimizer.get_op_replacement(op)
         self.emitted_pure_operations[op] = True
 
     def produce_potential_short_preamble_ops(self, sb):
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -284,7 +284,7 @@
             raise InvalidLoop('A GUARD_NONNULL (%s) was proven to always fail'
                               % r)
         self.emit_operation(op)
-        value.make_nonnull(op)
+        value.make_nonnull(self.optimizer)
 
     def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
@@ -296,11 +296,11 @@
             else:
                 name = "<unknown>"
             raise InvalidLoop('A promote of a virtual %s (a recently allocated object) never makes sense!' % name)
-        if value.get_last_guard():
+        old_guard_op = value.get_last_guard(self.optimizer)
+        if old_guard_op:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = value.get_last_guard()
             if old_guard_op.getopnum() != rop.GUARD_NONNULL:
                 # This is only safe if the class of the guard_value matches the
                 # class of the guard_*_class, otherwise the intermediate ops might
@@ -316,11 +316,11 @@
             op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
                         args = [old_guard_op.getarg(0), op.getarg(1)],
                         descr = descr)
-            self.optimizer.replaces_guard[op] = old_guard_op
+            self.optimizer.replace_guard(op, value)
             descr.make_a_counter_per_value(op)
             # to be safe
             if isinstance(value, PtrOptValue):
-                value.last_guard = None
+                value.last_guard_pos = -1
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
         self.optimize_guard(op, constbox)
@@ -339,7 +339,7 @@
         if realclassbox is not None:
             assert realclassbox.same_constant(expectedclassbox)
             return
-        value.make_constant_class(expectedclassbox, None)
+        value.make_constant_class(None, expectedclassbox)
 
     def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
@@ -353,10 +353,10 @@
             raise InvalidLoop('A GUARD_CLASS (%s) was proven to always fail'
                               % r)
         assert isinstance(value, PtrOptValue)
-        if value.last_guard:
+        old_guard_op = value.get_last_guard(self.optimizer)
+        if old_guard_op:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
@@ -364,9 +364,14 @@
                 op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                             args = [old_guard_op.getarg(0), op.getarg(1)],
                             descr=descr)
-                self.optimizer.replaces_guard[op] = old_guard_op
+                self.optimizer.replace_guard(op, value)
+                # not emitting the guard, so we have to pass None to
+                # make_constant_class, so last_guard_pos is not updated
+                self.emit_operation(op)
+                value.make_constant_class(None, expectedclassbox)
+                return
         self.emit_operation(op)
-        value.make_constant_class(expectedclassbox, op)
+        value.make_constant_class(self.optimizer, expectedclassbox)
 
     def optimize_GUARD_NONNULL_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py
--- a/rpython/jit/metainterp/optimizeopt/simplify.py
+++ b/rpython/jit/metainterp/optimizeopt/simplify.py
@@ -48,6 +48,7 @@
 
     def optimize_JUMP(self, op):
         if not self.unroll:
+            op = op.clone()
             descr = op.getdescr()
             assert isinstance(descr, JitCellToken)
             if not descr.target_tokens:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -125,12 +125,12 @@
 
         ptr = PtrOptValue(BoxPtr())
         nonnull = PtrOptValue(BoxPtr())
-        nonnull.make_nonnull(0)
+        nonnull.make_nonnull(None)
         knownclass = PtrOptValue(BoxPtr())
         clsbox = self.cpu.ts.cls_of_box(BoxPtr(self.myptr))
-        knownclass.make_constant_class(clsbox, 0)
+        knownclass.make_constant_class(None, clsbox)
         const = PtrOptValue(BoxPtr)
-        const.make_constant_class(clsbox, 0)
+        const.make_constant_class(None, clsbox)
         const.make_constant(ConstPtr(self.myptr))
         inorder = [ptr, nonnull, knownclass, const]
         for i in range(len(inorder)):
@@ -181,18 +181,18 @@
 
     def test_known_class_generalization(self):
         knownclass1 = PtrOptValue(BoxPtr())
-        knownclass1.make_constant_class(ConstPtr(self.myptr), 0)
+        knownclass1.make_constant_class(None, ConstPtr(self.myptr))
         info1 = NotVirtualStateInfo(knownclass1)
         info1.position = 0
         knownclass2 = PtrOptValue(BoxPtr())
-        knownclass2.make_constant_class(ConstPtr(self.myptr), 0)
+        knownclass2.make_constant_class(None, ConstPtr(self.myptr))
         info2 = NotVirtualStateInfo(knownclass2)
         info2.position = 0
         self.check_no_guards(info1, info2)
         self.check_no_guards(info2, info1)
 
         knownclass3 = PtrOptValue(BoxPtr())
-        knownclass3.make_constant_class(ConstPtr(self.myptr2), 0)
+        knownclass3.make_constant_class(None, ConstPtr(self.myptr2))
         info3 = NotVirtualStateInfo(knownclass3)
         info3.position = 0
         self.check_invalid(info1, info3)
@@ -221,11 +221,11 @@
 
         knownclass_val = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        knownclass_val.make_constant_class(classbox, -1)
+        knownclass_val.make_constant_class(None, classbox,)
         knownclass_info = NotVirtualStateInfo(knownclass_val)
         knownclass2_val = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        knownclass2_val.make_constant_class(classbox, -1)
+        knownclass2_val.make_constant_class(None, classbox)
         knownclass2_info = NotVirtualStateInfo(knownclass2_val)
 
         constant_val = IntOptValue(BoxInt())
@@ -384,7 +384,7 @@
     def test_known_class(self):
         value1 = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value1.make_constant_class(classbox, -1)
+        value1.make_constant_class(None, classbox)
         info1 = NotVirtualStateInfo(value1)
         info2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
         expected = """
@@ -397,7 +397,7 @@
     def test_known_class_value(self):
         value1 = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value1.make_constant_class(classbox, -1)
+        value1.make_constant_class(None, classbox)
         box = self.nodebox
         guards = value1.make_guards(box)
         expected = """
@@ -421,7 +421,7 @@
     def test_equal_inputargs(self):
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         vstate1 = VirtualState([knownclass_info, knownclass_info])
         assert vstate1.generalization_of(vstate1)
@@ -458,7 +458,7 @@
     def test_generate_guards_on_virtual_fields_matches_array(self):
         innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(constclassbox, -1)
+        innervalue1.make_constant_class(None, constclassbox)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
         innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
@@ -484,7 +484,7 @@
     def test_generate_guards_on_virtual_fields_matches_instance(self):
         innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(constclassbox, -1)
+        innervalue1.make_constant_class(None, constclassbox)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
         innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
@@ -508,7 +508,7 @@
     def test_generate_guards_on_virtual_fields_matches_struct(self):
         innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(constclassbox, -1)
+        innervalue1.make_constant_class(None, constclassbox)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
         innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
@@ -534,7 +534,7 @@
     def test_generate_guards_on_virtual_fields_matches_arraystruct(self):
         innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(constclassbox, -1)
+        innervalue1.make_constant_class(None, constclassbox)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
         innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
@@ -565,7 +565,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -593,7 +593,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -602,7 +602,7 @@
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
@@ -615,7 +615,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [10, 20])
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -624,7 +624,7 @@
         info2 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
@@ -637,7 +637,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -646,7 +646,7 @@
         info2 = VirtualStateInfo(ConstInt(7), [1, 2])
         value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
@@ -659,7 +659,7 @@
         info1 = VirtualStateInfo(ConstInt(42), [1, 2])
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -676,7 +676,7 @@
         info1 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -685,7 +685,7 @@
         info2 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
@@ -698,7 +698,7 @@
         info1 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -707,7 +707,7 @@
         info2 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info]
         vstate2 = VirtualState([info2])
@@ -720,7 +720,7 @@
         info1 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -729,7 +729,7 @@
         info2 = VArrayStateInfo(7)
         value = PtrOptValue(self.nodebox2)
         classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info2.fieldstate = [knownclass_info, knownclass_info]
         vstate2 = VirtualState([info2])
@@ -742,7 +742,7 @@
         info1 = VArrayStateInfo(42)
         value = PtrOptValue(self.nodebox)
         classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(classbox, -1)
+        value.make_constant_class(None, classbox)
         knownclass_info = NotVirtualStateInfo(value)
         info1.fieldstate = [knownclass_info, knownclass_info]
         vstate1 = VirtualState([info1])
@@ -759,7 +759,7 @@
     def test_crash_varay_clear(self):
         innervalue1 = PtrOptValue(self.nodebox)
         constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(constclassbox, -1)
+        innervalue1.make_constant_class(None, constclassbox)
         innerinfo1 = NotVirtualStateInfo(innervalue1)
         innerinfo1.position = 1
         innerinfo1.position_in_notvirtuals = 0
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -135,6 +135,7 @@
                 else:
                     debug_print("Retrace count reached, jumping to preamble")
                     assert cell_token.target_tokens[0].virtual_state is None
+                    jumpop = jumpop.clone()
                     jumpop.setdescr(cell_token.target_tokens[0])
                     self.optimizer.send_extra_operation(jumpop)
                     return
@@ -507,6 +508,7 @@
             self.import_box(a, inputargs, short_jumpargs, jumpargs)
 
     def jump_to_already_compiled_trace(self, jumpop, patchguardop):
+        jumpop = jumpop.clone()
         assert jumpop.getopnum() == rop.JUMP
         cell_token = jumpop.getdescr()
 

From noreply at buildbot.pypy.org  Tue Dec 30 17:22:38 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 17:22:38 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Apply the same changes as the py3k branch
Message-ID: <20141230162238.F15BA1C030C@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75162:0edf9ccb3b63
Date: 2014-12-29 22:34 +0100
http://bitbucket.org/pypy/pypy/changeset/0edf9ccb3b63/

Log:	Apply the same changes as the py3k branch

diff --git a/lib-python/3/test/test_code.py b/lib-python/3/test/test_code.py
--- a/lib-python/3/test/test_code.py
+++ b/lib-python/3/test/test_code.py
@@ -81,7 +81,7 @@
 cellvars: ()
 freevars: ()
 nlocals: 0
-flags: 67
+flags: 1048643
 consts: ("'doc string'", 'None')
 
 >>> def keywordonly_args(a,b,*,k1):
@@ -104,7 +104,7 @@
 
 import unittest
 import weakref
-from test.support import run_doctest, run_unittest, cpython_only
+from test.support import run_doctest, run_unittest, cpython_only, gc_collect
 
 
 def consts(t):
@@ -156,6 +156,7 @@
         coderef = weakref.ref(f.__code__, callback)
         self.assertTrue(bool(coderef()))
         del f
+        gc_collect()
         self.assertFalse(bool(coderef()))
         self.assertTrue(self.called)
 

From noreply at buildbot.pypy.org  Tue Dec 30 17:22:40 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 17:22:40 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Update the _multibytecodec C files with
 the ones from CPython version 3.3.5
Message-ID: <20141230162240.40F431C030C@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75163:342524c121fd
Date: 2014-12-29 23:38 +0100
http://bitbucket.org/pypy/pypy/changeset/342524c121fd/

Log:	Update the _multibytecodec C files with the ones from CPython
	version 3.3.5

diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
--- a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_cn.c
@@ -85,7 +85,7 @@
         TRYMAP_DEC(gb2312, **outbuf, c ^ 0x80, IN2 ^ 0x80) {
             NEXT(2, 1)
         }
-        else return 2;
+        else return 1;
     }
 
     return 0;
@@ -141,7 +141,7 @@
         REQUIRE_INBUF(2)
 
         GBK_DECODE(c, IN2, **outbuf)
-        else return 2;
+        else return 1;
 
         NEXT(2, 1)
     }
@@ -267,7 +267,7 @@
             c3 = IN3;
             c4 = IN4;
             if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39)
-                return 4;
+                return 1;
             c -= 0x81;  c2 -= 0x30;
             c3 -= 0x81; c4 -= 0x30;
 
@@ -292,12 +292,12 @@
                     continue;
                 }
             }
-            return 4;
+            return 1;
         }
 
         GBK_DECODE(c, c2, **outbuf)
         else TRYMAP_DEC(gb18030ext, **outbuf, c, c2);
-        else return 2;
+        else return 1;
 
         NEXT(2, 1)
     }
@@ -400,7 +400,7 @@
             else if (c2 == '\n')
                 ; /* line-continuation */
             else
-                return 2;
+                return 1;
             NEXT(2, 0);
             continue;
         }
@@ -419,7 +419,7 @@
                 NEXT(2, 1)
             }
             else
-                return 2;
+                return 1;
         }
     }
 
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_hk.c b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_hk.c
--- a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_hk.c
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_hk.c
@@ -112,55 +112,56 @@
 
         REQUIRE_INBUF(2)
 
-        if (0xc6 <= c && c <= 0xc8 && (c >= 0xc7 || IN2 >= 0xa1))
-            goto hkscsdec;
+        if (0xc6 > c || c > 0xc8 || (c < 0xc7 && IN2 < 0xa1)) {
+            TRYMAP_DEC(big5, **outbuf, c, IN2) {
+                NEXT(2, 1)
+                continue;
+            }
+        }
 
-        TRYMAP_DEC(big5, **outbuf, c, IN2) {
-            NEXT(2, 1)
+        TRYMAP_DEC(big5hkscs, decoded, c, IN2)
+        {
+            int s = BH2S(c, IN2);
+            const unsigned char *hintbase;
+
+            assert(0x87 <= c && c <= 0xfe);
+            assert(0x40 <= IN2 && IN2 <= 0xfe);
+
+            if (BH2S(0x87, 0x40) <= s && s <= BH2S(0xa0, 0xfe)) {
+                    hintbase = big5hkscs_phint_0;
+                    s -= BH2S(0x87, 0x40);
+            }
+            else if (BH2S(0xc6,0xa1) <= s && s <= BH2S(0xc8,0xfe)){
+                    hintbase = big5hkscs_phint_12130;
+                    s -= BH2S(0xc6, 0xa1);
+            }
+            else if (BH2S(0xf9,0xd6) <= s && s <= BH2S(0xfe,0xfe)){
+                    hintbase = big5hkscs_phint_21924;
+                    s -= BH2S(0xf9, 0xd6);
+            }
+            else
+                    return MBERR_INTERNAL;
+
+            if (hintbase[s >> 3] & (1 << (s & 7))) {
+                    WRITEUCS4(decoded | 0x20000)
+                    NEXT_IN(2)
+            }
+            else {
+                    OUT1(decoded)
+                    NEXT(2, 1)
+            }
+            continue;
         }
-        else
-hkscsdec:       TRYMAP_DEC(big5hkscs, decoded, c, IN2) {
-                        int s = BH2S(c, IN2);
-                        const unsigned char *hintbase;
 
-                        assert(0x87 <= c && c <= 0xfe);
-                        assert(0x40 <= IN2 && IN2 <= 0xfe);
+        switch ((c << 8) | IN2) {
+        case 0x8862: WRITE2(0x00ca, 0x0304); break;
+        case 0x8864: WRITE2(0x00ca, 0x030c); break;
+        case 0x88a3: WRITE2(0x00ea, 0x0304); break;
+        case 0x88a5: WRITE2(0x00ea, 0x030c); break;
+        default: return 1;
+        }
 
-                        if (BH2S(0x87, 0x40) <= s && s <= BH2S(0xa0, 0xfe)) {
-                                hintbase = big5hkscs_phint_0;
-                                s -= BH2S(0x87, 0x40);
-                        }
-                        else if (BH2S(0xc6,0xa1) <= s && s <= BH2S(0xc8,0xfe)){
-                                hintbase = big5hkscs_phint_12130;
-                                s -= BH2S(0xc6, 0xa1);
-                        }
-                        else if (BH2S(0xf9,0xd6) <= s && s <= BH2S(0xfe,0xfe)){
-                                hintbase = big5hkscs_phint_21924;
-                                s -= BH2S(0xf9, 0xd6);
-                        }
-                        else
-                                return MBERR_INTERNAL;
-
-                        if (hintbase[s >> 3] & (1 << (s & 7))) {
-                                WRITEUCS4(decoded | 0x20000)
-                                NEXT_IN(2)
-                        }
-                        else {
-                                OUT1(decoded)
-                                NEXT(2, 1)
-                        }
-                }
-                else {
-                        switch ((c << 8) | IN2) {
-                        case 0x8862: WRITE2(0x00ca, 0x0304); break;
-                        case 0x8864: WRITE2(0x00ca, 0x030c); break;
-                        case 0x88a3: WRITE2(0x00ea, 0x0304); break;
-                        case 0x88a5: WRITE2(0x00ea, 0x030c); break;
-                        default: return 2;
-                        }
-
-                        NEXT(2, 2) /* all decoded codepoints are pairs, above. */
-        }
+        NEXT(2, 2) /* all decoded codepoints are pairs, above. */
     }
 
     return 0;
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_iso2022.c b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_iso2022.c
--- a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_iso2022.c
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_iso2022.c
@@ -123,7 +123,7 @@
 
 CODEC_INIT(iso2022)
 {
-    const struct iso2022_designation *desig = CONFIG_DESIGNATIONS;
+    const struct iso2022_designation *desig;
     for (desig = CONFIG_DESIGNATIONS; desig->mark; desig++)
         if (desig->initializer != NULL && desig->initializer() != 0)
             return -1;
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_jp.c b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_jp.c
--- a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_jp.c
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_jp.c
@@ -112,7 +112,7 @@
         TRYMAP_DEC(cp932ext, **outbuf, c, c2);
         else if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xea)){
             if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc)
-                return 2;
+                return 1;
 
             c = (c < 0xe0 ? c - 0x81 : c - 0xc1);
             c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41);
@@ -120,7 +120,7 @@
             c2 = (c2 < 0x5e ? c2 : c2 - 0x5e) + 0x21;
 
             TRYMAP_DEC(jisx0208, **outbuf, c, c2);
-            else return 2;
+            else return 1;
         }
         else if (c >= 0xf0 && c <= 0xf9) {
             if ((c2 >= 0x40 && c2 <= 0x7e) ||
@@ -128,10 +128,10 @@
                 OUT1(0xe000 + 188 * (c - 0xf0) +
                      (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41))
             else
-                return 2;
+                return 1;
         }
         else
-            return 2;
+            return 1;
 
         NEXT(2, 1)
     }
@@ -256,7 +256,7 @@
                 NEXT(2, 1)
             }
             else
-                return 2;
+                return 1;
         }
         else if (c == 0x8f) {
             unsigned char c2, c3;
@@ -274,7 +274,7 @@
                 continue;
             }
             else TRYMAP_DEC(jisx0212, **outbuf, c2, c3) ;
-            else return 3;
+            else return 1;
             NEXT(3, 1)
         }
         else {
@@ -300,7 +300,7 @@
                 NEXT(2, 2)
                 continue;
             }
-            else return 2;
+            else return 1;
             NEXT(2, 1)
         }
     }
@@ -371,11 +371,11 @@
 
         REQUIRE_OUTBUF(1)
 
-            if (c < 0x80) {
-                OUT1(c)
-                NEXT(1, 1)
-                continue;
-            }
+        if (c < 0x80) {
+            OUT1(c)
+            NEXT(1, 1)
+            continue;
+        }
 
         if (c == 0x8e) {
             /* JIS X 0201 half-width katakana */
@@ -388,7 +388,7 @@
                 NEXT(2, 1)
             }
             else
-                return 2;
+                return 1;
         }
         else if (c == 0x8f) {
             unsigned char c2, c3;
@@ -401,7 +401,7 @@
                 NEXT(3, 1)
             }
             else
-                return 3;
+                return 1;
         }
         else {
             unsigned char c2;
@@ -417,7 +417,7 @@
 #endif
                 TRYMAP_DEC(jisx0208, **outbuf,
                            c ^ 0x80, c2 ^ 0x80) ;
-            else return 2;
+            else return 1;
             NEXT(2, 1)
         }
     }
@@ -502,7 +502,7 @@
             REQUIRE_INBUF(2)
             c2 = IN2;
             if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc)
-                return 2;
+                return 1;
 
             c1 = (c < 0xe0 ? c - 0x81 : c - 0xc1);
             c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41);
@@ -522,10 +522,10 @@
                 continue;
             }
             else
-                return 2;
+                return 1;
         }
         else
-            return 2;
+            return 1;
 
         NEXT(1, 1) /* JIS X 0201 */
     }
@@ -645,7 +645,7 @@
             REQUIRE_INBUF(2)
             c2 = IN2;
             if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc)
-                return 2;
+                return 1;
 
             c1 = (c < 0xe0 ? c - 0x81 : c - 0xc1);
             c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41);
@@ -671,7 +671,7 @@
                     NEXT_OUT(2)
                 }
                 else
-                    return 2;
+                    return 1;
                 NEXT_IN(2)
             }
             else { /* Plane 2 */
@@ -689,13 +689,13 @@
                     continue;
                 }
                 else
-                    return 2;
+                    return 1;
                 NEXT(2, 1)
             }
             continue;
         }
         else
-            return 2;
+            return 1;
 
         NEXT(1, 1) /* JIS X 0201 */
     }
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_kr.c b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_kr.c
--- a/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_kr.c
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/_codecs_kr.c
@@ -123,7 +123,7 @@
             if ((*inbuf)[2] != EUCKR_JAMO_FIRSTBYTE ||
                 (*inbuf)[4] != EUCKR_JAMO_FIRSTBYTE ||
                 (*inbuf)[6] != EUCKR_JAMO_FIRSTBYTE)
-                return 8;
+                return 1;
 
             c = (*inbuf)[3];
             if (0xa1 <= c && c <= 0xbe)
@@ -143,7 +143,7 @@
                 jong = NONE;
 
             if (cho == NONE || jung == NONE || jong == NONE)
-                return 8;
+                return 1;
 
             OUT1(0xac00 + cho*588 + jung*28 + jong);
             NEXT(8, 1)
@@ -152,7 +152,7 @@
             NEXT(2, 1)
         }
         else
-            return 2;
+            return 1;
     }
 
     return 0;
@@ -208,7 +208,7 @@
         REQUIRE_INBUF(2)
         TRYMAP_DEC(ksx1001, **outbuf, c ^ 0x80, IN2 ^ 0x80);
         else TRYMAP_DEC(cp949ext, **outbuf, c, IN2);
-        else return 2;
+        else return 1;
 
         NEXT(2, 1)
     }
@@ -375,7 +375,7 @@
             i_jong = johabidx_jongseong[c_jong];
 
             if (i_cho == NONE || i_jung == NONE || i_jong == NONE)
-                return 2;
+                return 1;
 
             /* we don't use U+1100 hangul jamo yet. */
             if (i_cho == FILL) {
@@ -391,7 +391,7 @@
                         OUT1(0x3100 |
                           johabjamo_jungseong[c_jung])
                     else
-                        return 2;
+                        return 1;
                 }
             } else {
                 if (i_jung == FILL) {
@@ -399,7 +399,7 @@
                         OUT1(0x3100 |
                           johabjamo_choseong[c_cho])
                     else
-                        return 2;
+                        return 1;
                 }
                 else
                     OUT1(0xac00 +
@@ -414,7 +414,7 @@
                 c2 < 0x31 || (c2 >= 0x80 && c2 < 0x91) ||
                 (c2 & 0x7f) == 0x7f ||
                 (c == 0xda && (c2 >= 0xa1 && c2 <= 0xd3)))
-                return 2;
+                return 1;
             else {
                 unsigned char t1, t2;
 
@@ -425,7 +425,7 @@
                 t2 = (t2 < 0x5e ? t2 : t2 - 0x5e) + 0x21;
 
                 TRYMAP_DEC(ksx1001, **outbuf, t1, t2);
-                else return 2;
+                else return 1;
                 NEXT(2, 1)
             }
         }
diff --git a/pypy/module/_multibytecodec/test/test_app_codecs.py b/pypy/module/_multibytecodec/test/test_app_codecs.py
--- a/pypy/module/_multibytecodec/test/test_app_codecs.py
+++ b/pypy/module/_multibytecodec/test/test_app_codecs.py
@@ -30,23 +30,23 @@
         assert e.reason == "incomplete multibyte sequence"
         #
         e = raises(UnicodeDecodeError, codec.decode, b"~{xyz}").value
-        assert e.args == ('hz', b'~{xyz}', 2, 4, 'illegal multibyte sequence')
+        assert e.args == ('hz', b'~{xyz}', 2, 3, 'illegal multibyte sequence')
 
     def test_decode_hz_ignore(self):
         import _codecs_cn
         codec = _codecs_cn.getcodec("hz")
         r = codec.decode(b"def~{}abc", errors='ignore')
-        assert r == ('def\u5fcf', 9)
+        assert r == ('def\u5f95', 9)
         r = codec.decode(b"def~{}abc", 'ignore')
-        assert r == ('def\u5fcf', 9)
+        assert r == ('def\u5f95', 9)
 
     def test_decode_hz_replace(self):
         import _codecs_cn
         codec = _codecs_cn.getcodec("hz")
         r = codec.decode(b"def~{}abc", errors='replace')
-        assert r == ('def\ufffd\u5fcf', 9)
+        assert r == ('def\ufffd\u5f95\ufffd', 9)
         r = codec.decode(b"def~{}abc", 'replace')
-        assert r == ('def\ufffd\u5fcf', 9)
+        assert r == ('def\ufffd\u5f95\ufffd', 9)
 
     def test_decode_custom_error_handler(self):
         import codecs
diff --git a/pypy/module/_multibytecodec/test/test_app_incremental.py b/pypy/module/_multibytecodec/test/test_app_incremental.py
--- a/pypy/module/_multibytecodec/test/test_app_incremental.py
+++ b/pypy/module/_multibytecodec/test/test_app_incremental.py
@@ -21,11 +21,11 @@
             return IncrementalHzEncoder
         """)
         cls.w_IncrementalBig5hkscsEncoder = cls.space.appexec([], """():
-            import _codecs_cn
+            import _codecs_hk
             from _multibytecodec import MultibyteIncrementalEncoder
 
             class IncrementalBig5hkscsEncoder(MultibyteIncrementalEncoder):
-                codec = _codecs_cn.getcodec('big5hkscs')
+                codec = _codecs_hk.getcodec('big5hkscs')
 
             return IncrementalBig5hkscsEncoder
         """)
diff --git a/pypy/module/_multibytecodec/test/test_c_codecs.py b/pypy/module/_multibytecodec/test/test_c_codecs.py
--- a/pypy/module/_multibytecodec/test/test_c_codecs.py
+++ b/pypy/module/_multibytecodec/test/test_c_codecs.py
@@ -80,18 +80,18 @@
     #
     e = py.test.raises(EncodeDecodeError, decode, c, "~{xyz}").value
     assert e.start == 2
-    assert e.end == 4
+    assert e.end == 3
     assert e.reason == "illegal multibyte sequence"
 
 def test_decode_hz_ignore():
     c = getcodec("hz")
     u = decode(c, 'def~{}abc', 'ignore')
-    assert u == u'def\u5fcf'
+    assert u == u'def\u5f95'
 
 def test_decode_hz_replace():
     c = getcodec("hz")
     u = decode(c, 'def~{}abc', 'replace')
-    assert u == u'def\ufffd\u5fcf'
+    assert u == u'def\ufffd\u5f95\ufffd'
 
 def test_encode_hz():
     c = getcodec("hz")

From noreply at buildbot.pypy.org  Tue Dec 30 17:22:41 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 17:22:41 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix test, like in py3k branch
Message-ID: <20141230162241.5D5D61C030C@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75164:a03e2a0dd546
Date: 2014-12-30 16:14 +0100
http://bitbucket.org/pypy/pypy/changeset/a03e2a0dd546/

Log:	Fix test, like in py3k branch

diff --git a/lib-python/3/test/test_fileio.py b/lib-python/3/test/test_fileio.py
--- a/lib-python/3/test/test_fileio.py
+++ b/lib-python/3/test/test_fileio.py
@@ -128,8 +128,8 @@
         self.assertTrue(f.closed)
 
     def testMethods(self):
-        methods = ['fileno', 'isatty', 'read', 'readinto',
-                   'seek', 'tell', 'truncate', 'write', 'seekable',
+        methods = ['fileno', 'isatty', 'read',
+                   'tell', 'truncate', 'seekable',
                    'readable', 'writable']
 
         self.f.close()
@@ -139,6 +139,10 @@
             method = getattr(self.f, methodname)
             # should raise on closed file
             self.assertRaises(ValueError, method)
+        # methods with one argument
+        self.assertRaises(ValueError, self.f.readinto, 0)
+        self.assertRaises(ValueError, self.f.write, 0)
+        self.assertRaises(ValueError, self.f.seek, 0)
 
     def testOpendir(self):
         # Issue 3703: opening a directory should fill the errno

From noreply at buildbot.pypy.org  Tue Dec 30 17:22:42 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 17:22:42 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Avoid crash in OSError.__str__()
Message-ID: <20141230162242.7FE321C030C@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75165:d04014eeb0f2
Date: 2014-12-30 16:33 +0100
http://bitbucket.org/pypy/pypy/changeset/d04014eeb0f2/

Log:	Avoid crash in OSError.__str__()

diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -543,8 +543,10 @@
 
     def _init_error(self, space, args_w, w_errno, w_strerror, w_filename):
         W_BaseException.descr_init(self, space, args_w)
-        self.w_errno = w_errno
-        self.w_strerror = w_strerror
+        if w_errno:
+            self.w_errno = w_errno
+        if w_strerror:
+            self.w_strerror = w_strerror
 
         if not space.is_none(w_filename):
             if space.isinstance_w(

From noreply at buildbot.pypy.org  Tue Dec 30 17:22:43 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 17:22:43 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: regrtest.py has changed,
 and many test modules use the standard unittest
Message-ID: <20141230162243.A09081C030C@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75166:bf3f2a3c98f3
Date: 2014-12-30 17:20 +0100
http://bitbucket.org/pypy/pypy/changeset/bf3f2a3c98f3/

Log:	regrtest.py has changed, and many test modules use the standard
	unittest discovery method. This should get many new failures...

diff --git a/pypy/tool/pytest/run-script/regrverbose.py b/pypy/tool/pytest/run-script/regrverbose.py
--- a/pypy/tool/pytest/run-script/regrverbose.py
+++ b/pypy/tool/pytest/run-script/regrverbose.py
@@ -1,4 +1,4 @@
-# refer to 2.4.1/test/regrtest.py's runtest() for comparison
+# refer to 3/test/regrtest.py's runtest() for comparison
 import sys
 import unittest
 from test import regrtest, support
@@ -10,10 +10,13 @@
 try:
     regrtest.replace_stdout()
     mod = __import__(impname, globals(), locals(), [modname])
-    indirect_test = getattr(mod, 'test_main', None)
-    if indirect_test is not None:
-        indirect_test()
+    # If the test has a test_main, that will run the appropriate
+    # tests.  If not, use normal unittest test loading.
+    test_runner = getattr(mod, "test_main", None)
+    if test_runner is None:
+        tests = unittest.TestLoader().loadTestsFromModule(mod)
+        test_runner = lambda: support.run_unittest(tests)
+    test_runner()
 except unittest.SkipTest:
     sys.stderr.write("="*26 + "skipped" + "="*26 + "\n")
     raise
-# else the test already ran during import

From noreply at buildbot.pypy.org  Tue Dec 30 17:22:50 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 17:22:50 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: hg merge heads
Message-ID: <20141230162250.304B81C030C@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75167:c4e17f979ee1
Date: 2014-12-30 17:21 +0100
http://bitbucket.org/pypy/pypy/changeset/c4e17f979ee1/

Log:	hg merge heads

diff too long, truncating to 2000 out of 15310 lines

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -37,4 +37,4 @@
 to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
-http://doc.pypy.org/en/latest/getting-started.html#installing-pypy
+    http://doc.pypy.org/en/latest/install.html
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -655,6 +655,21 @@
         """Create new Popen instance."""
         _cleanup()
 
+        # --- PyPy hack, see _pypy_install_libs_after_virtualenv() ---
+        # match arguments passed by different versions of virtualenv
+        if args[1:] in (
+            ['-c', 'import sys; print(sys.prefix)'],        # 1.6 10ba3f3c
+            ['-c', "\nimport sys\nprefix = sys.prefix\n"    # 1.7 0e9342ce
+             "if sys.version_info[0] == 3:\n"
+             "    prefix = prefix.encode('utf8')\n"
+             "if hasattr(sys.stdout, 'detach'):\n"
+             "    sys.stdout = sys.stdout.detach()\n"
+             "elif hasattr(sys.stdout, 'buffer'):\n"
+             "    sys.stdout = sys.stdout.buffer\nsys.stdout.write(prefix)\n"],
+            ['-c', 'import sys;out=sys.stdout;getattr(out, "buffer"'
+             ', out).write(sys.prefix.encode("utf-8"))']):  # 1.7.2 a9454bce
+            _pypy_install_libs_after_virtualenv(args[0])
+
         if not isinstance(bufsize, (int, long)):
             raise TypeError("bufsize must be an integer")
 
@@ -1560,6 +1575,27 @@
             self.send_signal(signal.SIGKILL)
 
 
+def _pypy_install_libs_after_virtualenv(target_executable):
+    # https://bitbucket.org/pypy/pypy/issue/1922/future-proofing-virtualenv
+    #
+    # PyPy 2.4.1 turned --shared on by default.  This means the pypy binary
+    # depends on the 'libpypy-c.so' shared library to be able to run.
+    # The virtualenv code existing at the time did not account for this
+    # and would break.  Try to detect that we're running under such a
+    # virtualenv in the "Testing executable with" phase and copy the
+    # library ourselves.
+    caller = sys._getframe(2)
+    if ('virtualenv_version' in caller.f_globals and
+                  'copyfile' in caller.f_globals):
+        dest_dir = sys.pypy_resolvedirof(target_executable)
+        src_dir = sys.pypy_resolvedirof(sys.executable)
+        for libname in ['libpypy-c.so']:
+            dest_library = os.path.join(dest_dir, libname)
+            src_library = os.path.join(src_dir, libname)
+            if os.path.exists(src_library):
+                caller.f_globals['copyfile'](src_library, dest_library)
+
+
 def _demo_posix():
     #
     # Example 1: Simple redirection: Get process list
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -60,7 +60,7 @@
                  skip=None):
         self.basename = basename
         self._usemodules = usemodules.split() + [
-            '_socket', 'binascii', 'rctime',
+            '_socket', 'binascii', 'time',
             'select', 'signal', 'faulthandler']
         if not sys.platform == 'win32':
             self._usemodules.extend(['_posixsubprocess', 'fcntl'])
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -1127,6 +1127,8 @@
 
     if term is None:
         term = ffi.NULL
+    elif isinstance(term, str):
+        term = term.encode('utf-8')
     err = ffi.new("int *")
     if lib.setupterm(term, fd, err) == lib.ERR:
         err = err[0]
diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -72,7 +72,7 @@
         raise TypeError("expected string")
     res = libc.getgrnam(os.fsencode(name))
     if not res:
-        raise KeyError(name)
+        raise KeyError("'getgrnam(): name not found: %s'" % name)
     return _group_from_gstruct(res)
 
 @builtinify
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -30,7 +30,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "rctime" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_minimal_curses",
     "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
@@ -42,7 +42,7 @@
 
 translation_modules = default_modules.copy()
 translation_modules.update([
-    "fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct",
+    "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct",
     "array", "binascii",
     # the following are needed for pyrepl (and hence for the
     # interactive prompt/pdb)
@@ -67,19 +67,15 @@
     default_modules.add("_locale")
 
 if sys.platform == "sunos5":
-    working_modules.remove('mmap')   # depend on ctypes, can't get at c-level 'errono'
-    working_modules.remove('rctime') # depend on ctypes, missing tm_zone/tm_gmtoff
-    working_modules.remove('signal') # depend on ctypes, can't get at c-level 'errono'
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    working_modules.remove("_multiprocessing")   # depends on rctime
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # depends on ctypes
 
 
 module_dependencies = {
-    '_multiprocessing': [('objspace.usemodules.rctime', True),
+    '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
     'cppyy': [('objspace.usemodules.cpyext', True)],
@@ -89,9 +85,10 @@
     # itself needs the interp-level struct module
     # because 'P' is missing from the app-level one
     "_rawffi": [("objspace.usemodules.struct", True)],
-    "cpyext": [("translation.secondaryentrypoints", "cpyext,main"),
-               ("translation.shared", sys.platform == "win32")],
+    "cpyext": [("translation.secondaryentrypoints", "cpyext,main")],
 }
+if sys.platform == "win32":
+    module_suggests["cpyext"].append(("translation.shared", True))
 
 module_import_dependencies = {
     # no _rawffi if importing rpython.rlib.clibffi raises ImportError
diff --git a/pypy/doc/config/objspace.usemodules.rctime.txt b/pypy/doc/config/objspace.usemodules.rctime.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules.rctime.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Use the 'rctime' module. 
-
-'rctime' is our `rffi`_ based implementation of the builtin 'time' module.
-It supersedes the less complete :config:`objspace.usemodules.time`,
-at least for C-like targets (the C and LLVM backends).
-
-.. _`rffi`: ../rffi.html
diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt
--- a/pypy/doc/config/objspace.usemodules.time.txt
+++ b/pypy/doc/config/objspace.usemodules.time.txt
@@ -1,5 +1,1 @@
 Use the 'time' module. 
-
-Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version
-of the application-level 'time' module, at least for C-like targets (the C
-and LLVM backends).
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -205,23 +205,28 @@
 The above is true both in CPython and in PyPy.  Differences
 can occur about whether a built-in function or method will
 call an overridden method of *another* object than ``self``.
-In PyPy, they are generally always called, whereas not in
-CPython.  For example, in PyPy, ``dict1.update(dict2)``
-considers that ``dict2`` is just a general mapping object, and
-will thus call overridden ``keys()``  and ``__getitem__()``
-methods on it.  So the following code prints ``42`` on PyPy
-but ``foo`` on CPython::
+In PyPy, they are often called in cases where CPython would not.
+Two examples::
 
-    >>>> class D(dict):
-    ....     def __getitem__(self, key):
-    ....         return 42
-    ....
-    >>>>
-    >>>> d1 = {}
-    >>>> d2 = D(a='foo')
-    >>>> d1.update(d2)
-    >>>> print d1['a']
-    42
+    class D(dict):
+        def __getitem__(self, key):
+            return "%r from D" % (key,)
+
+    class A(object):
+        pass
+
+    a = A()
+    a.__dict__ = D()
+    a.foo = "a's own foo"
+    print a.foo
+    # CPython => a's own foo
+    # PyPy => 'foo' from D
+
+    glob = D(foo="base item")
+    loc = {}
+    exec "print foo" in glob, loc
+    # CPython => base item
+    # PyPy => 'foo' from D
 
 
 Mutating classes of objects which are already used as dictionary keys
@@ -292,6 +297,9 @@
 above types will return a value that is computed from the argument, and can
 thus be larger than ``sys.maxint`` (i.e. it can be an arbitrary long).
 
+Notably missing from the list above are ``str`` and ``unicode``.  If your
+code relies on comparing strings with ``is``, then it might break in PyPy.
+
 
 Miscellaneous
 -------------
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -39,3 +39,15 @@
 .. branch: kill-multimethod
 
 Kill multimethod machinery, all multimethods were removed earlier.
+
+.. branch nditer-external_loop
+
+Implement `external_loop` arguement to numpy's nditer
+
+.. branch kill-rctime
+
+Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
+
+.. branch: ssa-flow
+
+Use SSA form for flow graphs inside build_flow() and part of simplify_graph()
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -1,15 +1,13 @@
-"""
-Python control flow graph generation and bytecode assembly.
-"""
+"""Python control flow graph generation and bytecode assembly."""
 
-from pypy.interpreter.astcompiler import ast, consts, symtable
-from pypy.interpreter import pycode
+from rpython.rlib import rfloat
+from rpython.rlib.objectmodel import specialize, we_are_translated
+
+from pypy.interpreter.astcompiler import ast, consts, misc, symtable
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pycode import PyCode
 from pypy.tool import stdlib_opcode as ops
 
-from pypy.interpreter.error import OperationError
-from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib import rfloat
-
 
 class Instruction(object):
     """Represents a single opcode."""
@@ -21,14 +19,12 @@
         self.has_jump = False
 
     def size(self):
-        """Return the size of bytes of this instruction when it is encoded."""
+        """Return the size of bytes of this instruction when it is
+        encoded.
+        """
         if self.opcode >= ops.HAVE_ARGUMENT:
-            if self.arg > 0xFFFF:
-                return 6
-            else:
-                return 3
-        else:
-            return 1
+            return (6 if self.arg > 0xFFFF else 3)
+        return 1
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -54,9 +50,9 @@
 class Block(object):
     """A basic control flow block.
 
-    It has one entry point and several possible exit points.  Its instructions
-    may be jumps to other blocks, or if control flow reaches the end of the
-    block, it continues to next_block.
+    It has one entry point and several possible exit points.  Its
+    instructions may be jumps to other blocks, or if control flow
+    reaches the end of the block, it continues to next_block.
     """
 
     def __init__(self):
@@ -71,10 +67,10 @@
             stack.append(nextblock)
 
     def post_order(self):
-        """Return this block and its children in post order.
-        This means that the graph of blocks is first cleaned up to
-        ignore back-edges, thus turning it into a DAG.  Then the DAG
-        is linearized.  For example:
+        """Return this block and its children in post order.  This means
+        that the graph of blocks is first cleaned up to ignore
+        back-edges, thus turning it into a DAG.  Then the DAG is
+        linearized.  For example:
 
                    A --> B -\           =>     [A, D, B, C]
                      \-> D ---> C
@@ -105,7 +101,9 @@
         return resultblocks
 
     def code_size(self):
-        """Return the encoded size of all the instructions in this block."""
+        """Return the encoded size of all the instructions in this
+        block.
+        """
         i = 0
         for instr in self.instructions:
             i += instr.size()
@@ -141,6 +139,7 @@
             i += 1
     return result
 
+
 @specialize.argtype(0)
 def _iter_to_dict(iterable, offset=0):
     result = {}
@@ -302,11 +301,11 @@
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
         last_extended_arg_count = 0
-        # The reason for this loop is extended jumps.  EXTENDED_ARG extends the
-        # bytecode size, so it might invalidate the offsets we've already given.
-        # Thus we have to loop until the number of extended args is stable.  Any
-        # extended jump at all is extremely rare, so performance is not too
-        # concerning.
+        # The reason for this loop is extended jumps.  EXTENDED_ARG
+        # extends the bytecode size, so it might invalidate the offsets
+        # we've already given.  Thus we have to loop until the number of
+        # extended args is stable.  Any extended jump at all is
+        # extremely rare, so performance is not too concerning.
         while True:
             extended_arg_count = 0
             offset = 0
@@ -332,7 +331,8 @@
                                     instr.opcode = ops.JUMP_ABSOLUTE
                                     absolute = True
                                 elif target_op == ops.RETURN_VALUE:
-                                    # Replace JUMP_* to a RETURN into just a RETURN
+                                    # Replace JUMP_* to a RETURN into
+                                    # just a RETURN
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
@@ -347,7 +347,8 @@
                         instr.arg = jump_arg
                         if jump_arg > 0xFFFF:
                             extended_arg_count += 1
-            if extended_arg_count == last_extended_arg_count and not force_redo:
+            if (extended_arg_count == last_extended_arg_count and
+                not force_redo):
                 break
             else:
                 last_extended_arg_count = extended_arg_count
@@ -362,12 +363,14 @@
         while True:
             try:
                 w_key = space.next(w_iter)
-            except OperationError, e:
+            except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise
                 break
             w_index = space.getitem(w_consts, w_key)
-            consts_w[space.int_w(w_index)] = space.getitem(w_key, first)
+            w_constant = space.getitem(w_key, first)
+            w_constant = misc.intern_if_common_string(space, w_constant)
+            consts_w[space.int_w(w_index)] = w_constant
         return consts_w
 
     def _get_code_flags(self):
@@ -435,15 +438,16 @@
                         continue
                     addr = offset - current_off
                     # Python assumes that lineno always increases with
-                    # increasing bytecode address (lnotab is unsigned char).
-                    # Depending on when SET_LINENO instructions are emitted this
-                    # is not always true.  Consider the code:
+                    # increasing bytecode address (lnotab is unsigned
+                    # char).  Depending on when SET_LINENO instructions
+                    # are emitted this is not always true.  Consider the
+                    # code:
                     #     a = (1,
                     #          b)
-                    # In the bytecode stream, the assignment to "a" occurs after
-                    # the loading of "b".  This works with the C Python compiler
-                    # because it only generates a SET_LINENO instruction for the
-                    # assignment.
+                    # In the bytecode stream, the assignment to "a"
+                    # occurs after the loading of "b".  This works with
+                    # the C Python compiler because it only generates a
+                    # SET_LINENO instruction for the assignment.
                     if line or addr:
                         while addr > 255:
                             push(chr(255))
@@ -488,23 +492,23 @@
         # (Only) inherit compilerflags in PyCF_MASK
         flags |= (self.compile_info.flags & consts.PyCF_MASK)
         bytecode = ''.join([block.get_code() for block in blocks])
-        return pycode.PyCode(self.space,
-                             self.argcount,
-                             self.kwonlyargcount,
-                             len(self.var_names),
-                             stack_depth,
-                             flags,
-                             bytecode,
-                             list(consts_w),
-                             names,
-                             var_names,
-                             self.compile_info.filename,
-                             self.name,
-                             self.first_lineno,
-                             lnotab,
-                             free_names,
-                             cell_names,
-                             self.compile_info.hidden_applevel)
+        return PyCode(self.space,
+                      self.argcount,
+                      self.kwonlyargcount,
+                      len(self.var_names),
+                      stack_depth,
+                      flags,
+                      bytecode,
+                      list(consts_w),
+                      names,
+                      var_names,
+                      self.compile_info.filename,
+                      self.name,
+                      self.first_lineno,
+                      lnotab,
+                      free_names,
+                      cell_names,
+                      self.compile_info.hidden_applevel)
 
 
 def _list_from_dict(d, offset=0):
@@ -515,116 +519,116 @@
 
 
 _static_opcode_stack_effects = {
-    ops.NOP : 0,
+    ops.NOP: 0,
 
-    ops.POP_TOP : -1,
-    ops.ROT_TWO : 0,
-    ops.ROT_THREE : 0,
-    ops.DUP_TOP : 1,
-    ops.DUP_TOP_TWO : 2,
+    ops.POP_TOP: -1,
+    ops.ROT_TWO: 0,
+    ops.ROT_THREE: 0,
+    ops.DUP_TOP: 1,
+    ops.DUP_TOP_TWO: 2,
 
-    ops.UNARY_POSITIVE : 0,
-    ops.UNARY_NEGATIVE : 0,
-    ops.UNARY_NOT : 0,
-    ops.UNARY_INVERT : 0,
+    ops.UNARY_POSITIVE: 0,
+    ops.UNARY_NEGATIVE: 0,
+    ops.UNARY_NOT: 0,
+    ops.UNARY_INVERT: 0,
 
-    ops.LIST_APPEND : -1,
-    ops.SET_ADD : -1,
-    ops.MAP_ADD : -2,
-    ops.STORE_MAP : -2,
+    ops.LIST_APPEND: -1,
+    ops.SET_ADD: -1,
+    ops.MAP_ADD: -2,
+    ops.STORE_MAP: -2,
 
-    ops.BINARY_POWER : -1,
-    ops.BINARY_MULTIPLY : -1,
-    ops.BINARY_MODULO : -1,
-    ops.BINARY_ADD : -1,
-    ops.BINARY_SUBTRACT : -1,
-    ops.BINARY_SUBSCR : -1,
-    ops.BINARY_FLOOR_DIVIDE : -1,
-    ops.BINARY_TRUE_DIVIDE : -1,
-    ops.BINARY_LSHIFT : -1,
-    ops.BINARY_RSHIFT : -1,
-    ops.BINARY_AND : -1,
-    ops.BINARY_OR : -1,
-    ops.BINARY_XOR : -1,
+    ops.BINARY_POWER: -1,
+    ops.BINARY_MULTIPLY: -1,
+    ops.BINARY_MODULO: -1,
+    ops.BINARY_ADD: -1,
+    ops.BINARY_SUBTRACT: -1,
+    ops.BINARY_SUBSCR: -1,
+    ops.BINARY_FLOOR_DIVIDE: -1,
+    ops.BINARY_TRUE_DIVIDE: -1,
+    ops.BINARY_LSHIFT: -1,
+    ops.BINARY_RSHIFT: -1,
+    ops.BINARY_AND: -1,
+    ops.BINARY_OR: -1,
+    ops.BINARY_XOR: -1,
 
-    ops.INPLACE_FLOOR_DIVIDE : -1,
-    ops.INPLACE_TRUE_DIVIDE : -1,
-    ops.INPLACE_ADD : -1,
-    ops.INPLACE_SUBTRACT : -1,
-    ops.INPLACE_MULTIPLY : -1,
-    ops.INPLACE_MODULO : -1,
-    ops.INPLACE_POWER : -1,
-    ops.INPLACE_LSHIFT : -1,
-    ops.INPLACE_RSHIFT : -1,
-    ops.INPLACE_AND : -1,
-    ops.INPLACE_OR : -1,
-    ops.INPLACE_XOR : -1,
+    ops.INPLACE_FLOOR_DIVIDE: -1,
+    ops.INPLACE_TRUE_DIVIDE: -1,
+    ops.INPLACE_ADD: -1,
+    ops.INPLACE_SUBTRACT: -1,
+    ops.INPLACE_MULTIPLY: -1,
+    ops.INPLACE_MODULO: -1,
+    ops.INPLACE_POWER: -1,
+    ops.INPLACE_LSHIFT: -1,
+    ops.INPLACE_RSHIFT: -1,
+    ops.INPLACE_AND: -1,
+    ops.INPLACE_OR: -1,
+    ops.INPLACE_XOR: -1,
 
-    ops.STORE_SUBSCR : -2,
-    ops.DELETE_SUBSCR : -2,
+    ops.STORE_SUBSCR: -2,
+    ops.DELETE_SUBSCR: -2,
 
-    ops.GET_ITER : 0,
-    ops.FOR_ITER : 1,
-    ops.BREAK_LOOP : 0,
-    ops.CONTINUE_LOOP : 0,
-    ops.SETUP_LOOP : 0,
+    ops.GET_ITER: 0,
+    ops.FOR_ITER: 1,
+    ops.BREAK_LOOP: 0,
+    ops.CONTINUE_LOOP: 0,
+    ops.SETUP_LOOP: 0,
 
-    ops.PRINT_EXPR : -1,
+    ops.PRINT_EXPR: -1,
 
-    ops.WITH_CLEANUP : -1,
-    ops.LOAD_BUILD_CLASS : 1,
-    ops.STORE_LOCALS : -1,
-    ops.POP_BLOCK : 0,
-    ops.POP_EXCEPT : 0,
-    ops.END_FINALLY : -1,
-    ops.SETUP_WITH : 1,
-    ops.SETUP_FINALLY : 0,
-    ops.SETUP_EXCEPT : 4,
+    ops.WITH_CLEANUP: -1,
+    ops.LOAD_BUILD_CLASS: 1,
+    ops.STORE_LOCALS: -1,
+    ops.POP_BLOCK: 0,
+    ops.POP_EXCEPT: 0,
+    ops.END_FINALLY: -1,
+    ops.SETUP_WITH: 1,
+    ops.SETUP_FINALLY: 0,
+    ops.SETUP_EXCEPT: 4,
 
-    ops.RETURN_VALUE : -1,
-    ops.YIELD_VALUE : 0,
-    ops.YIELD_FROM : -1,
-    ops.BUILD_MAP : 1,
-    ops.BUILD_SET : 1,
-    ops.COMPARE_OP : -1,
+    ops.RETURN_VALUE: -1,
+    ops.YIELD_VALUE: 0,
+    ops.YIELD_FROM: -1,
+    ops.BUILD_MAP: 1,
+    ops.BUILD_SET: 1,
+    ops.COMPARE_OP: -1,
 
-    ops.LOOKUP_METHOD : 1,
+    ops.LOOKUP_METHOD: 1,
 
-    ops.LOAD_NAME : 1,
-    ops.STORE_NAME : -1,
-    ops.DELETE_NAME : 0,
+    ops.LOAD_NAME: 1,
+    ops.STORE_NAME: -1,
+    ops.DELETE_NAME: 0,
 
-    ops.LOAD_FAST : 1,
-    ops.STORE_FAST : -1,
-    ops.DELETE_FAST : 0,
+    ops.LOAD_FAST: 1,
+    ops.STORE_FAST: -1,
+    ops.DELETE_FAST: 0,
 
-    ops.LOAD_ATTR : 0,
-    ops.STORE_ATTR : -2,
-    ops.DELETE_ATTR : -1,
+    ops.LOAD_ATTR: 0,
+    ops.STORE_ATTR: -2,
+    ops.DELETE_ATTR: -1,
 
-    ops.LOAD_GLOBAL : 1,
-    ops.STORE_GLOBAL : -1,
-    ops.DELETE_GLOBAL : 0,
-    ops.DELETE_DEREF : 0,
-
-    ops.LOAD_CLOSURE : 1,
-    ops.LOAD_DEREF : 1,
-    ops.STORE_DEREF : -1,
+    ops.LOAD_GLOBAL: 1,
+    ops.STORE_GLOBAL: -1,
+    ops.DELETE_GLOBAL: 0,
     ops.DELETE_DEREF: 0,
 
-    ops.LOAD_CONST : 1,
+    ops.LOAD_CLOSURE: 1,
+    ops.LOAD_DEREF: 1,
+    ops.STORE_DEREF: -1,
+    ops.DELETE_DEREF: 0,
 
-    ops.IMPORT_STAR : -1,
-    ops.IMPORT_NAME : -1,
-    ops.IMPORT_FROM : 1,
+    ops.LOAD_CONST: 1,
 
-    ops.JUMP_FORWARD : 0,
-    ops.JUMP_ABSOLUTE : 0,
-    ops.JUMP_IF_TRUE_OR_POP : 0,
-    ops.JUMP_IF_FALSE_OR_POP : 0,
-    ops.POP_JUMP_IF_TRUE : -1,
-    ops.POP_JUMP_IF_FALSE : -1,
-    ops.JUMP_IF_NOT_DEBUG : 0,
+    ops.IMPORT_STAR: -1,
+    ops.IMPORT_NAME: -1,
+    ops.IMPORT_FROM: 1,
+
+    ops.JUMP_FORWARD: 0,
+    ops.JUMP_ABSOLUTE: 0,
+    ops.JUMP_IF_TRUE_OR_POP: 0,
+    ops.JUMP_IF_FALSE_OR_POP: 0,
+    ops.POP_JUMP_IF_TRUE: -1,
+    ops.POP_JUMP_IF_FALSE: -1,
+    ops.JUMP_IF_NOT_DEBUG: 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
 }
diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -108,6 +108,15 @@
     return "_%s%s" % (klass[i:], name)
 
 
+def intern_if_common_string(space, w_const):
+    # only intern identifier-like strings
+    from pypy.objspace.std.unicodeobject import _isidentifier
+    if (space.is_w(space.type(w_const), space.w_unicode) and
+        _isidentifier(space.unicode_w(w_const))):
+        return space.new_interned_w_str(w_const)
+    return w_const
+
+
 def new_identifier(space, name):
     # Check whether there are non-ASCII characters in the identifier; if
     # so, normalize to NFKC
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -279,6 +279,11 @@
                 if w_const is None:
                     return tup
                 consts_w[i] = w_const
+            # intern the string constants packed into the tuple here,
+            # because assemble.py will see the result as just a tuple constant
+            for i in range(len(consts_w)):
+                consts_w[i] = misc.intern_if_common_string(
+                    self.space, consts_w[i])
         else:
             consts_w = []
         w_consts = self.space.newtuple(consts_w)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -15,7 +15,7 @@
     UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
-from pypy.interpreter.miscutils import ThreadLocals
+from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
 
 
 __all__ = ['ObjSpace', 'OperationError', 'W_Root']
@@ -362,7 +362,7 @@
         self.builtin_modules = {}
         self.reloading_modules = {}
 
-        self.interned_strings = {}
+        self.interned_strings = make_weak_value_dictionary(self, unicode, W_Root)
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
@@ -501,11 +501,6 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: rctime take precedence over time
-        # XXX this could probably be done as a "requires" in the config
-        if 'rctime' in modules and 'time' in modules:
-            modules.remove('time')
-
         self._builtinmodule_list = modules
         return self._builtinmodule_list
 
@@ -772,26 +767,35 @@
         else:
             return self.w_False
 
-    def new_interned_w_str(self, w_s):
-        s = self.str_w(w_s)
+    def new_interned_w_str(self, w_u):
+        assert isinstance(w_u, W_Root)   # and is not None
+        u = self.unicode_w(w_u)
+        if not we_are_translated():
+            assert type(u) is unicode
+        w_u1 = self.interned_strings.get(u)
+        if w_u1 is None:
+            w_u1 = w_u
+            self.interned_strings.set(u, w_u1)
+        return w_u1
+
+    def new_interned_str(self, s):
+        """Assumes an identifier (utf-8 encoded str)"""
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        self.interned_strings[s] = w_s
-        return w_s
+        u = s.decode('utf-8')
+        w_s1 = self.interned_strings.get(u)
+        if w_s1 is None:
+            w_s1 = self.wrap(u)
+            self.interned_strings.set(u, w_s1)
+        return w_s1
 
-    def new_interned_str(self, s):
+    def is_interned_str(self, s):
+        """Assumes an identifier (utf-8 encoded str)"""
+        # interface for marshal_impl
         if not we_are_translated():
             assert type(s) is str
-        try:
-            return self.interned_strings[s]
-        except KeyError:
-            pass
-        w_s = self.interned_strings[s] = self.wrap(s)
-        return w_s
+        u = s.decode('utf-8')
+        return self.interned_strings.get(u) is not None
 
     def descr_self_interp_w(self, RequiredClass, w_obj):
         if not isinstance(w_obj, RequiredClass):
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -32,6 +32,17 @@
         self.compiler = space.createcompiler()
         self.profilefunc = None
         self.w_profilefuncarg = None
+        self.thread_disappeared = False   # might be set to True after os.fork()
+
+    @staticmethod
+    def _mark_thread_disappeared(space):
+        # Called in the child process after os.fork() by interp_posix.py.
+        # Marks all ExecutionContexts except the current one
+        # with 'thread_disappeared = True'.
+        me = space.getexecutioncontext()
+        for ec in space.threadlocals.getallvalues().values():
+            if ec is not me:
+                ec.thread_disappeared = True
 
     def gettopframe(self):
         return self.topframeref()
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -31,3 +31,19 @@
 
     def getallvalues(self):
         return {0: self._value}
+
+
+def make_weak_value_dictionary(space, keytype, valuetype):
+    "NOT_RPYTHON"
+    if space.config.translation.rweakref:
+        from rpython.rlib.rweakref import RWeakValueDictionary
+        return RWeakValueDictionary(keytype, valuetype)
+    else:
+        class FakeWeakValueDict(object):
+            def __init__(self):
+                self._dict = {}
+            def get(self, key):
+                return self._dict.get(key, None)
+            def set(self, key, value):
+                self._dict[key] = value
+        return FakeWeakValueDict()
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -131,7 +131,6 @@
         # class bodies only have CO_NEWLOCALS.
         # CO_NEWLOCALS: make a locals dict unless optimized is also set
         # CO_OPTIMIZED: no locals dict needed at all
-        # NB: this method is overridden in nestedscope.py
         flags = code.co_flags
         if not (flags & pycode.CO_OPTIMIZED):
             if flags & pycode.CO_NEWLOCALS:
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -1121,7 +1121,13 @@
             sys.stdout = out
         output = s.getvalue()
         assert "CALL_METHOD" in output
-            
+
+    def test_interned_strings(self):
+        source = """x = ('foo_bar42', 5); y = 'foo_bar42'; z = x[0]"""
+        ns = {}
+        exec(source, ns)
+        assert ns['y'] is ns['z']
+
 
 class AppTestExceptions:
     def test_indentation_error(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -378,3 +378,41 @@
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
+
+    def test_interned_strings_are_weak(self):
+        import weakref, gc, random
+        space = self.space
+        assert space.config.translation.rweakref
+        w1 = space.new_interned_str("abcdef")
+        w2 = space.new_interned_str("abcdef")
+        assert w2 is w1
+        #
+        # check that 'w1' goes away if we don't hold a reference to it
+        rw1 = weakref.ref(w1)
+        del w1, w2
+        i = 10
+        while rw1() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
+        #
+        s = "foobar%r" % random.random()
+        w0 = space.wrap(s)
+        w1 = space.new_interned_w_str(w0)
+        assert w1 is w0
+        w2 = space.new_interned_w_str(w0)
+        assert w2 is w0
+        w3 = space.wrap(s)
+        assert w3 is not w0
+        w4 = space.new_interned_w_str(w3)
+        assert w4 is w0
+        #
+        # check that 'w0' goes away if we don't hold a reference to it
+        # (even if we hold a reference to 'w3')
+        rw0 = weakref.ref(w0)
+        del w0, w1, w2, w4
+        i = 10
+        while rw0() is not None:
+            i -= 1
+            assert i >= 0
+            gc.collect()
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -158,21 +158,14 @@
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
-    _attrs_            = ['value_fits_long', 'vmin', 'vrangemax']
-    _immutable_fields_ = ['value_fits_long', 'vmin', 'vrangemax']
+    _attrs_            = ['value_fits_long', 'value_smaller_than_long']
+    _immutable_fields_ = ['value_fits_long', 'value_smaller_than_long']
     is_primitive_integer = True
 
     def __init__(self, *args):
         W_CTypePrimitive.__init__(self, *args)
         self.value_fits_long = self.size <= rffi.sizeof(lltype.Signed)
-        if self.size < rffi.sizeof(lltype.Signed):
-            assert self.value_fits_long
-            sh = self.size * 8
-            self.vmin = r_uint(-1) << (sh - 1)
-            self.vrangemax = (r_uint(1) << sh) - 1
-        else:
-            self.vmin = r_uint(0)
-            self.vrangemax = r_uint(-1)
+        self.value_smaller_than_long = self.size < rffi.sizeof(lltype.Signed)
 
     def cast_to_int(self, cdata):
         return self.convert_to_object(cdata)
@@ -192,8 +185,17 @@
     def convert_from_object(self, cdata, w_ob):
         if self.value_fits_long:
             value = misc.as_long(self.space, w_ob)
-            if self.size < rffi.sizeof(lltype.Signed):
-                if r_uint(value) - self.vmin > self.vrangemax:
+            if self.value_smaller_than_long:
+                size = self.size
+                if size == 1:
+                    signextended = misc.signext(value, 1)
+                elif size == 2:
+                    signextended = misc.signext(value, 2)
+                elif size == 4:
+                    signextended = misc.signext(value, 4)
+                else:
+                    raise AssertionError("unsupported size")
+                if value != signextended:
                     self._overflow(w_ob)
             misc.write_raw_signed_data(cdata, value, self.size)
         else:
@@ -221,7 +223,7 @@
             length = w_cdata.get_array_length()
             populate_list_from_raw_array(res, buf, length)
             return res
-        elif self.value_fits_long:
+        elif self.value_smaller_than_long:
             res = [0] * w_cdata.get_array_length()
             misc.unpack_list_from_raw_array(res, w_cdata._cdata, self.size)
             return res
@@ -235,8 +237,8 @@
                 cdata = rffi.cast(rffi.LONGP, cdata)
                 copy_list_to_raw_array(int_list, cdata)
             else:
-                overflowed = misc.pack_list_to_raw_array_bounds(
-                    int_list, cdata, self.size, self.vmin, self.vrangemax)
+                overflowed = misc.pack_list_to_raw_array_bounds_signed(
+                    int_list, cdata, self.size)
                 if overflowed != 0:
                     self._overflow(self.space.wrap(overflowed))
             return True
@@ -314,8 +316,8 @@
     def pack_list_of_items(self, cdata, w_ob):
         int_list = self.space.listview_int(w_ob)
         if int_list is not None:
-            overflowed = misc.pack_list_to_raw_array_bounds(
-                int_list, cdata, self.size, r_uint(0), self.vrangemax)
+            overflowed = misc.pack_list_to_raw_array_bounds_unsigned(
+                int_list, cdata, self.size, self.vrangemax)
             if overflowed != 0:
                 self._overflow(self.space.wrap(overflowed))
             return True
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -222,6 +222,19 @@
 neg_msg = "can't convert negative number to unsigned"
 ovf_msg = "long too big to convert"
 
+ at specialize.arg(1)
+def signext(value, size):
+    # 'value' is sign-extended from 'size' bytes to a full integer.
+    # 'size' should be a constant smaller than a full integer size.
+    if size == rffi.sizeof(rffi.SIGNEDCHAR):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SIGNEDCHAR, value))
+    elif size == rffi.sizeof(rffi.SHORT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.SHORT, value))
+    elif size == rffi.sizeof(rffi.INT):
+        return rffi.cast(lltype.Signed, rffi.cast(rffi.INT, value))
+    else:
+        raise AssertionError("unsupported size")
+
 # ____________________________________________________________
 
 class _NotStandardObject(Exception):
@@ -339,13 +352,26 @@
 
 # ____________________________________________________________
 
-def pack_list_to_raw_array_bounds(int_list, target, size, vmin, vrangemax):
+def pack_list_to_raw_array_bounds_signed(int_list, target, size):
     for TP, TPP in _prim_signed_types:
         if size == rffi.sizeof(TP):
             ptr = rffi.cast(TPP, target)
             for i in range(len(int_list)):
                 x = int_list[i]
-                if r_uint(x) - vmin > vrangemax:
+                y = rffi.cast(TP, x)
+                if x != rffi.cast(lltype.Signed, y):
+                    return x      # overflow
+                ptr[i] = y
+            return 0
+    raise NotImplementedError("bad integer size")
+
+def pack_list_to_raw_array_bounds_unsigned(int_list, target, size, vrangemax):
+    for TP, TPP in _prim_signed_types:
+        if size == rffi.sizeof(TP):
+            ptr = rffi.cast(TPP, target)
+            for i in range(len(int_list)):
+                x = int_list[i]
+                if r_uint(x) > vrangemax:
                     return x      # overflow
                 ptr[i] = rffi.cast(TP, x)
             return 0
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -168,7 +168,10 @@
         w_import = space.getattr(space.builtin, space.wrap("__import__"))
         space.call_function(w_import, space.wrap("encodings"))
         from pypy.module.sys.interp_encoding import base_encoding
-        space.call_function(w_import, space.wrap("encodings." + base_encoding))
+        # May be 'utf-8'
+        normalized_base = base_encoding.replace("-", "_").lower()
+        space.call_function(w_import, space.wrap("encodings." +
+                                                 normalized_base))
         state.codec_need_encodings = False
         if len(state.codec_search_path) == 0:
             raise OperationError(
diff --git a/pypy/module/_codecs/locale_codec.h b/pypy/module/_codecs/locale_codec.h
--- a/pypy/module/_codecs/locale_codec.h
+++ b/pypy/module/_codecs/locale_codec.h
@@ -2,7 +2,7 @@
 #include <wchar.h>
 #include "src/precommondefs.h"
 
-RPY_EXPORTED_FOR_TESTS wchar_t* pypy_char2wchar(const char* arg, size_t *size);
-RPY_EXPORTED_FOR_TESTS void pypy_char2wchar_free(wchar_t *text);
-RPY_EXPORTED_FOR_TESTS char* pypy_wchar2char(const wchar_t *text, size_t *error_pos);
-RPY_EXPORTED_FOR_TESTS void pypy_wchar2char_free(char *bytes);
+RPY_EXTERN wchar_t* pypy_char2wchar(const char* arg, size_t *size);
+RPY_EXTERN void pypy_char2wchar_free(wchar_t *text);
+RPY_EXTERN char* pypy_wchar2char(const wchar_t *text, size_t *error_pos);
+RPY_EXTERN void pypy_wchar2char_free(char *bytes);
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -1,6 +1,6 @@
 class AppTestCProfile(object):
     spaceconfig = {
-        "usemodules": ['_lsprof', 'rctime'],
+        "usemodules": ['_lsprof', 'time'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -7,7 +7,7 @@
 
 class AppTestMD5(object):
     spaceconfig = {
-        'usemodules': ['_md5', 'binascii', 'rctime', 'struct'],
+        'usemodules': ['_md5', 'binascii', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
--- a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
+++ b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h
@@ -97,24 +97,24 @@
   Py_UNICODE *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_dec_s *pypy_cjk_dec_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_init(struct pypy_cjk_dec_s *d,
                              char *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_dec_free(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_chunk(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_UNICODE *pypy_cjk_dec_outbuf(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_outlen(struct pypy_cjk_dec_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_remaining(struct pypy_cjk_dec_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_inbuf_consumed(struct pypy_cjk_dec_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_dec_replace_on_error(struct pypy_cjk_dec_s* d,
                                          Py_UNICODE *, Py_ssize_t, Py_ssize_t);
 
@@ -125,35 +125,35 @@
   unsigned char *outbuf_start, *outbuf, *outbuf_end;
 };
 
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 struct pypy_cjk_enc_s *pypy_cjk_enc_new(const MultibyteCodec *codec);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_init(struct pypy_cjk_enc_s *d,
                              Py_UNICODE *inbuf, Py_ssize_t inlen);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 void pypy_cjk_enc_free(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_chunk(struct pypy_cjk_enc_s *, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_reset(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 char *pypy_cjk_enc_outbuf(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_outlen(struct pypy_cjk_enc_s *);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_remaining(struct pypy_cjk_enc_s *d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_inbuf_consumed(struct pypy_cjk_enc_s* d);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 Py_ssize_t pypy_cjk_enc_replace_on_error(struct pypy_cjk_enc_s* d,
                                          char *, Py_ssize_t, Py_ssize_t);
-RPY_EXPORTED_FOR_TESTS
+RPY_EXTERN
 const MultibyteCodec *pypy_cjk_enc_getcodec(struct pypy_cjk_enc_s *);
 
 /* list of codecs defined in the .c files */
 
 #define DEFINE_CODEC(name)                              \
-    RPY_EXPORTED_FOR_TESTS MultibyteCodec *pypy_cjkcodec_##name(void);
+    RPY_EXTERN MultibyteCodec *pypy_cjkcodec_##name(void);
 
 // _codecs_cn
 DEFINE_CODEC(gb2312)
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -254,7 +254,7 @@
         start = _GetTickCount()
 
         while True:
-            from pypy.module.rctime.interp_time import State
+            from pypy.module.time.interp_time import State
             interrupt_event = space.fromcache(State).get_interrupt_event()
             handles = [self.handle, interrupt_event]
 
diff --git a/pypy/module/_posixsubprocess/_posixsubprocess.h b/pypy/module/_posixsubprocess/_posixsubprocess.h
--- a/pypy/module/_posixsubprocess/_posixsubprocess.h
+++ b/pypy/module/_posixsubprocess/_posixsubprocess.h
@@ -1,6 +1,6 @@
 #include "src/precommondefs.h"
 
-RPY_EXPORTED_FOR_TESTS void
+RPY_EXTERN void
 pypy_subprocess_child_exec(
            char *const exec_array[],
            char *const argv[],
@@ -17,8 +17,8 @@
            int (*preexec_fn)(void*),
            void *preexec_fn_arg);
 
-RPY_EXPORTED_FOR_TESTS int
+RPY_EXTERN int
 pypy_subprocess_cloexec_pipe(int *fds);
 
-RPY_EXPORTED_FOR_TESTS void
+RPY_EXTERN void
 pypy_subprocess_init(void);
diff --git a/pypy/module/_random/test/test_random.py b/pypy/module/_random/test/test_random.py
--- a/pypy/module/_random/test/test_random.py
+++ b/pypy/module/_random/test/test_random.py
@@ -1,6 +1,6 @@
 class AppTestRandom:
     spaceconfig = {
-        "usemodules": ['_random', 'rctime'],
+        "usemodules": ['_random', 'time'],
     }
 
     def test_dict(self):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -685,7 +685,8 @@
                     raise _ssl_seterror(space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return space.wrap(rffi.charpsize2str(buf_ptr[0], length))
+                    return space.wrapbytes(
+                        rffi.charpsize2str(buf_ptr[0], length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -181,6 +181,14 @@
         self.s.close()
         del ss; gc.collect()
         
+    def test_peer_certificate(self):
+        import gc, ssl
+        ss = ssl.wrap_socket(self.s)
+        assert ss.getpeercert() == {}
+        assert type(ss.getpeercert(True)) is bytes
+        self.s.close()
+        del ss; gc.collect()
+
     def test_npn_protocol(self):
         import socket, _ssl, gc
         ctx = _ssl._SSLContext()
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
--- a/pypy/module/_ssl/thread_lock.py
+++ b/pypy/module/_ssl/thread_lock.py
@@ -24,12 +24,19 @@
 
 separate_module_source = """
 #include <openssl/crypto.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 
 static unsigned int _ssl_locks_count = 0;
 static struct RPyOpaque_ThreadLock *_ssl_locks;
 
 static unsigned long _ssl_thread_id_function(void) {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (unsigned long)GetCurrentThreadId();
+#else
+    return (unsigned long)pthread_self();
+#endif
 }
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
@@ -65,7 +72,7 @@
 eci = rthread.eci.merge(ExternalCompilationInfo(
     separate_module_sources=[separate_module_source],
     post_include_bits=[
-        "RPY_EXPORTED_FOR_TESTS int _PyPy_SSL_SetupThreads(void);"],
+        "RPY_EXTERN int _PyPy_SSL_SetupThreads(void);"],
     libraries = libraries,
 ))
 
diff --git a/pypy/module/bz2/test/test_bz2_file.py b/pypy/module/bz2/test/test_bz2_file.py
--- a/pypy/module/bz2/test/test_bz2_file.py
+++ b/pypy/module/bz2/test/test_bz2_file.py
@@ -54,7 +54,7 @@
     # XXX: CheckAllocation fails on py3 (seems to false positive on
     # BZ2File's RLocks)
     spaceconfig = {
-        'usemodules': ['bz2', 'binascii', 'rctime', 'struct', 'thread']
+        'usemodules': ['bz2', 'binascii', 'time', 'struct', 'thread']
     }
 
     def setup_class(cls):
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -2,6 +2,7 @@
 #define CPPYY_CAPI
 
 #include <stddef.h>
+#include "src/precommondefs.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -15,102 +16,167 @@
     typedef void* (*cppyy_methptrgetter_t)(cppyy_object_t);
 
     /* name to opaque C++ scope representation -------------------------------- */
+    RPY_EXTERN
     int cppyy_num_scopes(cppyy_scope_t parent);
+    RPY_EXTERN
     char* cppyy_scope_name(cppyy_scope_t parent, int iscope);
 
+    RPY_EXTERN
     char* cppyy_resolve_name(const char* cppitem_name);
+    RPY_EXTERN
     cppyy_scope_t cppyy_get_scope(const char* scope_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_get_template(const char* template_name);
+    RPY_EXTERN
     cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj);
 
     /* memory management ------------------------------------------------------ */
+    RPY_EXTERN
     cppyy_object_t cppyy_allocate(cppyy_type_t type);
+    RPY_EXTERN
     void cppyy_deallocate(cppyy_type_t type, cppyy_object_t self);
+    RPY_EXTERN
     void cppyy_destruct(cppyy_type_t type, cppyy_object_t self);
 
     /* method/function dispatching -------------------------------------------- */
+    RPY_EXTERN
     void   cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char   cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     short  cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     int    cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long   cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     float  cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     void*  cppyy_call_r(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
+    RPY_EXTERN
     char*  cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t klass, int nargs, void* args);
+    RPY_EXTERN
     cppyy_object_t cppyy_call_o(cppyy_method_t method, cppyy_object_t self, int nargs, void* args, cppyy_type_t result_type);
 
+    RPY_EXTERN
     cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_scope_t scope, cppyy_index_t idx);
 
     /* handling of function argument buffer ----------------------------------- */
+    RPY_EXTERN
     void*  cppyy_allocate_function_args(int nargs);
+    RPY_EXTERN
     void   cppyy_deallocate_function_args(void* args);
+    RPY_EXTERN
     size_t cppyy_function_arg_sizeof();
+    RPY_EXTERN
     size_t cppyy_function_arg_typeoffset();
 
     /* scope reflection information ------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_namespace(cppyy_scope_t scope);
+    RPY_EXTERN
     int cppyy_is_enum(const char* type_name);
 
     /* class reflection information ------------------------------------------- */
+    RPY_EXTERN
     char* cppyy_final_name(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_scoped_final_name(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_has_complex_hierarchy(cppyy_type_t type);
+    RPY_EXTERN
     int cppyy_num_bases(cppyy_type_t type);
+    RPY_EXTERN
     char* cppyy_base_name(cppyy_type_t type, int base_index);
+    RPY_EXTERN
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
 
     /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */
+    RPY_EXTERN
     ptrdiff_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
 
     /* method/function reflection information --------------------------------- */
+    RPY_EXTERN
     int cppyy_num_methods(cppyy_scope_t scope);
+    RPY_EXTERN
     cppyy_index_t cppyy_method_index_at(cppyy_scope_t scope, int imeth);
+    RPY_EXTERN
     cppyy_index_t* cppyy_method_indices_from_name(cppyy_scope_t scope, const char* name);
 
+    RPY_EXTERN
     char* cppyy_method_name(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_result_type(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_req_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_arg_type(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_arg_default(cppyy_scope_t scope, cppyy_index_t idx, int arg_index);
+    RPY_EXTERN
     char* cppyy_method_signature(cppyy_scope_t scope, cppyy_index_t idx);
 
+    RPY_EXTERN
     int cppyy_method_is_template(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_method_num_template_args(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     char* cppyy_method_template_arg_name(cppyy_scope_t scope, cppyy_index_t idx, cppyy_index_t iarg);
 
+    RPY_EXTERN
     cppyy_method_t cppyy_get_method(cppyy_scope_t scope, cppyy_index_t idx);
+    RPY_EXTERN
     cppyy_index_t cppyy_get_global_operator(
         cppyy_scope_t scope, cppyy_scope_t lc, cppyy_scope_t rc, const char* op);
 
     /* method properties ------------------------------------------------------ */
+    RPY_EXTERN
     int cppyy_is_constructor(cppyy_type_t type, cppyy_index_t idx);
+    RPY_EXTERN
     int cppyy_is_staticmethod(cppyy_type_t type, cppyy_index_t idx);
 
     /* data member reflection information ------------------------------------- */
+    RPY_EXTERN
     int cppyy_num_datamembers(cppyy_scope_t scope);
+    RPY_EXTERN
     char* cppyy_datamember_name(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
+    RPY_EXTERN
     ptrdiff_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
 
+    RPY_EXTERN
     int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 
     /* data member properties ------------------------------------------------- */
+    RPY_EXTERN
     int cppyy_is_publicdata(cppyy_type_t type, int datamember_index);
+    RPY_EXTERN
     int cppyy_is_staticdata(cppyy_type_t type, int datamember_index);
 
     /* misc helpers ----------------------------------------------------------- */
+    RPY_EXTERN
     long long cppyy_strtoll(const char* str);
+    RPY_EXTERN
     unsigned long long cppyy_strtoull(const char* str);
+    RPY_EXTERN
     void cppyy_free(void* ptr);
 
+    RPY_EXTERN
     cppyy_object_t cppyy_charp2stdstring(const char* str);
+    RPY_EXTERN
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
 
 #ifdef __cplusplus
diff --git a/pypy/module/cppyy/src/dummy_backend.cxx b/pypy/module/cppyy/src/dummy_backend.cxx
--- a/pypy/module/cppyy/src/dummy_backend.cxx
+++ b/pypy/module/cppyy/src/dummy_backend.cxx
@@ -1,4 +1,3 @@
-#include "src/precommondefs.h"
 #include "cppyy.h"
 #include "capi.h"
 
@@ -349,29 +348,24 @@
 
 
 /* name to opaque C++ scope representation -------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_scopes(cppyy_scope_t handle) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_resolve_name(const char* cppitem_name) {
     return cppstring_to_cstring(cppitem_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_scope_t cppyy_get_scope(const char* scope_name) {
     return s_handles[scope_name];  // lookup failure will return 0 (== error)
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t /* obj */) {
     return klass;
 }
 
 
 /* memory management ------------------------------------------------------ */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_destruct(cppyy_type_t handle, cppyy_object_t self) {
     if (handle == s_handles["example01"])
        delete (dummy::example01*)self;
@@ -379,7 +373,6 @@
 
 
 /* method/function dispatching -------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_call_v(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long idx = (long)method;
     if (idx == s_methods["static_example01::staticSetPayload_payload*_double"]) {
@@ -469,7 +462,6 @@
     }
 }
 
-RPY_EXPORTED_FOR_TESTS
 unsigned char cppyy_call_b(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     unsigned char result = 0;
     const long idx = (long)method;
@@ -482,7 +474,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char cppyy_call_c(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char result = 0;
     const long idx = (long)method;
@@ -498,7 +489,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 short cppyy_call_h(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     short result = 0;
     const long idx = (long)method; 
@@ -514,7 +504,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_call_i(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     int result = 0;
     const long idx = (long)method;
@@ -547,7 +536,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 long cppyy_call_l(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long result = 0;
     const long idx = (long)method;
@@ -689,7 +677,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_call_ll(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     long long result = 0;
     const long idx = (long)method;
@@ -705,7 +692,6 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 float cppyy_call_f(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     float result = 0;
     const long idx = (long)method;
@@ -718,7 +704,6 @@
     return result;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 double cppyy_call_d(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     double result = 0.;
     const long idx = (long)method;
@@ -740,7 +725,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_call_s(cppyy_method_t method, cppyy_object_t self, int nargs, void* args) {
     char* result = 0;
     const long idx = (long)method;
@@ -753,7 +737,6 @@
     return result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_constructor(cppyy_method_t method, cppyy_type_t handle, int nargs, void* args) {
     void* result = 0;
     const long idx = (long)method;
@@ -776,14 +759,12 @@
     return (cppyy_object_t)result;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_methptrgetter_t cppyy_get_methptr_getter(cppyy_type_t /* handle */, cppyy_index_t /* method_index */) {
     return (cppyy_methptrgetter_t)0;
 }
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void* cppyy_allocate_function_args(int nargs) {
     CPPYY_G__value* args = (CPPYY_G__value*)malloc(nargs*sizeof(CPPYY_G__value));
     for (int i = 0; i < nargs; ++i)
@@ -793,36 +774,30 @@
 
 
 /* handling of function argument buffer ----------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 void cppyy_deallocate_function_args(void* args) {
     free(args);
 }
 
-RPY_EXPORTED_FOR_TESTS
 size_t cppyy_function_arg_sizeof() {
     return sizeof(CPPYY_G__value);
 }
 
-RPY_EXPORTED_FOR_TESTS
 size_t cppyy_function_arg_typeoffset() {
     return offsetof(CPPYY_G__value, type);
 }
 
 
 /* scope reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_namespace(cppyy_scope_t /* handle */) {
     return 0;
 }   
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_enum(const char* /* type_name */) {
     return 0;
 }
     
     
 /* class reflection information ------------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_final_name(cppyy_type_t handle) {
     for (Handles_t::iterator isp = s_handles.begin(); isp != s_handles.end(); ++isp) {
         if (isp->second == handle)
@@ -831,75 +806,61 @@
     return cppstring_to_cstring("<unknown>");
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_scoped_final_name(cppyy_type_t handle) {
     return cppyy_final_name(handle);
 }   
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_has_complex_hierarchy(cppyy_type_t /* handle */) {
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_bases(cppyy_type_t /*handle*/) {
    return 0;
 }
 
 
 /* method/function reflection information --------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_methods(cppyy_scope_t handle) {
     return s_scopes[handle].m_methods.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_index_t cppyy_method_index_at(cppyy_scope_t /* scope */, int imeth) {
     return (cppyy_index_t)imeth;
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_name(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[(int)method_index].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_result_type(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_returntype);
 }
     
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_num_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return s_scopes[handle].m_methods[method_index].m_argtypes.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_req_args(cppyy_scope_t handle, cppyy_index_t method_index) {
     return cppyy_method_num_args(handle, method_index);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_arg_type(cppyy_scope_t handle, cppyy_index_t method_index, int arg_index) {
     return cppstring_to_cstring(s_scopes[handle].m_methods[method_index].m_argtypes[arg_index]);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_arg_default(
         cppyy_scope_t /* handle */, cppyy_index_t /* method_index */, int /* arg_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_method_signature(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return cppstring_to_cstring("");
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_method_is_template(cppyy_scope_t /* handle */, cppyy_index_t /* method_index */) {
     return 0;
 }
     
-RPY_EXPORTED_FOR_TESTS
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end()) {
         long id = s_scopes[handle].m_method_offset + (long)method_index;
@@ -911,7 +872,6 @@
 
 
 /* method properties -----------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_constructor(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kConstructor;
@@ -919,7 +879,6 @@
     return 0;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_staticmethod(cppyy_type_t handle, cppyy_index_t method_index) {
     if (s_scopes.find(handle) != s_scopes.end())
         return s_scopes[handle].m_methods[method_index].m_type == kStatic;
@@ -929,34 +888,28 @@
 
 
 /* data member reflection information ------------------------------------- */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_num_datamembers(cppyy_scope_t handle) {
     return s_scopes[handle].m_datambrs.size();
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_datamember_name(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_name);
 }
 
-RPY_EXPORTED_FOR_TESTS
 char* cppyy_datamember_type(cppyy_scope_t handle, int idatambr) {
     return cppstring_to_cstring(s_scopes[handle].m_datambrs[idatambr].m_type);
 }
 
-RPY_EXPORTED_FOR_TESTS
 ptrdiff_t cppyy_datamember_offset(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_offset;
 }
 
 
 /* data member properties ------------------------------------------------  */
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_publicdata(cppyy_scope_t handle, int idatambr) {
     return 1;
 }
 
-RPY_EXPORTED_FOR_TESTS
 int cppyy_is_staticdata(cppyy_scope_t handle, int idatambr) {
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
@@ -964,44 +917,37 @@
 
 /* misc helpers ----------------------------------------------------------- */
 #if defined(_MSC_VER)
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_strtoll(const char* str) {
     return _strtoi64(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
 unsigned long long cppyy_strtoull(const char* str) {
     return _strtoui64(str, NULL, 0);
 }
 }
 #else
-RPY_EXPORTED_FOR_TESTS
 long long cppyy_strtoll(const char* str) {
     return strtoll(str, NULL, 0);
 }
 
 extern "C" {
-RPY_EXPORTED_FOR_TESTS
 unsigned long long cppyy_strtoull(const char* str) {
     return strtoull(str, NULL, 0);
 }
 }
 #endif
 
-RPY_EXPORTED_FOR_TESTS
 void cppyy_free(void* ptr) {
     free(ptr);
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_charp2stdstring(const char* str) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(str);
     return (cppyy_object_t)arena;
 }
 
-RPY_EXPORTED_FOR_TESTS
 cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr) {
     void* arena = new char[sizeof(std::string)];
     new (arena) std::string(*(std::string*)ptr);
diff --git a/pypy/module/cppyy/test/conftest.py b/pypy/module/cppyy/test/conftest.py
--- a/pypy/module/cppyy/test/conftest.py
+++ b/pypy/module/cppyy/test/conftest.py
@@ -50,7 +50,7 @@
             eci = ExternalCompilationInfo(
                 separate_module_files=[srcpath.join('dummy_backend.cxx')],
                 include_dirs=[incpath, tstpath, cdir],
-                compile_extra=['-DRPY_EXPORTED_FOR_TESTS=RPY_EXPORTED'],
+                compile_extra=['-DRPY_EXTERN=RPY_EXPORTED'],
                 use_cpp_linker=True,
             )
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -779,8 +779,7 @@
     struct PyPyAPI {
     %(members)s
     } _pypyAPI;
-    RPY_EXPORTED_FOR_TESTS
-    struct PyPyAPI* pypyAPI = &_pypyAPI;
+    RPY_EXTERN struct PyPyAPI* pypyAPI = &_pypyAPI;
     """ % dict(members=structmembers)
 
     functions = generate_decls_and_callbacks(db, export_symbols)
@@ -951,7 +950,7 @@
         name_no_star = process_va_name(name)
         header = ('%s pypy_va_get_%s(va_list* vp)' %
                   (name, name_no_star))
-        pypy_decls.append('RPY_EXPORTED_FOR_TESTS ' + header + ';')
+        pypy_decls.append('RPY_EXTERN ' + header + ';')
         functions.append(header + '\n{return va_arg(*vp, %s);}\n' % name)
 
     for name, (typ, expr) in GLOBALS.iteritems():
@@ -1011,7 +1010,7 @@
     if sys.platform == 'win32':
         get_pythonapi_source = '''
         #include <windows.h>
-        RPY_EXPORTED_FOR_TESTS
+        RPY_EXTERN
         HANDLE pypy_get_pythonapi_handle() {
             MEMORY_BASIC_INFORMATION  mi;
             memset(&mi, 0, sizeof(mi));
diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h
--- a/pypy/module/cpyext/include/modsupport.h
+++ b/pypy/module/cpyext/include/modsupport.h
@@ -82,11 +82,20 @@
 /*
  * This is from pyport.h.  Perhaps it belongs elsewhere.
  */
+#ifdef _WIN32
+/* explicitly export since PyAPI_FUNC is usually dllimport */
+#ifdef __cplusplus
+#define PyMODINIT_FUNC extern "C" __declspec(dllexport) void
+#else
+#define PyMODINIT_FUNC __declspec(dllexport) void
+#endif
+#else
 #ifdef __cplusplus
 #define PyMODINIT_FUNC extern "C" PyAPI_FUNC(PyObject *)
 #else
 #define PyMODINIT_FUNC PyAPI_FUNC(PyObject *)
 #endif
+#endif /* WIN32 */
 
 PyAPI_DATA(char *) _Py_PackageContext;
 
diff --git a/pypy/module/cpyext/src/pythread.c b/pypy/module/cpyext/src/pythread.c
--- a/pypy/module/cpyext/src/pythread.c
+++ b/pypy/module/cpyext/src/pythread.c
@@ -1,11 +1,18 @@
 #include <Python.h>
+#ifndef _WIN32
+# include <pthread.h>
+#endif
 #include "pythread.h"
 #include "src/thread.h"
 
 long
 PyThread_get_thread_ident(void)
 {
-    return RPyThreadGetIdent();
+#ifdef _WIN32
+    return (long)GetCurrentThreadId();
+#else
+    return (long)pthread_self();
+#endif
 }
 
 PyThread_type_lock
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -7,7 +7,7 @@
     # it's necessary to run "import time" at least once before any
     # other cpyext test, otherwise the same statement will fail in
     # test_datetime.py.
-    space = gettestobjspace(usemodules=['rctime'])
+    space = gettestobjspace(usemodules=['time'])
     space.getbuiltinmodule("time")
 
 def pytest_ignore_collect(path, config):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -102,7 +102,7 @@
 class LeakCheckingTest(object):
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
-                                   'itertools', 'rctime', 'binascii', 'micronumpy'])
+                                   'itertools', 'time', 'binascii', 'micronumpy'])
     spaceconfig['std.withmethodcache'] = True
 
     enable_leak_checking = True
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -12,7 +12,7 @@
 
 class AppTestFcntl:
     spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios',
-                                   'select', 'rctime'))
+                                   'select', 'time'))
 
     def setup_class(cls):
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -4,7 +4,7 @@
 
 class AppTestImpModule:
     spaceconfig = {
-        'usemodules': ['binascii', 'imp', 'itertools', 'rctime', 'struct'],
+        'usemodules': ['binascii', 'imp', 'itertools', 'time', 'struct'],
     }
 
     def setup_class(cls):
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -163,7 +163,7 @@
 
 class AppTestImport(BaseImportTest):
     spaceconfig = {
-        "usemodules": ['rctime'],
+        "usemodules": ['time'],
     }
 
     def setup_class(cls):
@@ -1049,7 +1049,7 @@
 
 class AppTestImportHooks(object):
     spaceconfig = {
-        "usemodules": ['struct', 'itertools', 'rctime'],
+        "usemodules": ['struct', 'itertools', 'time'],
     }
 
     def setup_class(cls):
@@ -1266,7 +1266,7 @@
 
 
 class AppTestMultithreadedImp(object):
-    spaceconfig = dict(usemodules=['thread', 'rctime'])
+    spaceconfig = dict(usemodules=['thread', 'time'])
 
     def setup_class(cls):
         #if not conftest.option.runappdirect:
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -152,7 +152,6 @@
     atom_int(tc, int)           puts code and int
     atom_int64(tc, int64)       puts code and int64

From noreply at buildbot.pypy.org  Tue Dec 30 18:02:49 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 18:02:49 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix compilation, I think
Message-ID: <20141230170249.D77F51C1066@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75168:4c7c31a7377d
Date: 2014-12-30 18:02 +0100
http://bitbucket.org/pypy/pypy/changeset/4c7c31a7377d/

Log:	Fix compilation, I think

diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h
--- a/pypy/module/faulthandler/faulthandler.h
+++ b/pypy/module/faulthandler/faulthandler.h
@@ -4,16 +4,16 @@
 #include <signal.h>
 #include "src/precommondefs.h"
 
-RPY_EXPORTED_FOR_TESTS int pypy_faulthandler_read_null(void);
-RPY_EXPORTED_FOR_TESTS void pypy_faulthandler_sigsegv(void);
-RPY_EXPORTED_FOR_TESTS int pypy_faulthandler_sigfpe(void);
-RPY_EXPORTED_FOR_TESTS void pypy_faulthandler_sigabrt();
+RPY_EXTERN int pypy_faulthandler_read_null(void);
+RPY_EXTERN void pypy_faulthandler_sigsegv(void);
+RPY_EXTERN int pypy_faulthandler_sigfpe(void);
+RPY_EXTERN void pypy_faulthandler_sigabrt();
 #ifdef SIGBUS
-RPY_EXPORTED_FOR_TESTS void pypy_faulthandler_sigbus(void);
+RPY_EXTERN void pypy_faulthandler_sigbus(void);
 #endif
 
 #ifdef SIGILL
-RPY_EXPORTED_FOR_TESTS void pypy_faulthandler_sigill(void);
+RPY_EXTERN void pypy_faulthandler_sigill(void);
 #endif
 
 #endif  /* PYPY_FAULTHANDLER_H */

From noreply at buildbot.pypy.org  Tue Dec 30 20:44:35 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Tue, 30 Dec 2014 20:44:35 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: rctime -> time
Message-ID: <20141230194435.DDD8C1D34DD@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.3
Changeset: r75169:f5ffe71d0a97
Date: 2014-12-30 11:44 -0800
http://bitbucket.org/pypy/pypy/changeset/f5ffe71d0a97/

Log:	rctime -> time

diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py b/pypy/module/_posixsubprocess/test/test_subprocess.py
--- a/pypy/module/_posixsubprocess/test/test_subprocess.py
+++ b/pypy/module/_posixsubprocess/test/test_subprocess.py
@@ -2,7 +2,7 @@
 
 class AppTestSubprocess:
     spaceconfig = dict(usemodules=('_posixsubprocess', 'signal',
-                                   'fcntl', 'select', 'rctime'))
+                                   'fcntl', 'select', 'time'))
     # XXX write more tests
 
     def setup_class(cls):
diff --git a/pypy/module/bz2/test/test_bz2_compdecomp.py b/pypy/module/bz2/test/test_bz2_compdecomp.py
--- a/pypy/module/bz2/test/test_bz2_compdecomp.py
+++ b/pypy/module/bz2/test/test_bz2_compdecomp.py
@@ -41,7 +41,7 @@
     interp_bz2.SMALLCHUNK = mod.OLD_SMALLCHUNK
 
 class AppTestBZ2Compressor(CheckAllocation):
-    spaceconfig = dict(usemodules=('bz2', 'rctime'))
+    spaceconfig = dict(usemodules=('bz2', 'time'))
 
     def setup_class(cls):
         cls.w_TEXT = cls.space.wrapbytes(TEXT)
@@ -117,7 +117,7 @@
 
 
 class AppTestBZ2Decompressor(CheckAllocation):
-    spaceconfig = dict(usemodules=('bz2', 'rctime'))
+    spaceconfig = dict(usemodules=('bz2', 'time'))
 
     def setup_class(cls):
         cls.w_TEXT = cls.space.wrapbytes(TEXT)
@@ -202,7 +202,7 @@
 
 
 class AppTestBZ2ModuleFunctions(CheckAllocation):
-    spaceconfig = dict(usemodules=('bz2', 'rctime'))
+    spaceconfig = dict(usemodules=('bz2', 'time'))
 
     def setup_class(cls):
         cls.w_TEXT = cls.space.wrapbytes(TEXT)

From noreply at buildbot.pypy.org  Tue Dec 30 22:27:12 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 22:27:12 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix tests to pass with -A and CPython3.3
Message-ID: <20141230212712.46F721C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75170:edc9c50db790
Date: 2014-12-30 19:30 +0100
http://bitbucket.org/pypy/pypy/changeset/edc9c50db790/

Log:	Fix tests to pass with -A and CPython3.3

diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -838,7 +838,8 @@
         assert getattr(T, '\u03bc') == 2
         assert getattr(T, '\u87d2') == 3
         #assert getattr(T, 'x\U000E0100') == 4
-        expected = ("['__dict__', '__doc__', '__module__', '__weakref__', "
+        expected = ("['__dict__', '__doc__', '__module__', "
+                    "'__qualname__', '__weakref__', "
         #            "x&#917760;", "'&#228;', '&#956;', '&#34770;']")
                     "'&#228;', '&#956;', '&#34770;']")
         assert expected in str(sorted(T.__dict__.keys()))
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -89,7 +89,7 @@
         import os
 
         assert sys.__package__ is None
-        assert os.__package__ is None
+        assert os.__package__ == ''
         assert not hasattr(type(sys)('foo'), '__package__')
 
     def test_name_nonascii(self):

From noreply at buildbot.pypy.org  Tue Dec 30 22:27:13 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 22:27:13 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Kill the interp-level version of the
	_lzma module, 
Message-ID: <20141230212713.951101C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75171:db1b4ba4d7e5
Date: 2014-12-30 19:48 +0100
http://bitbucket.org/pypy/pypy/changeset/db1b4ba4d7e5/

Log:	Kill the interp-level version of the _lzma module, and import the
	one from lzmaffi on PyPI.

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_lzma.py
@@ -0,0 +1,920 @@
+# This file was copied from lzmaffi version 0.3.0.
+# It is an almost exact copy of lzmaffi/_lzmamodule2.py.
+
+# PyPy changes:
+# - added __getstate__() methods that raise TypeError on pickling.
+
+from cffi import FFI
+import threading
+import functools
+import collections
+import weakref
+import sys
+import io
+
+SUPPORTED_STREAM_FLAGS_VERSION = 0
+
+__all__ = ['CHECK_CRC32',
+ 'CHECK_CRC64',
+ 'CHECK_ID_MAX',
+ 'CHECK_NONE',
+ 'CHECK_SHA256',
+ 'CHECK_UNKNOWN',
+ 'FILTER_ARM',
+ 'FILTER_ARMTHUMB',
+ 'FILTER_DELTA',
+ 'FILTER_IA64',
+ 'FILTER_LZMA1',
+ 'FILTER_LZMA2',
+ 'FILTER_POWERPC',
+ 'FILTER_SPARC',
+ 'FILTER_X86',
+ 'FORMAT_ALONE',
+ 'FORMAT_AUTO',
+ 'FORMAT_RAW',
+ 'FORMAT_XZ',
+ 'FORMAT_BLOCK',
+ 'LZMACompressor',
+ 'LZMADecompressor',
+ 'LZMAError',
+ 'MF_BT2',
+ 'MF_BT3',
+ 'MF_BT4',
+ 'MF_HC3',
+ 'MF_HC4',
+ 'MODE_FAST',
+ 'MODE_NORMAL',
+ 'PRESET_DEFAULT',
+ 'PRESET_EXTREME',
+ 'STREAM_HEADER_SIZE',
+ 'decode_block_header_size',
+ 'decode_stream_header',
+ 'decode_stream_footer',
+ 'decode_index',
+ '_decode_filter_properties',
+ '_encode_filter_properties',
+ 'is_check_supported']
+
+_owns = weakref.WeakKeyDictionary()
+
+ffi = FFI()
+ffi.cdef("""
+#define UINT64_MAX ...
+#define LZMA_CONCATENATED ...
+#define LZMA_CHECK_NONE ...
+#define LZMA_CHECK_CRC32 ...
+#define LZMA_CHECK_CRC64 ...
+#define LZMA_CHECK_SHA256 ...
+#define LZMA_CHECK_ID_MAX ...
+#define LZMA_DELTA_TYPE_BYTE ...
+#define LZMA_TELL_ANY_CHECK ...
+#define LZMA_TELL_NO_CHECK ...
+#define LZMA_VLI_UNKNOWN ...
+#define LZMA_FILTER_LZMA1 ...
+#define LZMA_FILTER_LZMA2 ...
+#define LZMA_FILTER_DELTA ...
+#define LZMA_FILTER_X86 ...
+#define LZMA_FILTER_IA64 ...
+#define LZMA_FILTER_ARM ...
+#define LZMA_FILTER_ARMTHUMB ...
+#define LZMA_FILTER_SPARC ...
+#define LZMA_FILTER_POWERPC ...
+#define LZMA_FILTERS_MAX ...
+#define LZMA_STREAM_HEADER_SIZE ...
+#define LZMA_MF_HC3 ...
+#define LZMA_MF_HC4 ...
+#define LZMA_MF_BT2 ...
+#define LZMA_MF_BT3 ...
+#define LZMA_MF_BT4 ...
+#define LZMA_MODE_FAST ...
+#define LZMA_MODE_NORMAL ...
+#define LZMA_PRESET_DEFAULT ...
+#define LZMA_PRESET_EXTREME ...
+
+enum lzma_ret { LZMA_OK, LZMA_STREAM_END, LZMA_NO_CHECK,
+    LZMA_UNSUPPORTED_CHECK, LZMA_GET_CHECK,
+    LZMA_MEM_ERROR, LZMA_MEMLIMIT_ERROR,
+    LZMA_FORMAT_ERROR, LZMA_OPTIONS_ERROR,
+    LZMA_DATA_ERROR, LZMA_BUF_ERROR,
+    LZMA_PROG_ERROR, ... };
+
+enum lzma_action { LZMA_RUN, LZMA_FINISH, ...};
+
+enum lzma_check { ... };
+
+typedef uint64_t lzma_vli;
+
+typedef struct {
+    void* (*alloc)(void*, size_t, size_t);
+    void (*free)(void*, void*);
+    void* opaque;
+    ...;
+} lzma_allocator;
+
+typedef struct {
+    const uint8_t *next_in;
+    size_t avail_in;
+    uint64_t total_in;
+
+    uint8_t *next_out;
+    size_t avail_out;
+    uint64_t total_out;
+    lzma_allocator *allocator;
+    ...;
+} lzma_stream;
+
+typedef struct {
+    int type;
+    uint32_t dist;
+    ...;
+} lzma_options_delta;
+
+typedef struct {
+    uint32_t start_offset;
+    ...;
+} lzma_options_bcj;
+
+typedef struct {
+    uint32_t dict_size;
+    uint32_t lc;
+    uint32_t lp;
+    uint32_t pb;
+    int mode;
+    uint32_t nice_len;
+    int mf;
+    uint32_t depth;
+    ...;
+} lzma_options_lzma;
+
+typedef struct {
+    lzma_vli id;
+    void *options;
+    ...;
+} lzma_filter;
+
+typedef struct {
+    uint32_t version;
+    lzma_vli backward_size;
+    int check;
+    ...;
+} lzma_stream_flags;
+
+typedef ... lzma_index;
+
+typedef struct {
+    uint32_t version;
+    uint32_t header_size;
+    int check;
+    lzma_vli compressed_size;
+    lzma_filter* filters;
+    ...;
+} lzma_block;
+
+bool lzma_check_is_supported(int check);
+
+// Encoder/Decoder
+int lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags);
+int lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags);
+int lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit);
+int lzma_raw_decoder(lzma_stream *strm, const lzma_filter *filters);
+int lzma_block_decoder(lzma_stream *strm, lzma_block *block);
+
+int lzma_easy_encoder(lzma_stream *strm, uint32_t preset, int check);
+int lzma_alone_encoder(lzma_stream *strm, lzma_options_lzma* options);
+int lzma_raw_encoder(lzma_stream *strm, const lzma_filter *filters);
+
+int lzma_get_check(const lzma_stream *strm);
+
+int lzma_code(lzma_stream *strm, int action);
+
+void lzma_end(lzma_stream *strm);
+
+// Extras
+int lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in);
+int lzma_stream_footer_decode(lzma_stream_flags *options, const uint8_t *in);
+int lzma_stream_flags_compare(const lzma_stream_flags *a,
+    const lzma_stream_flags *b);
+
+enum lzma_index_iter_mode { LZMA_INDEX_ITER_ANY, LZMA_INDEX_ITER_STREAM,
+    LZMA_INDEX_ITER_BLOCK, LZMA_INDEX_ITER_NONEMPTY_BLOCK, ... };
+
+// Indexes
+lzma_index* lzma_index_init(lzma_allocator *al);
+void lzma_index_end(lzma_index *i, lzma_allocator *al);
+int lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding);
+lzma_index* lzma_index_dup(const lzma_index *i, lzma_allocator *al);
+int lzma_index_cat(lzma_index *dest, lzma_index *src, lzma_allocator *al);
+int lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit,
+    lzma_allocator *allocator, const uint8_t *in, size_t *in_pos,
+    size_t in_size);
+lzma_vli lzma_index_block_count(const lzma_index *i);
+lzma_vli lzma_index_stream_size(const lzma_index *i);
+lzma_vli lzma_index_uncompressed_size(const lzma_index *i);
+lzma_vli lzma_index_size(const lzma_index *i);
+lzma_vli lzma_index_total_size(const lzma_index *i);
+
+// Blocks
+int lzma_block_header_decode(lzma_block *block, lzma_allocator *al,
+    const uint8_t *in);
+int lzma_block_compressed_size(lzma_block *block, lzma_vli unpadded_size);
+
+typedef struct {
+    // cffi doesn't support partial anonymous structs
+    // so we write the definition in full
+	struct {
+		const lzma_stream_flags *flags;
+		const void *reserved_ptr1;
+		const void *reserved_ptr2;
+		const void *reserved_ptr3;
+		lzma_vli number;
+		lzma_vli block_count;
+		lzma_vli compressed_offset;
+		lzma_vli uncompressed_offset;
+		lzma_vli compressed_size;
+		lzma_vli uncompressed_size;
+		lzma_vli padding;
+		lzma_vli reserved_vli1;
+		lzma_vli reserved_vli2;
+		lzma_vli reserved_vli3;
+		lzma_vli reserved_vli4;
+	} stream;
+	struct {
+		lzma_vli number_in_file;
+		lzma_vli compressed_file_offset;
+		lzma_vli uncompressed_file_offset;
+		lzma_vli number_in_stream;
+		lzma_vli compressed_stream_offset;
+		lzma_vli uncompressed_stream_offset;
+		lzma_vli uncompressed_size;
+		lzma_vli unpadded_size;
+		lzma_vli total_size;
+		lzma_vli reserved_vli1;
+		lzma_vli reserved_vli2;
+		lzma_vli reserved_vli3;
+		lzma_vli reserved_vli4;
+		const void *reserved_ptr1;
+		const void *reserved_ptr2;
+		const void *reserved_ptr3;
+		const void *reserved_ptr4;
+	} block;
+    ...;
+} lzma_index_iter;
+
+void lzma_index_iter_init(lzma_index_iter *iter, const lzma_index *i);
+int lzma_index_iter_next(lzma_index_iter *iter, int mode);
+int lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target);
+
+// Properties
+int lzma_properties_size(uint32_t *size, const lzma_filter *filter);
+int lzma_properties_encode(const lzma_filter *filter, uint8_t *props);
+int lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator,
+    const uint8_t *props, size_t props_size);
+int lzma_lzma_preset(lzma_options_lzma* options, uint32_t preset);
+
+// Special functions
+void _pylzma_stream_init(lzma_stream *strm);
+void _pylzma_block_header_size_decode(uint32_t b);
+
+void *malloc(size_t size);
+void free(void *ptr);
+void *realloc(void *ptr, size_t size);
+""")
+
+m = ffi.verify("""
+#include <lzma.h>
+#include <stdlib.h>
+void _pylzma_stream_init(lzma_stream *strm) {
+    lzma_stream tmp = LZMA_STREAM_INIT; // macro from lzma.h
+    *strm = tmp;
+}
+
+uint32_t _pylzma_block_header_size_decode(uint32_t b) {
+    return lzma_block_header_size_decode(b); // macro from lzma.h
+}
+""",
+    libraries=['lzma'],
+    include_dirs=['/opt/local/include', '/usr/local/include'],
+    library_dirs=['/opt/local/include', '/usr/local/include'],
+    ext_package='_lzmaffi_mods',
+    modulename='_compiled_module')
+
+def _new_lzma_stream():
+    ret = ffi.new('lzma_stream*')
+    m._pylzma_stream_init(ret)
+    return ffi.gc(ret, m.lzma_end)
+
+def add_constant(c):
+    globals()[c] = getattr(m, 'LZMA_' + c)
+
+if sys.version_info >= (2,7):
+    def to_bytes(data):
+        return memoryview(data).tobytes()
+else:
+    def to_bytes(data):
+        if not isinstance(data, basestring):
+            raise TypeError("lzma: must be str/unicode, got %s" % (type(data),))
+        return bytes(data)
+
+if sys.version_info >= (3,0):
+    long = int
+
+for c in ['CHECK_CRC32', 'CHECK_CRC64', 'CHECK_ID_MAX', 'CHECK_NONE', 'CHECK_SHA256', 'FILTER_ARM', 'FILTER_ARMTHUMB', 'FILTER_DELTA', 'FILTER_IA64', 'FILTER_LZMA1', 'FILTER_LZMA2', 'FILTER_POWERPC', 'FILTER_SPARC', 'FILTER_X86', 'MF_BT2', 'MF_BT3', 'MF_BT4', 'MF_HC3', 'MF_HC4', 'MODE_FAST', 'MODE_NORMAL', 'PRESET_DEFAULT', 'PRESET_EXTREME', 'STREAM_HEADER_SIZE']:
+    add_constant(c)
+
+def _parse_format(format):
+    if isinstance(format, (int, long)):
+        return format
+    else:
+        raise TypeError
+
+CHECK_UNKNOWN = CHECK_ID_MAX + 1
+FORMAT_AUTO, FORMAT_XZ, FORMAT_ALONE, FORMAT_RAW, FORMAT_BLOCK = range(5)
+
+BCJ_FILTERS = (m.LZMA_FILTER_X86,
+    m.LZMA_FILTER_POWERPC,
+    m.LZMA_FILTER_IA64,
+    m.LZMA_FILTER_ARM,
+    m.LZMA_FILTER_ARMTHUMB,
+    m.LZMA_FILTER_SPARC)
+
+class LZMAError(Exception):
+    """Call to liblzma failed."""
+
+def is_check_supported(check):
+    """is_check_supported(check_id) -> bool
+    
+    Test whether the given integrity check is supported.
+    
+    Always returns True for CHECK_NONE and CHECK_CRC32."""
+    return bool(m.lzma_check_is_supported(check))
+
+def catch_lzma_error(fun, *args):
+    try:
+        lzret = fun(*args)
+    except:
+        raise
+    if lzret in (m.LZMA_OK, m.LZMA_GET_CHECK, m.LZMA_NO_CHECK, m.LZMA_STREAM_END):
+        return lzret
+    elif lzret == m.LZMA_DATA_ERROR:
+        raise LZMAError("Corrupt input data")
+    elif lzret == m.LZMA_UNSUPPORTED_CHECK:
+        raise LZMAError("Unsupported integrity check")
+    elif lzret == m.LZMA_FORMAT_ERROR:
+        raise LZMAError("Input format not supported by decoder")
+    elif lzret == m.LZMA_OPTIONS_ERROR:
+        raise LZMAError("Invalid or unsupported options")
+    elif lzret == m.LZMA_BUF_ERROR:
+        raise LZMAError("Insufficient buffer space")
+    elif lzret == m.LZMA_PROG_ERROR:
+        raise LZMAError("Internal error")
+    elif lzret == m.LZMA_MEM_ERROR:
+        raise MemoryError
+    else:
+        raise LZMAError("Unrecognised error from liblzma: %d" % lzret)
+
+def parse_filter_spec_delta(id, dist=1):
+    ret = ffi.new('lzma_options_delta*')
+    ret.type = m.LZMA_DELTA_TYPE_BYTE
+    ret.dist = dist
+    return ret
+
+def parse_filter_spec_bcj(id, start_offset=0):
+    ret = ffi.new('lzma_options_bcj*')
+    ret.start_offset = start_offset
+    return ret
+
+def parse_filter_spec_lzma(id, preset=m.LZMA_PRESET_DEFAULT, **kwargs):
+    ret = ffi.new('lzma_options_lzma*')
+    if m.lzma_lzma_preset(ret, preset):
+        raise LZMAError("Invalid...")
+    for arg, val in kwargs.items():
+        if arg in ('dict_size', 'lc', 'lp', 'pb', 'nice_len', 'depth'):
+            setattr(ret, arg, val)
+        elif arg in ('mf', 'mode'):
+            setattr(ret, arg, int(val))
+        else:
+            raise ValueError("Invalid...")
+    return ret
+
+def parse_filter_spec(spec):
+    if not isinstance(spec, collections.Mapping):
+        raise TypeError("Filter...")
+    ret = ffi.new('lzma_filter*')
+    try:
+        ret.id = spec['id']
+    except KeyError:
+        raise ValueError("Filter...")
+    if ret.id in (m.LZMA_FILTER_LZMA1, m.LZMA_FILTER_LZMA2):
+        try:
+            options = parse_filter_spec_lzma(**spec)
+        except TypeError:
+            raise ValueError("Invalid...")
+    elif ret.id == m.LZMA_FILTER_DELTA:
+        try:
+            options = parse_filter_spec_delta(**spec)
+        except TypeError:
+            raise ValueError("Invalid...")
+    elif ret.id in BCJ_FILTERS:
+        try:
+            options = parse_filter_spec_bcj(**spec)
+        except TypeError:
+            raise ValueError("Invalid...")
+    else:
+        raise ValueError("Invalid %d" % (ret.id,))
+
+    ret.options = options
+    _owns[ret] = options
+    return ret
+
+def _encode_filter_properties(filterspec):
+    """_encode_filter_properties(filter) -> bytes
+
+    Return a bytes object encoding the options (properties) of the filter
+    specified by *filter* (a dict).
+
+    The result does not include the filter ID itself, only the options."""
+    filter = parse_filter_spec(filterspec)
+    size = ffi.new("uint32_t*")
+    catch_lzma_error(m.lzma_properties_size, size, filter)
+    result = ffi.new('char[]', size[0])
+    catch_lzma_error(m.lzma_properties_encode, filter, result)
+    return ffi.buffer(result)[:]
+
+def parse_filter_chain_spec(filterspecs):
+    if len(filterspecs) > m.LZMA_FILTERS_MAX:
+        raise ValueError("Too...")
+    filters = ffi.new('lzma_filter[]', m.LZMA_FILTERS_MAX+1)
+    _owns[filters] = children = []
+    for i in range(m.LZMA_FILTERS_MAX+1):
+        try:
+            filterspec = filterspecs[i]
+        except KeyError:
+            raise TypeError
+        except IndexError:
+            filters[i].id = m.LZMA_VLI_UNKNOWN
+        else:
+            filter = parse_filter_spec(filterspecs[i])
+            children.append(filter)
+            filters[i].id = filter.id
+            filters[i].options = filter.options
+    return filters
+
+def build_filter_spec(filter):
+    spec = {'id': filter.id}
+    def add_opts(options_type, *opts):
+        options = ffi.cast('%s*' % (options_type,), filter.options)
+        for v in opts:
+            spec[v] = getattr(options, v)
+    if filter.id == m.LZMA_FILTER_LZMA1:
+        add_opts('lzma_options_lzma', 'lc', 'lp', 'pb', 'dict_size')
+    elif filter.id == m.LZMA_FILTER_LZMA2:
+        add_opts('lzma_options_lzma', 'dict_size')
+    elif filter.id == m.LZMA_FILTER_DELTA:
+        add_opts('lzma_options_delta', 'dist')
+    elif filter.id in BCJ_FILTERS:
+        add_opts('lzma_options_bcj', 'start_offset')
+    else:
+        raise ValueError("Invalid...")
+    return spec
+
+def _decode_filter_properties(filter_id, encoded_props):
+    """_decode_filter_properties(filter_id, encoded_props) -> dict
+
+    Return a dict describing a filter with ID *filter_id*, and options
+    (properties) decoded from the bytes object *encoded_props*."""
+    filter = ffi.new('lzma_filter*')
+    filter.id = filter_id
+    catch_lzma_error(m.lzma_properties_decode,
+        filter, ffi.NULL, encoded_props, len(encoded_props))
+    try:
+        return build_filter_spec(filter)
+    finally:
+        # TODO do we need this, the only use of m.free?
+        m.free(filter.options)
+
+def _decode_stream_header_or_footer(decode_f, in_bytes):
+    footer_o = ffi.new('char[]', to_bytes(in_bytes))
+    stream_flags = ffi.new('lzma_stream_flags*')
+    catch_lzma_error(decode_f, stream_flags, footer_o)
+    return StreamFlags(stream_flags)
+
+decode_stream_footer = functools.partial(_decode_stream_header_or_footer,
+    m.lzma_stream_footer_decode)
+
+decode_stream_header = functools.partial(_decode_stream_header_or_footer,
+    m.lzma_stream_header_decode)
+
+def decode_block_header_size(in_byte):
+    # lzma_block_header_size_decode(b) (((uint32_t)(b) + 1) * 4)
+    return (ord(in_byte) + 1) * 4
+
+def decode_index(s, stream_padding=0):
+    indexp = ffi.new('lzma_index**')
+    memlimit = ffi.new('uint64_t*')
+    memlimit[0] = m.UINT64_MAX
+    allocator = ffi.NULL
+    in_buf = ffi.new('char[]', to_bytes(s))
+    in_pos = ffi.new('size_t*')
+    in_pos[0] = 0
+    catch_lzma_error(m.lzma_index_buffer_decode, indexp,
+        memlimit, allocator, in_buf, in_pos, len(s))
+    return Index(indexp[0], allocator, stream_padding)
+
+class Index(object):
+    def __init__(self, i, allocator, stream_padding=0):
+        self.i = i
+        self.allocator = allocator
+        m.lzma_index_stream_padding(i, stream_padding)
+
+    @property
+    def uncompressed_size(self):
+        return m.lzma_index_uncompressed_size(self.i)
+
+    @property
+    def block_count(self):
+        return m.lzma_index_block_count(self.i)
+
+    @property
+    def index_size(self):
+        return m.lzma_index_size(self.i)
+
+    @property
+    def blocks_size(self):
+        return m.lzma_index_total_size(self.i)
+
+    def __iter__(self):
+        return self.iterator()
+
+    def iterator(self, type=m.LZMA_INDEX_ITER_BLOCK):
+        iterator = ffi.new('lzma_index_iter*')
+        m.lzma_index_iter_init(iterator, self.i)
+        while not m.lzma_index_iter_next(iterator, type):
+            yield (IndexStreamData(iterator.stream), IndexBlockData(iterator.block))
+
+    def find(self, offset):
+        iterator = ffi.new('lzma_index_iter*')
+        m.lzma_index_iter_init(iterator, self.i)
+        if m.lzma_index_iter_locate(iterator, offset):
+            # offset too high
+            return None
+        return (IndexStreamData(iterator.stream), IndexBlockData(iterator.block))
+
+    def __del__(self):
+        m.lzma_index_end(self.i, self.allocator)
+
+    def copy(self):
+        new_i = m.lzma_index_dup(self.i, self.allocator)
+        return Index(new_i, self.allocator)
+
+    deepcopy = copy
+
+    def append(self, other_index):
+        # m.lzma_index_cat frees its second parameter so we
+        # must copy it first
+        other_index_i = m.lzma_index_dup(other_index.i, self.allocator)
+        catch_lzma_error(m.lzma_index_cat, self.i, 
+            other_index_i, self.allocator)
+
+class _StructToPy(object):
+    __slots__ = ()
+    def __init__(self, struct_obj):
+        # TODO make PyPy-fast
+        for attr in self.__slots__:
+            setattr(self, attr, getattr(struct_obj, attr))
+    def __repr__(self):
+        descriptions = ('%s=%r' % (attr, getattr(self, attr)) for attr in self.__slots__)
+        return "<%s %s>" % (type(self).__name__, ' '.join(descriptions))
+
+class IndexStreamData(_StructToPy):
+    __slots__ = ('number', 'block_count', 'compressed_offset', 'uncompressed_offset',
+        'compressed_size', 'uncompressed_size')
+
+class IndexBlockData(_StructToPy):
+    __slots__ = ('number_in_file', 'compressed_file_offset', 'uncompressed_file_offset',
+        'compressed_stream_offset', 'uncompressed_stream_offset',
+        'uncompressed_size', 'unpadded_size', 'total_size')
+
+class StreamFlags(object):
+    def __init__(self, i):
+        self.i = i
+
+    version = property(lambda self: self.i.version)
+    check = property(lambda self: self.i.check)
+    backward_size = property(lambda self: self.i.backward_size)
+
+    @property
+    def supported(self):
+        return self.version > SUPPORTED_STREAM_FLAGS_VERSION
+
+    def check_supported(self):
+        if not self.supported:
+            raise LZMAError("Stream is too new for liblzma version")
+
+    def matches(self, other):
+        return m.lzma_stream_flags_compare(self.i, other.i) == m.LZMA_OK
+
+    def copy(self):
+        other_i = ffi.new('lzma_stream_flags*', self.i)
+        return StreamFlags(other_i)
+
+class Allocator(object):
+    def __init__(self):
+        self.owns = {}
+        self.lzma_allocator = ffi.new('lzma_allocator*')
+        alloc = self.owns['a'] = ffi.callback("void*(void*, size_t, size_t)", self.__alloc)
+        free = self.owns['b'] = ffi.callback("void(void*, void*)", self.__free)
+        self.lzma_allocator.alloc = alloc
+        self.lzma_allocator.free = free
+        self.lzma_allocator.opaque = ffi.NULL
+    def __alloc(self, _opaque, _nmemb, size):
+        new_mem = ffi.new('char[]', size)
+        self.owns[self._addr(new_mem)] = new_mem
+        return new_mem
+    def _addr(self, ptr):
+        return long(ffi.cast('uintptr_t', ptr))
+    def __free(self, _opaque, ptr):
+        if self._addr(ptr) == 0: return
+        del self.owns[self._addr(ptr)]
+
+class LZMADecompressor(object):
+    """
+    LZMADecompressor(format=FORMAT_AUTO, memlimit=None, filters=None)
+
+    Create a decompressor object for decompressing data incrementally.
+
+    format specifies the container format of the input stream. If this is
+    FORMAT_AUTO (the default), the decompressor will automatically detect
+    whether the input is FORMAT_XZ or FORMAT_ALONE. Streams created with
+    FORMAT_RAW cannot be autodetected.
+
+    memlimit can be specified to limit the amount of memory used by the
+    decompressor. This will cause decompression to fail if the input
+    cannot be decompressed within the given limit.
+
+    filters specifies a custom filter chain. This argument is required for
+    FORMAT_RAW, and not accepted with any other format. When provided,
+    this should be a sequence of dicts, each indicating the ID and options
+    for a single filter.
+
+    For one-shot decompression, use the decompress() function instead.
+    """
+    def __init__(self, format=FORMAT_AUTO, memlimit=None, filters=None, header=None, check=None, unpadded_size=None):
+        decoder_flags = m.LZMA_TELL_ANY_CHECK | m.LZMA_TELL_NO_CHECK
+        #decoder_flags = 0
+        if memlimit is not None:
+            if format == FORMAT_RAW:
+                raise ValueError("Cannot sp...")
+            #memlimit = long(memlimit)
+        else:
+            memlimit = m.UINT64_MAX
+
+        if format == FORMAT_RAW and filters is None:
+            raise ValueError("Must...")
+        elif format != FORMAT_RAW and filters is not None:
+            raise ValueError("Cannot...")
+
+        if format == FORMAT_BLOCK and (header is None or unpadded_size is None or check is None):
+            raise ValueError("Must...")
+        elif format != FORMAT_BLOCK and (header is not None or unpadded_size is not None or check is not None):
+            raise ValueError("Cannot...")
+
+        format = _parse_format(format)
+        self.lock = threading.Lock()
+        self.check = CHECK_UNKNOWN
+        self.unused_data = b''
+        self.eof = False
+        self.lzs = _new_lzma_stream()
+        self._bufsiz = max(8192, io.DEFAULT_BUFFER_SIZE)
+
+        if format == FORMAT_AUTO:
+            catch_lzma_error(m.lzma_auto_decoder, self.lzs, memlimit, decoder_flags)
+        elif format == FORMAT_XZ:
+            catch_lzma_error(m.lzma_stream_decoder, self.lzs, memlimit, decoder_flags)
+        elif format == FORMAT_ALONE:
+            self.check = CHECK_NONE
+            catch_lzma_error(m.lzma_alone_decoder, self.lzs, memlimit)
+        elif format == FORMAT_RAW:
+            self.check = CHECK_NONE
+            filters = parse_filter_chain_spec(filters)
+            catch_lzma_error(m.lzma_raw_decoder, self.lzs,
+                filters)
+        elif format == FORMAT_BLOCK:
+            self.__block = block = ffi.new('lzma_block*')
+            block.version = 0
+            block.check = check
+            block.header_size = len(header)
+            block.filters = self.__filters = ffi.new('lzma_filter[]', m.LZMA_FILTERS_MAX+1)
+            header_b = ffi.new('char[]', to_bytes(header))
+            catch_lzma_error(m.lzma_block_header_decode, block, self.lzs.allocator, header_b)
+            if unpadded_size is not None:
+                catch_lzma_error(m.lzma_block_compressed_size, block, unpadded_size)
+            self.expected_size = block.compressed_size
+            catch_lzma_error(m.lzma_block_decoder, self.lzs, block)
+        else:
+            raise ValueError("invalid...")
+
+    def decompress(self, data):
+        """
+        decompress(data) -> bytes
+
+        Provide data to the decompressor object. Returns a chunk of
+        decompressed data if possible, or b"" otherwise.
+
+        Attempting to decompress data after the end of the stream is
+        reached raises an EOFError. Any data found after the end of the
+        stream is ignored, and saved in the unused_data attribute.
+        """
+        with self.lock:
+            if self.eof:
+                raise EOFError("Already...")
+            return self._decompress(data)
+
+    def _decompress(self, data):
+        lzs = self.lzs
+
+        # we need in_ so that lzs.next_in doesn't get garbage collected until
+        # in_ goes out of scope
+        data = to_bytes(data)
+        lzs.next_in = in_ = ffi.new('char[]', data)
+        lzs.avail_in = len(data)
+
+        bufsiz = self._bufsiz
+
+        lzs.next_out = orig_out = m.malloc(bufsiz)
+        if orig_out == ffi.NULL:
+            raise MemoryError
+
+        lzs.avail_out = bufsiz
+
+        data_size = 0
+
+        try:
+            while True:
+                ret = catch_lzma_error(m.lzma_code, lzs, m.LZMA_RUN)
+                data_size = int(ffi.cast('uintptr_t', lzs.next_out)) - int(ffi.cast('uintptr_t', orig_out))
+                # data_size is the amount lzma_code has already outputted
+
+                if ret in (m.LZMA_NO_CHECK, m.LZMA_GET_CHECK):
+                    self.check = m.lzma_get_check(lzs)
+
+                if ret == m.LZMA_STREAM_END:
+                    self.eof = True
+                    if lzs.avail_in > 0:
+                        self.unused_data = ffi.buffer(lzs.next_in, lzs.avail_in)[:]
+                    break
+                elif lzs.avail_in == 0:
+                    # it ate everything
+                    break
+                elif lzs.avail_out == 0:
+                    # ran out of space in the output buffer, let's grow it
+                    bufsiz += (bufsiz >> 3) + 6
+                    next_out = m.realloc(orig_out, bufsiz)
+                    if next_out == ffi.NULL:
+                        # realloc unsuccessful
+                        m.free(orig_out)
+                        orig_out = ffi.NULL
+                        raise MemoryError
+
+                    orig_out = next_out
+
+                    lzs.next_out = orig_out + data_size
+                    lzs.avail_out = bufsiz - data_size
+
+            result = ffi.buffer(orig_out, data_size)[:]
+        finally:
+            m.free(orig_out)
+
+        return result
+
+    def __getstate__(self):
+        raise TypeError("cannot serialize '%s' object" %
+                        self.__class__.__name__)
+
+class LZMACompressor(object):
+    """
+    LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None)
+
+    Create a compressor object for compressing data incrementally.
+
+    format specifies the container format to use for the output. This can
+    be FORMAT_XZ (default), FORMAT_ALONE, or FORMAT_RAW.
+
+    check specifies the integrity check to use. For FORMAT_XZ, the default
+    is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not suport integrity
+    checks; for these formats, check must be omitted, or be CHECK_NONE.
+
+    The settings used by the compressor can be specified either as a
+    preset compression level (with the 'preset' argument), or in detail
+    as a custom filter chain (with the 'filters' argument). For FORMAT_XZ
+    and FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset
+    level. For FORMAT_RAW, the caller must always specify a filter chain;
+    the raw compressor does not support preset compression levels.
+
+    preset (if provided) should be an integer in the range 0-9, optionally
+    OR-ed with the constant PRESET_EXTREME.
+
+    filters (if provided) should be a sequence of dicts. Each dict should
+    have an entry for "id" indicating the ID of the filter, plus
+    additional entries for options to the filter.
+
+    For one-shot compression, use the compress() function instead.
+    """
+    def __init__(self, format=FORMAT_XZ, check=-1, preset=None, filters=None):
+        if format != FORMAT_XZ and check not in (-1, m.LZMA_CHECK_NONE):
+            raise ValueError("Integrity...")
+        if preset is not None and filters is not None:
+            raise ValueError("Cannot...")
+        if preset is None:
+            preset = m.LZMA_PRESET_DEFAULT
+        format = _parse_format(format)
+        self.lock = threading.Lock()
+        self.flushed = 0
+        self.lzs = _new_lzma_stream()
+        if format == FORMAT_XZ:
+            if filters is None:
+                if check == -1:
+                    check = m.LZMA_CHECK_CRC64
+                catch_lzma_error(m.lzma_easy_encoder, self.lzs,
+                    preset, check)
+            else:
+                filters = parse_filter_chain_spec(filters)
+                catch_lzma_error(m.lzma_stream_encoder, self.lzs,
+                    filters, check)
+        elif format == FORMAT_ALONE:
+            if filters is None:
+                options = ffi.new('lzma_options_lzma*')
+                if m.lzma_lzma_preset(options, preset):
+                    raise LZMAError("Invalid...")
+                catch_lzma_error(m.lzma_alone_encoder, self.lzs,
+                    options)
+            else:
+                raise NotImplementedError
+        elif format == FORMAT_RAW:
+            if filters is None:
+                raise ValueError("Must...")
+            filters = parse_filter_chain_spec(filters)
+            catch_lzma_error(m.lzma_raw_encoder, self.lzs,
+                filters)
+        else:
+            raise ValueError("Invalid...")
+
+    def compress(self, data):
+        """
+        compress(data) -> bytes
+
+        Provide data to the compressor object. Returns a chunk of
+        compressed data if possible, or b"" otherwise.
+
+        When you have finished providing data to the compressor, call the
+        flush() method to finish the conversion process.
+        """
+        with self.lock:
+            if self.flushed:
+                raise ValueError("Compressor...")
+            return self._compress(data)
+
+    def _compress(self, data, action=m.LZMA_RUN):
+        # TODO use realloc like in LZMADecompressor
+        BUFSIZ = 8192
+
+        lzs = self.lzs
+
+        lzs.next_in = input_ = ffi.new('char[]', to_bytes(data))
+        lzs.avail_in = len(data)
+        outs = [ffi.new('char[]', BUFSIZ)]
+        lzs.next_out, = outs
+        lzs.avail_out = BUFSIZ
+
+        siz = BUFSIZ
+
+        while True:
+            next_out_pos = int(ffi.cast('intptr_t', lzs.next_out))
+            ret = catch_lzma_error(m.lzma_code, lzs, action)
+            data_size = int(ffi.cast('intptr_t', lzs.next_out)) - next_out_pos
+            if (action == m.LZMA_RUN and lzs.avail_in == 0) or \
+                (action == m.LZMA_FINISH and ret == m.LZMA_STREAM_END):
+                break
+            elif lzs.avail_out == 0:
+                # ran out of space in the output buffer
+                #siz = (BUFSIZ << 1) + 6
+                siz = 512
+                outs.append(ffi.new('char[]', siz))
+                lzs.next_out = outs[-1]
+                lzs.avail_out = siz
+        last_out = outs.pop()
+        last_out_len = siz - lzs.avail_out
+        last_out_piece = ffi.buffer(last_out[0:last_out_len], last_out_len)[:]
+
+        return b''.join(ffi.buffer(nn)[:] for nn in outs) + last_out_piece
+
+    def flush(self):
+        with self.lock:
+            if self.flushed:
+                raise ValueError("Repeated...")
+            self.flushed = 1
+            return self._compress(b'', action=m.LZMA_FINISH)
+
+    def __getstate__(self):
+        raise TypeError("cannot serialize '%s' object" %
+                        self.__class__.__name__)
+        
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -37,7 +37,7 @@
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
     "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy"
-    "faulthandler", "_lzma",
+    "faulthandler",
 ])
 
 translation_modules = default_modules.copy()
diff --git a/pypy/doc/config/objspace.usemodules._lzma.txt b/pypy/doc/config/objspace.usemodules._lzma.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules._lzma.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-Use the '_lzma' module. 
-This module is expected to be working and is included by default.
diff --git a/pypy/module/_lzma/__init__.py b/pypy/module/_lzma/__init__.py
deleted file mode 100644
--- a/pypy/module/_lzma/__init__.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from pypy.interpreter.mixedmodule import MixedModule
-
-class Module(MixedModule):
-    # The private part of the lzma module.
-
-    applevel_name = '_lzma'
-
-    interpleveldefs = {
-        'LZMACompressor': 'interp_lzma.W_LZMACompressor',
-        'LZMADecompressor': 'interp_lzma.W_LZMADecompressor',
-        'LZMAError': 'interp_lzma.W_LZMAError',
-        '_encode_filter_properties': 'interp_lzma.encode_filter_properties',
-        '_decode_filter_properties': 'interp_lzma.decode_filter_properties',
-    }
-
-    for name in 'AUTO XZ ALONE RAW'.split():
-        interpleveldefs['FORMAT_%s' % name] = (
-            'space.wrap(interp_lzma.FORMAT_%s)' % name)
-    for name in 'DEFAULT EXTREME'.split():
-        interpleveldefs['PRESET_%s' % name] = (
-            'space.wrap(interp_lzma.LZMA_PRESET_%s)' % name)
-    for name in 'LZMA1 LZMA2 DELTA X86 IA64 ARM ARMTHUMB SPARC POWERPC'.split():
-        interpleveldefs['FILTER_%s' % name] = (
-            'space.wrap(interp_lzma.LZMA_FILTER_%s)' % name)
-
-    appleveldefs = {
-    }
diff --git a/pypy/module/_lzma/interp_lzma.py b/pypy/module/_lzma/interp_lzma.py
deleted file mode 100644
--- a/pypy/module/_lzma/interp_lzma.py
+++ /dev/null
@@ -1,371 +0,0 @@
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import (
-    TypeDef, interp_attrproperty_bytes, interp_attrproperty)
-from pypy.interpreter.error import oefmt
-from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from pypy.module.exceptions.interp_exceptions import _new_exception, W_Exception
-from pypy.module.thread.os_lock import Lock
-from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import LONGLONG_MASK, r_ulonglong
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rtyper.lltypesystem import rffi
-from rpython.rtyper.lltypesystem import lltype
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-
-
-FORMAT_AUTO, FORMAT_XZ, FORMAT_ALONE, FORMAT_RAW = range(4)
-R_LONGLONG_MASK = r_ulonglong(LONGLONG_MASK)
-
-
-eci = ExternalCompilationInfo(
-    includes = ['lzma.h'],
-    libraries = ['lzma'],
-    )
-eci = platform.configure_external_library(
-    'lzma', eci,
-    [dict(prefix='lzma-')])
-if not eci:
-    raise ImportError("Could not find lzma library")
-
-
-class CConfig:
-    _compilation_info_ = eci
-    calling_conv = 'c'
-
-    BUFSIZ = platform.ConstantInteger("BUFSIZ")
-
-    lzma_stream = platform.Struct(
-        'lzma_stream',
-        [('next_in', rffi.CCHARP),
-         ('avail_in', rffi.UINT),
-         ('total_in', rffi.UINT),
-         ('next_out', rffi.CCHARP),
-         ('avail_out', rffi.UINT),
-         ('total_out', rffi.UINT),
-         ])
-
-    lzma_options_lzma = platform.Struct(
-        'lzma_options_lzma',
-        [])
-
-constant_names = '''
-    LZMA_RUN LZMA_FINISH
-    LZMA_OK LZMA_GET_CHECK LZMA_NO_CHECK LZMA_STREAM_END
-    LZMA_PRESET_DEFAULT
-    LZMA_CHECK_ID_MAX
-    LZMA_TELL_ANY_CHECK LZMA_TELL_NO_CHECK
-    LZMA_FILTER_LZMA1 LZMA_FILTER_LZMA2 LZMA_FILTER_DELTA
-    LZMA_FILTER_IA64 LZMA_FILTER_X86 LZMA_FILTER_ARM LZMA_FILTER_ARMTHUMB
-    LZMA_FILTER_SPARC LZMA_FILTER_POWERPC
-    LZMA_PRESET_DEFAULT LZMA_PRESET_EXTREME
-    '''.split()
-for name in constant_names:
-    setattr(CConfig, name, platform.ConstantInteger(name))
-
-class cConfig(object):
-    pass
-for k, v in platform.configure(CConfig).items():
-    setattr(cConfig, k, v)
-
-for name in constant_names:
-    globals()[name] = getattr(cConfig, name)
-lzma_stream = lltype.Ptr(cConfig.lzma_stream)
-lzma_options_lzma = lltype.Ptr(cConfig.lzma_options_lzma)
-BUFSIZ = cConfig.BUFSIZ
-LZMA_CHECK_UNKNOWN = LZMA_CHECK_ID_MAX + 1
-
-def external(name, args, result, **kwds):
-    return rffi.llexternal(name, args, result, compilation_info=
-                           CConfig._compilation_info_, **kwds)
-
-lzma_ret = rffi.INT
-lzma_action = rffi.INT
-lzma_bool = rffi.INT
-
-lzma_lzma_preset = external('lzma_lzma_preset', [lzma_options_lzma, rffi.UINT], lzma_bool)
-lzma_alone_encoder = external('lzma_alone_encoder', [lzma_stream, lzma_options_lzma], lzma_ret)
-lzma_end = external('lzma_end', [lzma_stream], lltype.Void, releasegil=False)
-
-lzma_auto_decoder = external('lzma_auto_decoder', [lzma_stream, rffi.LONG, rffi.INT], lzma_ret)
-lzma_get_check = external('lzma_get_check', [lzma_stream], rffi.INT)
-
-lzma_code = external('lzma_code', [lzma_stream, lzma_action], rffi.INT)
-
-
- at specialize.arg(1)
-def raise_error(space, fmt, *args):
-    raise oefmt(space.w_RuntimeError, fmt, *args)
-
-
-def _catch_lzma_error(space, lzret):
-    if (lzret == LZMA_OK or lzret == LZMA_GET_CHECK or
-        lzret == LZMA_NO_CHECK or lzret == LZMA_STREAM_END):
-        return
-    raise raise_error(space, "Unrecognized error from liblzma: %d", lzret)
-
-
-if BUFSIZ < 8192:
-    SMALLCHUNK = 8192
-else:
-    SMALLCHUNK = BUFSIZ
-if rffi.sizeof(rffi.INT) > 4:
-    BIGCHUNK = 512 * 32
-else:
-    BIGCHUNK = 512 * 1024
-
-
-def _new_buffer_size(current_size):
-    # keep doubling until we reach BIGCHUNK; then the buffer size is no
-    # longer increased
-    if current_size < BIGCHUNK:
-        return current_size + current_size
-    return current_size
-
-
-class OutBuffer(object):
-    """Handler for the output buffer.  A bit custom code trying to
-    encapsulate the logic of setting up the fields of 'lzs' and
-    allocating raw memory as needed.
-    """
-    def __init__(self, lzs, initial_size=SMALLCHUNK):
-        # when the constructor is called, allocate a piece of memory
-        # of length 'piece_size' and make lzs ready to dump there.
-        self.temp = []
-        self.lzs = lzs
-        self._allocate_chunk(initial_size)
-
-    def _allocate_chunk(self, size):
-        self.raw_buf, self.gc_buf, self.case_num = rffi.alloc_buffer(size)
-        self.current_size = size
-        self.lzs.c_next_out = self.raw_buf
-        rffi.setintfield(self.lzs, 'c_avail_out', size)
-
-    def _get_chunk(self, chunksize):
-        assert 0 <= chunksize <= self.current_size
-        raw_buf = self.raw_buf
-        gc_buf = self.gc_buf
-        case_num = self.case_num
-        s = rffi.str_from_buffer(raw_buf, gc_buf, case_num,
-                                 self.current_size, chunksize)
-        rffi.keep_buffer_alive_until_here(raw_buf, gc_buf, case_num)
-        self.current_size = 0
-        return s
-
-    def prepare_next_chunk(self):
-        size = self.current_size
-        self.temp.append(self._get_chunk(size))
-        self._allocate_chunk(_new_buffer_size(size))
-
-    def make_result_string(self):
-        count_unoccupied = rffi.getintfield(self.lzs, 'c_avail_out')
-        s = self._get_chunk(self.current_size - count_unoccupied)
-        if self.temp:
-            self.temp.append(s)
-            return ''.join(self.temp)
-        else:
-            return s
-
-    def free(self):
-        if self.current_size > 0:
-            rffi.keep_buffer_alive_until_here(self.raw_buf, self.gc_buf,
-                                              self.case_num)
-
-    def __enter__(self):
-        return self
-    def __exit__(self, *args):
-        self.free()
-
-
-class W_LZMACompressor(W_Root):
-    def __init__(self, space, format):
-        self.format = format
-        self.lock = Lock(space)
-        self.flushed = False
-        self.lzs = lltype.malloc(lzma_stream.TO, flavor='raw', zero=True)
-
-    def __del__(self):
-        lzma_end(self.lzs)
-        lltype.free(self.lzs, flavor='raw')
-
-    def _init_alone(self, space, preset, w_filters):
-        if space.is_none(w_filters):
-            with lltype.scoped_alloc(lzma_options_lzma.TO) as options:
-                if lzma_lzma_preset(options, preset):
-                    raise_error(space, "Invalid compression preset: %d", preset)
-                lzret = lzma_alone_encoder(self.lzs, options)
-        else:
-            raise oefmt(space.w_NotImplementedError, "Filter specs")
-        _catch_lzma_error(space, lzret)
-
-    @staticmethod
-    @unwrap_spec(format=int,
-                 w_check=WrappedDefault(None),
-                 w_preset=WrappedDefault(None), 
-                 w_filters=WrappedDefault(None))
-    def descr_new_comp(space, w_subtype, format=FORMAT_XZ, 
-                       w_check=None, w_preset=None, w_filters=None):
-        w_self = space.allocate_instance(W_LZMACompressor, w_subtype)
-        self = space.interp_w(W_LZMACompressor, w_self)
-        W_LZMACompressor.__init__(self, space, format)
-
-        if space.is_none(w_preset):
-            preset = LZMA_PRESET_DEFAULT
-        else:
-            preset = space.int_w(w_preset)
-
-        if format == FORMAT_ALONE:
-            self._init_alone(space, preset, w_filters)
-        else:
-            raise oefmt(space.w_NotImplementedError, "Format %d", format)
-
-        return w_self
-
-    @unwrap_spec(data='bufferstr')
-    def compress_w(self, space, data):
-        with self.lock:
-            if self.flushed:
-                raise oefmt(space.w_ValueError, "Compressor has been flushed")
-            result = self._compress(space, data, LZMA_RUN)
-        return space.wrapbytes(result)
-
-    def flush_w(self, space):
-        with self.lock:
-            if self.flushed:
-                raise oefmt(space.w_ValueError, "Repeated call to flush()")
-            result = self._compress(space, "", LZMA_FINISH)
-        return space.wrapbytes(result)
-
-    def _compress(self, space, data, action):
-        datasize = len(data)
-        with lltype.scoped_alloc(rffi.CCHARP.TO, datasize) as in_buf:
-            for i in range(datasize):
-                in_buf[i] = data[i]
-
-            with OutBuffer(self.lzs) as out:
-                self.lzs.c_next_in = in_buf
-                rffi.setintfield(self.lzs, 'c_avail_in', datasize)
-
-                while True:
-                    lzret = lzma_code(self.lzs, action)
-                    _catch_lzma_error(space, lzret)
-
-                    if (action == LZMA_RUN and
-                        rffi.getintfield(self.lzs, 'c_avail_in') == 0):
-                        break
-                    if action == LZMA_FINISH and lzret == LZMA_STREAM_END:
-                        break
-                    elif rffi.getintfield(self.lzs, 'c_avail_out') == 0:
-                        out.prepare_next_chunk()
-
-                return out.make_result_string()
-
-
-W_LZMACompressor.typedef = TypeDef("LZMACompressor",
-    __new__ = interp2app(W_LZMACompressor.descr_new_comp),
-    compress = interp2app(W_LZMACompressor.compress_w),
-    flush = interp2app(W_LZMACompressor.flush_w),
-)
-
-
-class W_LZMADecompressor(W_Root):
-    def __init__(self, space, format):
-        self.format = format
-        self.lock = Lock(space)
-        self.eof = False
-        self.lzs = lltype.malloc(lzma_stream.TO, flavor='raw', zero=True)
-        self.check = LZMA_CHECK_UNKNOWN
-        self.unused_data = ''
-
-    def __del__(self):
-        lzma_end(self.lzs)
-        lltype.free(self.lzs, flavor='raw')
-
-    @staticmethod
-    @unwrap_spec(format=int,
-                 w_memlimit=WrappedDefault(None),
-                 w_filters=WrappedDefault(None))
-    def descr_new_dec(space, w_subtype, format=FORMAT_AUTO,
-                      w_memlimit=None, w_filters=None):
-        w_self = space.allocate_instance(W_LZMADecompressor, w_subtype)
-        self = space.interp_w(W_LZMADecompressor, w_self)
-        W_LZMADecompressor.__init__(self, space, format)
-
-        if space.is_none(w_memlimit):
-            memlimit = R_LONGLONG_MASK
-        else:
-            memlimit = space.r_ulonglong_w(w_memlimit)
-
-        decoder_flags = LZMA_TELL_ANY_CHECK | LZMA_TELL_NO_CHECK
-
-        if format == FORMAT_AUTO:
-            lzret = lzma_auto_decoder(self.lzs, memlimit, decoder_flags)
-            _catch_lzma_error(space, lzret)
-        else:
-            raise oefmt(space.w_NotImplementedError, "Format %d", format)
-
-        return w_self
-
-    @unwrap_spec(data='bufferstr')
-    def decompress_w(self, space, data):
-        with self.lock:
-            if self.eof:
-                raise oefmt(space.w_EOFError, "Already at end of stream")
-            result = self._decompress(space, data)
-        return space.wrapbytes(result)
-
-    def _decompress(self, space, data):
-        datasize = len(data)
-
-        with lltype.scoped_alloc(rffi.CCHARP.TO, datasize) as in_buf:
-            for i in range(datasize):
-                in_buf[i] = data[i]
-
-            with OutBuffer(self.lzs) as out:
-                self.lzs.c_next_in = in_buf
-                rffi.setintfield(self.lzs, 'c_avail_in', datasize)
-
-                while True:
-                    lzret = lzma_code(self.lzs, LZMA_RUN)
-                    _catch_lzma_error(space, lzret)
-                    if lzret == LZMA_GET_CHECK or lzret == LZMA_NO_CHECK:
-                        self.check = lzma_get_check(self.lzs)
-                    if lzret == LZMA_STREAM_END:
-                        self.eof = True
-                        if rffi.getintfield(self.lzs, 'c_avail_in') > 0:
-                            unused = [self.lzs.c_next_in[i]
-                                      for i in range(
-                                    rffi.getintfield(self.lzs,
-                                                     'c_avail_in'))]
-                            self.unused_data = "".join(unused)
-                            break
-                    if rffi.getintfield(self.lzs, 'c_avail_in') == 0:
-                        break
-                    elif rffi.getintfield(self.lzs, 'c_avail_out') == 0:
-                        out.prepare_next_chunk()
-
-                return out.make_result_string()
-
-
-W_LZMADecompressor.typedef = TypeDef("LZMADecompressor",
-    __new__ = interp2app(W_LZMADecompressor.descr_new_dec),
-    decompress = interp2app(W_LZMADecompressor.decompress_w),
-    eof = interp_attrproperty("eof", W_LZMADecompressor),
-    unused_data = interp_attrproperty_bytes("unused_data", W_LZMADecompressor),
-)
-
-
-W_LZMAError = _new_exception('LZMAError', W_Exception, 'Call to liblzma failed.')
-
-
-def encode_filter_properties(space, w_filter):
-    """Return a bytes object encoding the options (properties) of the filter
-       specified by *filter* (a dict).
-
-    The result does not include the filter ID itself, only the options.
-    """
-
-def decode_filter_properties(space, w_filter_id, w_encoded_props):
-    """Return a dict describing a filter with ID *filter_id*, and options
-       (properties) decoded from the bytes object *encoded_props*.
-    """
-    
diff --git a/pypy/module/_lzma/test/test_lzma.py b/pypy/module/_lzma/test/test_lzma.py
deleted file mode 100644
--- a/pypy/module/_lzma/test/test_lzma.py
+++ /dev/null
@@ -1,30 +0,0 @@
-class AppTestBZ2File:
-    spaceconfig = {
-        "usemodules": ["_lzma"]
-    }
-
-    def test_module(self):
-        import lzma
-
-    def test_simple_compress(self):
-        import lzma
-        compressed = lzma.compress(b'Insert Data Here', format=lzma.FORMAT_ALONE)
-        assert compressed == (b']\x00\x00\x80\x00\xff\xff\xff\xff\xff'
-                              b'\xff\xff\xff\x00$\x9b\x8afg\x91'
-                              b'(\xcb\xde\xfa\x03\r\x1eQT\xbe'
-                              b't\x9e\xdfI]\xff\xf4\x9d\x80\x00')
-        decompressed = lzma.decompress(compressed)
-        assert decompressed == b'Insert Data Here'
-
-    def test_exceptions(self):
-        import _lzma
-        import lzma
-
-        assert hasattr(_lzma, 'LZMAError')
-        assert hasattr(lzma, 'LZMAError')
-
-        assert _lzma.LZMAError is lzma.LZMAError
-        assert _lzma.LZMAError.__doc__ == 'Call to liblzma failed.'
-
-        exc = raises(_lzma.LZMAError, 'raise _lzma.LZMAError')
-        exc = raises(_lzma.LZMAError, 'raise _lzma.LZMAError("bad thing")')
diff --git a/pypy/module/_lzma/test/test_ztranslation.py b/pypy/module/_lzma/test/test_ztranslation.py
deleted file mode 100644
--- a/pypy/module/_lzma/test/test_ztranslation.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from pypy.objspace.fake.checkmodule import checkmodule
-
-def test_lzma_translates():
-    checkmodule('_lzma')

From noreply at buildbot.pypy.org  Tue Dec 30 22:27:14 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 22:27:14 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Add a License paragraph,
 and compile lzmaffi in the release step.
Message-ID: <20141230212714.D85BF1C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75172:862db03baae8
Date: 2014-12-30 20:05 +0100
http://bitbucket.org/pypy/pypy/changeset/862db03baae8/

Log:	Add a License paragraph, and compile lzmaffi in the release step.

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -407,3 +407,16 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 _gdbm module, provided in the file lib_pypy/_gdbm.py, is redistributed
 under the terms of the GPL license as well.
+
+License for 'liblzma and 'lzmaffi'
+----------------------------------
+
+This copy of PyPy may be linked (dynamically or statically) with the
+liblzma library, which was put in the "public domain":
+
+    http://tukaani.org/xz/
+
+The cffi bindings to liblzma (in lib_pypy/_lzma.py) are derived from
+the lzmaffi project which is distributed under a BSD license:
+
+    https://pypi.python.org/pypi/lzmaffi/0.3.0
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -111,7 +111,7 @@
 '''
 
 def create_cffi_import_libraries(pypy_c, options):
-    modules = ['_sqlite3']
+    modules = ['_sqlite3', '_lzma']
     subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
     if not sys.platform == 'win32':
         modules += ['_curses', 'syslog', '_gdbm', '_sqlite3']

From noreply at buildbot.pypy.org  Tue Dec 30 22:27:16 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 22:27:16 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Implement abstract methods and
 properties: they simply fetch the
Message-ID: <20141230212716.24B131C0F1B@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75173:f215d396c1f4
Date: 2014-12-30 22:25 +0100
http://bitbucket.org/pypy/pypy/changeset/f215d396c1f4/

Log:	Implement abstract methods and properties: they simply fetch the
	__isabstractmethod__ property from the underlying callables.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1139,6 +1139,15 @@
         # Equivalent to 'obj.__class__'.
         return self.type(w_obj)
 
+    def isabstractmethod_w(self, w_obj):
+        try:
+            w_result = self.getattr(w_obj, self.wrap("__isabstractmethod__"))
+        except OperationError, e:
+            if e.match(self, self.w_AttributeError):
+                return False
+            raise
+        return self.bool_w(self.nonzero(w_result))
+
     # CPython rules allows subclasses of BaseExceptions to be exceptions.
     # This is slightly less general than the case above, so we prefix
     # it with exception_
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -588,6 +588,9 @@
         instance.__init__(w_function)
         return space.wrap(instance)
 
+    def descr_isabstract(self, space):
+        return space.newbool(space.isabstractmethod_w(self.w_function))
+
 
 class ClassMethod(W_Root):
     """The classmethod objects."""
@@ -606,6 +609,10 @@
         instance.__init__(w_function)
         return space.wrap(instance)
 
+    def descr_isabstract(self, space):
+        return space.newbool(space.isabstractmethod_w(self.w_function))
+
+
 class FunctionWithFixedCode(Function):
     can_change_code = False
 
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -352,6 +352,37 @@
                         else:
                             assert w_res is space.w_False
 
+    def test_isabstract(self, space):
+        w_A = space.appexec([], """():
+            class A:
+               def f(): pass
+               f.__isabstractmethod__ = True
+
+               def g(): pass
+               g.__isabstractmethod__ = True
+               g = classmethod(g)
+
+               def h(): pass
+               h.__isabstractmethod__ = True
+               h = staticmethod(h)
+            return A""")
+        w_B = space.appexec([], """():
+            class B:
+               def f(): pass
+
+               @classmethod
+               def g(): pass
+
+               @staticmethod
+               def h(): pass
+            return B""")
+        assert space.isabstractmethod_w(space.getattr(w_A, space.wrap('f')))
+        assert space.isabstractmethod_w(space.getattr(w_A, space.wrap('g')))
+        assert space.isabstractmethod_w(space.getattr(w_A, space.wrap('h')))
+        assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('f')))
+        assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('g')))
+        assert not space.isabstractmethod_w(space.getattr(w_B, space.wrap('h')))
+
 class TestModuleMinimal: 
     def test_sys_exists(self):
         assert self.space.sys
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -882,6 +882,7 @@
     __get__ = interp2app(StaticMethod.descr_staticmethod_get),
     __new__ = interp2app(StaticMethod.descr_staticmethod__new__.im_func),
     __func__= interp_attrproperty_w('w_function', cls=StaticMethod),
+    __isabstractmethod__ = GetSetProperty(StaticMethod.descr_isabstract),
     )
 
 ClassMethod.typedef = TypeDef(
@@ -889,6 +890,7 @@
     __new__ = interp2app(ClassMethod.descr_classmethod__new__.im_func),
     __get__ = interp2app(ClassMethod.descr_classmethod_get),
     __func__= interp_attrproperty_w('w_function', cls=ClassMethod),
+    __isabstractmethod__ = GetSetProperty(ClassMethod.descr_isabstract),
     __doc__ = """classmethod(function) -> class method
 
 Convert a function to be a class method.
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -2,8 +2,8 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.function import StaticMethod, ClassMethod
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from pypy.interpreter.typedef import (TypeDef, interp_attrproperty_w,
-    generic_new_descr)
+from pypy.interpreter.typedef import (
+    TypeDef, interp_attrproperty_w, generic_new_descr, GetSetProperty)
 from pypy.objspace.descroperation import object_getattribute
 
 
@@ -199,6 +199,11 @@
         w_type = self.getclass(space)
         return space.call_function(w_type, w_getter, w_setter, w_deleter, w_doc)
 
+    def descr_isabstract(self, space):
+        return space.newbool(space.isabstractmethod_w(self.w_fget) or
+                             space.isabstractmethod_w(self.w_fset) or
+                             space.isabstractmethod_w(self.w_fdel))
+
 W_Property.typedef = TypeDef(
     'property',
     __doc__ = '''property(fget=None, fset=None, fdel=None, doc=None) -> property attribute
@@ -216,6 +221,7 @@
     __get__ = interp2app(W_Property.get),
     __set__ = interp2app(W_Property.set),
     __delete__ = interp2app(W_Property.delete),
+    __isabstractmethod__ = GetSetProperty(W_Property.descr_isabstract),
     fdel = interp_attrproperty_w('w_fdel', W_Property),
     fget = interp_attrproperty_w('w_fget', W_Property),
     fset = interp_attrproperty_w('w_fset', W_Property),
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -401,6 +401,12 @@
         del x.x
         assert x.z == 42
 
+    def test_abstract_property(self):
+        def foo(self): pass
+        foo.__isabstractmethod__ = True
+        foo = property(foo)
+        assert foo.__isabstractmethod__ is True
+
     def test___class___variable(self):
         class X:
             def f(self):

From noreply at buildbot.pypy.org  Tue Dec 30 23:39:22 2014
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 30 Dec 2014 23:39:22 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Implemented load_cert_chain on
	_ssl.SSLContext
Message-ID: <20141230223922.04CA91C03F4@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: stdlib-2.7.9
Changeset: r75174:5a86dc612e6b
Date: 2014-12-30 14:39 -0800
http://bitbucket.org/pypy/pypy/changeset/5a86dc612e6b/

Log:	Implemented load_cert_chain on _ssl.SSLContext

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -925,6 +925,43 @@
                         "CERT_OPTIONAL or CERT_REQUIRED")
         self.check_hostname = check_hostname
 
+    def load_cert_chain_w(self, space, w_certfile, w_keyfile=None):
+        if space.is_none(w_certfile):
+            certfile = None
+        else:
+            certfile = space.str_w(w_certfile)
+        if space.is_none(w_keyfile):
+            keyfile = certfile
+        else:
+            keyfile = space.str_w(w_keyfile)
+
+        set_errno(0)
+
+        ret = libssl_SSL_CTX_use_certificate_chain_file(self.ctx, certfile)
+        if ret != 1:
+            errno = get_errno()
+            if errno:
+                libssl_ERR_clear_error()
+                raise wrap_oserror(space, OSError(errno, ''),
+                                   exception_name = 'w_IOError')
+            else:
+                raise _ssl_seterror(space, None, -1)
+
+        ret = libssl_SSL_CTX_use_PrivateKey_file(self.ctx, keyfile,
+                                                 SSL_FILETYPE_PEM)
+        if ret != 1:
+            errno = get_errno()
+            if errno:
+                libssl_ERR_clear_error()
+                raise wrap_oserror(space, OSError(errno, ''),
+                                   exception_name = 'w_IOError')
+            else:
+                raise _ssl_seterror(space, None, -1)
+
+        ret = libssl_SSL_CTX_check_private_key(self.ctx)
+        if ret != 1:
+            raise _ssl_seterror(space, None, -1)
+
     def load_verify_locations_w(self, space, w_cafile=None, w_capath=None):
         if space.is_none(w_cafile):
             cafile = None
@@ -956,6 +993,7 @@
     _wrap_socket=interp2app(_SSLContext.descr_wrap_socket),
     set_ciphers=interp2app(_SSLContext.descr_set_ciphers),
     load_verify_locations=interp2app(_SSLContext.load_verify_locations_w),
+    load_cert_chain=interp2app(_SSLContext.load_cert_chain_w),
     set_default_verify_paths=interp2app(_SSLContext.descr_set_default_verify_paths),
 
     options=GetSetProperty(_SSLContext.descr_get_options,
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -261,6 +261,15 @@
         tmpfile.write(SSL_EMPTYCERT)
         cls.w_emptycert = cls.space.wrap(str(tmpfile))
 
+    def test_load_cert_chain(self):
+        import _ssl
+        ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
+        ctx.load_cert_chain(self.keycert)
+        ctx.load_cert_chain(self.cert, self.key)
+        raises(IOError, ctx.load_cert_chain, "inexistent.pem")
+        raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert)
+        raises(_ssl.SSLError, ctx.load_cert_chain, self.emptycert)
+
     def test_load_verify_locations(self):
         import _ssl
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)

From noreply at buildbot.pypy.org  Tue Dec 30 23:57:59 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 30 Dec 2014 23:57:59 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Full implementation of the audioop module!
Message-ID: <20141230225759.0D3461C03F4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75175:96b5281e7963
Date: 2014-12-30 23:57 +0100
http://bitbucket.org/pypy/pypy/changeset/96b5281e7963/

Log:	Full implementation of the audioop module! Reuse large parts of C
	code from CPython, and use cffi to access it.

diff --git a/lib-python/3/test/test_audioop.py b/lib-python/3/test/test_audioop.py
--- a/lib-python/3/test/test_audioop.py
+++ b/lib-python/3/test/test_audioop.py
@@ -1,7 +1,6 @@
 import audioop
 import sys
 import unittest
-from test.support import run_unittest, impl_detail
 
 def pack(width, data):
     return b''.join(v.to_bytes(width, sys.byteorder, signed=True) for v in data)
@@ -171,7 +170,6 @@
         self.assertEqual(audioop.lin2lin(datas[4], 4, 2),
             packs[2](0, 0x1234, 0x4567, -0x4568, 0x7fff, -0x8000, -1))
 
-    @impl_detail(pypy=False)
     def test_adpcm2lin(self):
         self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 1, None),
                          (b'\x00\x00\x00\xff\x00\xff', (-179, 40)))
@@ -186,7 +184,6 @@
             self.assertEqual(audioop.adpcm2lin(b'\0' * 5, w, None),
                              (b'\0' * w * 10, (0, 0)))
 
-    @impl_detail(pypy=False)
     def test_lin2adpcm(self):
         self.assertEqual(audioop.lin2adpcm(datas[1], 1, None),
                          (b'\x07\x7f\x7f', (-221, 39)))
@@ -200,7 +197,6 @@
             self.assertEqual(audioop.lin2adpcm(b'\0' * w * 10, w, None),
                              (b'\0' * 5, (0, 0)))
 
-    @impl_detail(pypy=False)
     def test_lin2alaw(self):
         self.assertEqual(audioop.lin2alaw(datas[1], 1),
                          b'\xd5\x87\xa4\x24\xaa\x2a\x5a')
@@ -209,7 +205,6 @@
         self.assertEqual(audioop.lin2alaw(datas[4], 4),
                          b'\xd5\x87\xa4\x24\xaa\x2a\x55')
 
-    @impl_detail(pypy=False)
     def test_alaw2lin(self):
         encoded = b'\x00\x03\x24\x2a\x51\x54\x55\x58\x6b\x71\x7f'\
                   b'\x80\x83\xa4\xaa\xd1\xd4\xd5\xd8\xeb\xf1\xff'
@@ -224,7 +219,6 @@
             decoded = audioop.alaw2lin(encoded, w)
             self.assertEqual(audioop.lin2alaw(decoded, w), encoded)
 
-    @impl_detail(pypy=False)
     def test_lin2ulaw(self):
         self.assertEqual(audioop.lin2ulaw(datas[1], 1),
                          b'\xff\xad\x8e\x0e\x80\x00\x67')
@@ -233,7 +227,6 @@
         self.assertEqual(audioop.lin2ulaw(datas[4], 4),
                          b'\xff\xad\x8e\x0e\x80\x00\x7e')
 
-    @impl_detail(pypy=False)
     def test_ulaw2lin(self):
         encoded = b'\x00\x0e\x28\x3f\x57\x6a\x76\x7c\x7e\x7f'\
                   b'\x80\x8e\xa8\xbf\xd7\xea\xf6\xfc\xfe\xff'
@@ -348,7 +341,6 @@
         self.assertRaises(audioop.error,
             audioop.findmax, bytes(range(256)), -2392392)
 
-    @impl_detail(pypy=False)
     def test_issue7673(self):
         state = None
         for data, size in INVALID_DATA:
@@ -373,7 +365,6 @@
             self.assertRaises(audioop.error, audioop.lin2alaw, data, size)
             self.assertRaises(audioop.error, audioop.lin2adpcm, data, size, state)
 
-    @impl_detail(pypy=False)
     def test_wrongsize(self):
         data = b'abcdefgh'
         state = None
diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -527,25 +527,524 @@
                 d -= inrate
 
 
+from cffi import FFI
+ffi = FFI()
+ffi.cdef("""
+typedef short PyInt16;
+
+/* 2's complement (14-bit range) */
+unsigned char
+st_14linear2ulaw(PyInt16 pcm_val);
+PyInt16 st_ulaw2linear16(unsigned char);
+
+/* 2's complement (13-bit range) */
+unsigned char
+st_linear2alaw(PyInt16 pcm_val);
+PyInt16 st_alaw2linear16(unsigned char);
+
+
+void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+""")
+
+# This code is directly copied from CPython file: Modules/audioop.c
+_AUDIOOP_C_MODULE = """
+typedef short PyInt16;
+typedef int Py_Int32;
+
+/* Code shamelessly stolen from sox, 12.17.7, g711.c
+** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */
+
+/* From g711.c:
+ *
+ * December 30, 1994:
+ * Functions linear2alaw, linear2ulaw have been updated to correctly
+ * convert unquantized 16 bit values.
+ * Tables for direct u- to A-law and A- to u-law conversions have been
+ * corrected.
+ * Borge Lindberg, Center for PersonKommunikation, Aalborg University.
+ * bli at cpk.auc.dk
+ *
+ */
+#define BIAS 0x84   /* define the add-in bias for 16 bit samples */
+#define CLIP 32635
+#define SIGN_BIT        (0x80)          /* Sign bit for a A-law byte. */
+#define QUANT_MASK      (0xf)           /* Quantization field mask. */
+#define SEG_SHIFT       (4)             /* Left shift for segment number. */
+#define SEG_MASK        (0x70)          /* Segment field mask. */
+
+static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF,
+                              0x1FF, 0x3FF, 0x7FF, 0xFFF};
+static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF,
+                              0x3FF, 0x7FF, 0xFFF, 0x1FFF};
+
+static PyInt16
+search(PyInt16 val, PyInt16 *table, int size)
+{
+    int i;
+
+    for (i = 0; i < size; i++) {
+        if (val <= *table++)
+            return (i);
+    }
+    return (size);
+}
+#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc])
+#define st_alaw2linear16(uc) (_st_alaw2linear16[uc])
+
+static PyInt16 _st_ulaw2linear16[256] = {
+    -32124,  -31100,  -30076,  -29052,  -28028,  -27004,  -25980,
+    -24956,  -23932,  -22908,  -21884,  -20860,  -19836,  -18812,
+    -17788,  -16764,  -15996,  -15484,  -14972,  -14460,  -13948,
+    -13436,  -12924,  -12412,  -11900,  -11388,  -10876,  -10364,
+     -9852,   -9340,   -8828,   -8316,   -7932,   -7676,   -7420,
+     -7164,   -6908,   -6652,   -6396,   -6140,   -5884,   -5628,
+     -5372,   -5116,   -4860,   -4604,   -4348,   -4092,   -3900,
+     -3772,   -3644,   -3516,   -3388,   -3260,   -3132,   -3004,
+     -2876,   -2748,   -2620,   -2492,   -2364,   -2236,   -2108,
+     -1980,   -1884,   -1820,   -1756,   -1692,   -1628,   -1564,
+     -1500,   -1436,   -1372,   -1308,   -1244,   -1180,   -1116,
+     -1052,    -988,    -924,    -876,    -844,    -812,    -780,
+      -748,    -716,    -684,    -652,    -620,    -588,    -556,
+      -524,    -492,    -460,    -428,    -396,    -372,    -356,
+      -340,    -324,    -308,    -292,    -276,    -260,    -244,
+      -228,    -212,    -196,    -180,    -164,    -148,    -132,
+      -120,    -112,    -104,     -96,     -88,     -80,     -72,
+       -64,     -56,     -48,     -40,     -32,     -24,     -16,
+    -8,       0,   32124,   31100,   30076,   29052,   28028,
+     27004,   25980,   24956,   23932,   22908,   21884,   20860,
+     19836,   18812,   17788,   16764,   15996,   15484,   14972,
+     14460,   13948,   13436,   12924,   12412,   11900,   11388,
+     10876,   10364,    9852,    9340,    8828,    8316,    7932,
+      7676,    7420,    7164,    6908,    6652,    6396,    6140,
+      5884,    5628,    5372,    5116,    4860,    4604,    4348,
+      4092,    3900,    3772,    3644,    3516,    3388,    3260,
+      3132,    3004,    2876,    2748,    2620,    2492,    2364,
+      2236,    2108,    1980,    1884,    1820,    1756,    1692,
+      1628,    1564,    1500,    1436,    1372,    1308,    1244,
+      1180,    1116,    1052,     988,     924,     876,     844,
+       812,     780,     748,     716,     684,     652,     620,
+       588,     556,     524,     492,     460,     428,     396,
+       372,     356,     340,     324,     308,     292,     276,
+       260,     244,     228,     212,     196,     180,     164,
+       148,     132,     120,     112,     104,      96,      88,
+    80,      72,      64,      56,      48,      40,      32,
+    24,      16,       8,       0
+};
+
+/*
+ * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 14-bits.
+ *
+ * In order to simplify the encoding process, the original linear magnitude
+ * is biased by adding 33 which shifts the encoding range from (0 - 8158) to
+ * (33 - 8191). The result can be seen in the following encoding table:
+ *
+ *      Biased Linear Input Code        Compressed Code
+ *      ------------------------        ---------------
+ *      00000001wxyza                   000wxyz
+ *      0000001wxyzab                   001wxyz
+ *      000001wxyzabc                   010wxyz
+ *      00001wxyzabcd                   011wxyz
+ *      0001wxyzabcde                   100wxyz
+ *      001wxyzabcdef                   101wxyz
+ *      01wxyzabcdefg                   110wxyz
+ *      1wxyzabcdefgh                   111wxyz
+ *
+ * Each biased linear code has a leading 1 which identifies the segment
+ * number. The value of the segment number is equal to 7 minus the number
+ * of leading 0's. The quantization interval is directly available as the
+ * four bits wxyz.  * The trailing bits (a - h) are ignored.
+ *
+ * Ordinarily the complement of the resulting code word is used for
+ * transmission, and so the code word is complemented before it is returned.
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_14linear2ulaw(PyInt16 pcm_val)       /* 2's complement (14-bit range) */
+{
+    PyInt16         mask;
+    PyInt16         seg;
+    unsigned char   uval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 2;
+
+    /* u-law inverts all bits */
+    /* Get the sign and the magnitude of the value. */
+    if (pcm_val < 0) {
+        pcm_val = -pcm_val;
+        mask = 0x7F;
+    } else {
+        mask = 0xFF;
+    }
+    if ( pcm_val > CLIP ) pcm_val = CLIP;           /* clip the magnitude */
+    pcm_val += (BIAS >> 2);
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_uend, 8);
+
+    /*
+     * Combine the sign, segment, quantization bits;
+     * and complement the code word.
+     */
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF);
+        return (uval ^ mask);
+    }
+
+}
+
+static PyInt16 _st_alaw2linear16[256] = {
+     -5504,   -5248,   -6016,   -5760,   -4480,   -4224,   -4992,
+     -4736,   -7552,   -7296,   -8064,   -7808,   -6528,   -6272,
+     -7040,   -6784,   -2752,   -2624,   -3008,   -2880,   -2240,
+     -2112,   -2496,   -2368,   -3776,   -3648,   -4032,   -3904,
+     -3264,   -3136,   -3520,   -3392,  -22016,  -20992,  -24064,
+    -23040,  -17920,  -16896,  -19968,  -18944,  -30208,  -29184,
+    -32256,  -31232,  -26112,  -25088,  -28160,  -27136,  -11008,
+    -10496,  -12032,  -11520,   -8960,   -8448,   -9984,   -9472,
+    -15104,  -14592,  -16128,  -15616,  -13056,  -12544,  -14080,
+    -13568,    -344,    -328,    -376,    -360,    -280,    -264,
+      -312,    -296,    -472,    -456,    -504,    -488,    -408,
+      -392,    -440,    -424,     -88,     -72,    -120,    -104,
+       -24,      -8,     -56,     -40,    -216,    -200,    -248,
+      -232,    -152,    -136,    -184,    -168,   -1376,   -1312,
+     -1504,   -1440,   -1120,   -1056,   -1248,   -1184,   -1888,
+     -1824,   -2016,   -1952,   -1632,   -1568,   -1760,   -1696,
+      -688,    -656,    -752,    -720,    -560,    -528,    -624,
+      -592,    -944,    -912,   -1008,    -976,    -816,    -784,
+      -880,    -848,    5504,    5248,    6016,    5760,    4480,
+      4224,    4992,    4736,    7552,    7296,    8064,    7808,
+      6528,    6272,    7040,    6784,    2752,    2624,    3008,
+      2880,    2240,    2112,    2496,    2368,    3776,    3648,
+      4032,    3904,    3264,    3136,    3520,    3392,   22016,
+     20992,   24064,   23040,   17920,   16896,   19968,   18944,
+     30208,   29184,   32256,   31232,   26112,   25088,   28160,
+     27136,   11008,   10496,   12032,   11520,    8960,    8448,
+      9984,    9472,   15104,   14592,   16128,   15616,   13056,
+     12544,   14080,   13568,     344,     328,     376,     360,
+       280,     264,     312,     296,     472,     456,     504,
+       488,     408,     392,     440,     424,      88,      72,
+       120,     104,      24,       8,      56,      40,     216,
+       200,     248,     232,     152,     136,     184,     168,
+      1376,    1312,    1504,    1440,    1120,    1056,    1248,
+      1184,    1888,    1824,    2016,    1952,    1632,    1568,
+      1760,    1696,     688,     656,     752,     720,     560,
+       528,     624,     592,     944,     912,    1008,     976,
+       816,     784,     880,     848
+};
+
+/*
+ * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 13-bits.
+ *
+ *              Linear Input Code       Compressed Code
+ *      ------------------------        ---------------
+ *      0000000wxyza                    000wxyz
+ *      0000001wxyza                    001wxyz
+ *      000001wxyzab                    010wxyz
+ *      00001wxyzabc                    011wxyz
+ *      0001wxyzabcd                    100wxyz
+ *      001wxyzabcde                    101wxyz
+ *      01wxyzabcdef                    110wxyz
+ *      1wxyzabcdefg                    111wxyz
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */
+{
+    PyInt16         mask;
+    short           seg;
+    unsigned char   aval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 3;
+
+    /* A-law using even bit inversion */
+    if (pcm_val >= 0) {
+        mask = 0xD5;            /* sign (7th) bit = 1 */
+    } else {
+        mask = 0x55;            /* sign bit = 0 */
+        pcm_val = -pcm_val - 1;
+    }
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_aend, 8);
+
+    /* Combine the sign, segment, and quantization bits. */
+
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        aval = (unsigned char) seg << SEG_SHIFT;
+        if (seg < 2)
+            aval |= (pcm_val >> 1) & QUANT_MASK;
+        else
+            aval |= (pcm_val >> seg) & QUANT_MASK;
+        return (aval ^ mask);
+    }
+}
+/* End of code taken from sox */
+
+/* Intel ADPCM step variation table */
+static int indexTable[16] = {
+    -1, -1, -1, -1, 2, 4, 6, 8,
+    -1, -1, -1, -1, 2, 4, 6, 8,
+};
+
+static int stepsizeTable[89] = {
+    7, 8, 9, 10, 11, 12, 13, 14, 16, 17,
+    19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
+    50, 55, 60, 66, 73, 80, 88, 97, 107, 118,
+    130, 143, 157, 173, 190, 209, 230, 253, 279, 307,
+    337, 371, 408, 449, 494, 544, 598, 658, 724, 796,
+    876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066,
+    2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358,
+    5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899,
+    15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767
+};
+
+#define CHARP(cp, i) ((signed char *)(cp+i))
+#define SHORTP(cp, i) ((short *)(cp+i))
+#define LONGP(cp, i) ((Py_Int32 *)(cp+i))
+"""
+
+lib = ffi.verify(_AUDIOOP_C_MODULE + """
+void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, outputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 1;
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = ((int)*CHARP(cp, i)) << 8;
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16;
+
+        /* Step 1 - compute difference with previous value */
+        diff = val - valpred;
+        sign = (diff < 0) ? 8 : 0;
+        if ( sign ) diff = (-diff);
+
+        /* Step 2 - Divide and clamp */
+        /* Note:
+        ** This code *approximately* computes:
+        **    delta = diff*4/step;
+        **    vpdiff = (delta+0.5)*step/4;
+        ** but in shift step bits are dropped. The net result of this
+        ** is that even if you have fast mul/div hardware you cannot
+        ** put it to good use since the fixup would be too expensive.
+        */
+        delta = 0;
+        vpdiff = (step >> 3);
+
+        if ( diff >= step ) {
+            delta = 4;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step  ) {
+            delta |= 2;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step ) {
+            delta |= 1;
+            vpdiff += step;
+        }
+
+        /* Step 3 - Update previous value */
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 4 - Clamp previous value to 16 bits */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 5 - Assemble value, update index and step values */
+        delta |= sign;
+
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( bufferstep ) {
+            outputbuffer = (delta << 4) & 0xf0;
+        } else {
+            *ncp++ = (delta & 0x0f) | outputbuffer;
+        }
+        bufferstep = !bufferstep;
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+
+
+void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, inputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 0;
+
+    for ( i=0; i < len*size*2; i += size ) {
+        /* Step 1 - get the delta value and compute next index */
+        if ( bufferstep ) {
+            delta = inputbuffer & 0xf;
+        } else {
+            inputbuffer = *cp++;
+            delta = (inputbuffer >> 4) & 0xf;
+        }
+
+        bufferstep = !bufferstep;
+
+        /* Step 2 - Find new index value (for later) */
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+
+        /* Step 3 - Separate sign and magnitude */
+        sign = delta & 8;
+        delta = delta & 7;
+
+        /* Step 4 - Compute difference and new predicted value */
+        /*
+        ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment
+        ** in adpcm_coder.
+        */
+        vpdiff = step >> 3;
+        if ( delta & 4 ) vpdiff += step;
+        if ( delta & 2 ) vpdiff += step>>1;
+        if ( delta & 1 ) vpdiff += step>>2;
+
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 5 - clamp output value */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 6 - Update step value */
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8);
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred);
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16);
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+""")
+
+def _get_lin_samples(cp, size):
+    for sample in _get_samples(cp, size):
+        if size == 1:
+            yield sample << 8
+        elif size == 2:
+            yield sample
+        elif size == 4:
+            yield sample >> 16
+
+def _put_lin_sample(result, size, i, sample):
+    if size == 1:
+        sample >>= 8
+    elif size == 2:
+        pass
+    elif size == 4:
+        sample <<= 16
+    _put_sample(result, size, i, sample)
+
 def lin2ulaw(cp, size):
-    raise NotImplementedError()
+    _check_params(len(cp), size)
+    rv = ffi.new("unsigned char[]", _sample_count(cp, size))
+    for i, sample in enumerate(_get_lin_samples(cp, size)):
+        rv[i] = lib.st_14linear2ulaw(sample)
+    return ffi.buffer(rv)[:]
 
 
 def ulaw2lin(cp, size):
-    raise NotImplementedError()
+    _check_size(size)
+    rv = ffi.new("unsigned char[]", len(cp) * size)
+    result = ffi.buffer(rv)
+    for i, value in enumerate(cp):
+        sample = lib.st_ulaw2linear16(value)
+        _put_lin_sample(result, size, i, sample)
+    return result[:]
 
 
 def lin2alaw(cp, size):
-    raise NotImplementedError()
+    _check_params(len(cp), size)
+    rv = ffi.new("unsigned char[]", _sample_count(cp, size))
+    for i, sample in enumerate(_get_lin_samples(cp, size)):
+        rv[i] = lib.st_linear2alaw(sample)
+    return ffi.buffer(rv)[:]
 
 
 def alaw2lin(cp, size):
-    raise NotImplementedError()
+    _check_size(size)
+    rv = ffi.new("unsigned char[]", len(cp) * size)
+    result = ffi.buffer(rv)
+    for i, value in enumerate(cp):
+        sample = lib.st_alaw2linear16(value)
+        _put_lin_sample(result, size, i, sample)
+    return result[:]
 
 
 def lin2adpcm(cp, size, state):
-    raise NotImplementedError()
+    _check_params(len(cp), size)
+    if state is None:
+        state = (0, 0)
+    rv = ffi.new("unsigned char[]", len(cp) // size // 2)
+    state_ptr = ffi.new("int[]", state)
+    lib.lin2adcpm(rv, cp, len(cp), size, state_ptr)
+    return ffi.buffer(rv)[:], tuple(state_ptr)
 
 
 def adpcm2lin(cp, size, state):
-    raise NotImplementedError()
+    _check_size(size)
+    if state is None:
+        state = (0, 0)
+    rv = ffi.new("unsigned char[]", len(cp) * size * 2)
+    state_ptr = ffi.new("int[]", state)
+    lib.adcpm2lin(rv, cp, len(cp), size, state_ptr)
+    return ffi.buffer(rv)[:], tuple(state_ptr)
+

From noreply at buildbot.pypy.org  Wed Dec 31 03:52:58 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Wed, 31 Dec 2014 03:52:58 +0100 (CET)
Subject: [pypy-commit] pypy default: add zlibVersion (for py3.3)
Message-ID: <20141231025258.959E31C3D78@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r75176:11240a20d1d2
Date: 2014-12-30 18:52 -0800
http://bitbucket.org/pypy/pypy/changeset/11240a20d1d2/

Log:	add zlibVersion (for py3.3)

diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -167,6 +167,7 @@
 
 _deflateSetDictionary = zlib_external('deflateSetDictionary', [z_stream_p, Bytefp, uInt], rffi.INT)
 _inflateSetDictionary = zlib_external('inflateSetDictionary', [z_stream_p, Bytefp, uInt], rffi.INT)
+_zlibVersion = zlib_external('zlibVersion', [], rffi.CCHARP)
 
 # ____________________________________________________________
 
@@ -208,6 +209,10 @@
         checksum = _adler32(start, rffi.cast(Bytefp, bytes), len(string))
     return checksum
 
+def zlibVersion():
+    """Return the runtime version of zlib library"""
+    return rffi.charp2str(_zlibVersion())
+
 # ____________________________________________________________
 
 class RZlibError(Exception):
diff --git a/rpython/rlib/test/test_rzlib.py b/rpython/rlib/test/test_rzlib.py
--- a/rpython/rlib/test/test_rzlib.py
+++ b/rpython/rlib/test/test_rzlib.py
@@ -269,3 +269,7 @@
         assert unused > 0
         buf = buf[-unused:]
     rzlib.deflateEnd(stream)
+
+def test_zlibVersion():
+    runtime_version = rzlib.zlibVersion()
+    assert runtime_version[0] == rzlib.ZLIB_VERSION[0]

From noreply at buildbot.pypy.org  Wed Dec 31 07:59:46 2014
From: noreply at buildbot.pypy.org (larstiq)
Date: Wed, 31 Dec 2014 07:59:46 +0100 (CET)
Subject: [pypy-commit] pypy osx-shared: Use @rpath on OSX to find
	libpypy-c.dylib
Message-ID: <20141231065946.C7EC71C243B@cobra.cs.uni-duesseldorf.de>

Author: Wouter van Heyst <wouter.pypy at richtlijn.be>
Branch: osx-shared
Changeset: r75177:3fa7727c46a3
Date: 2014-12-21 19:44 +0200
http://bitbucket.org/pypy/pypy/changeset/3fa7727c46a3/

Log:	Use @rpath on OSX to find libpypy-c.dylib

diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -13,7 +13,7 @@
 
     def _args_for_shared(self, args):
         return (list(self.shared_only)
-                + ['-dynamiclib', '-undefined', 'dynamic_lookup']
+                + ['-dynamiclib', '-install_name', '@rpath/$(TARGET)', '-undefined', 'dynamic_lookup']
                 + args)
 
     def _include_dirs_for_libffi(self):
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -180,7 +180,12 @@
                    'int $(PYPY_MAIN_FUNCTION)(int, char*[]); '
                    'int main(int argc, char* argv[]) '
                    '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); }" > $@')
-            m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'],
+            if sys.platform == 'darwin':
+                m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'],
+                       ['$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@',
+                        'install_name_tool -add_rpath @executable_path $@'])
+            else:
+                m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'],
                    '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ -Wl,-rpath=\'$$ORIGIN/\'')
 
         return m

From noreply at buildbot.pypy.org  Wed Dec 31 07:59:48 2014
From: noreply at buildbot.pypy.org (larstiq)
Date: Wed, 31 Dec 2014 07:59:48 +0100 (CET)
Subject: [pypy-commit] pypy osx-shared: Introduce a RPATH_FLAGS variable to
 factor out the platform dependent rpath flags
Message-ID: <20141231065948.2C8951C243B@cobra.cs.uni-duesseldorf.de>

Author: Wouter van Heyst <wouter.pypy at richtlijn.be>
Branch: osx-shared
Changeset: r75178:b4dea196a719
Date: 2014-12-22 20:35 +0200
http://bitbucket.org/pypy/pypy/changeset/b4dea196a719/

Log:	Introduce a RPATH_FLAGS variable to factor out the platform
	dependent rpath flags

diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -140,6 +140,11 @@
         rel_libdirs = [rpyrel(libdir) for libdir in
                        self.preprocess_library_dirs(eci.library_dirs)]
 
+        if os.platform == 'darwin':
+            rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path']
+        else:
+            rpath_flags = ['-Wl,-rpath=\'$$ORIGIN/\'']
+
         m.comment('automatically generated makefile')
         definitions = [
             ('RPYDIR', '"%s"' % rpydir),
@@ -158,6 +163,7 @@
             ('CC', self.cc),
             ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'),
             ('LINKFILES', eci.link_files),
+            ('RPATH_FLAGS', rpath_flags),
             ]
         for args in definitions:
             m.definition(*args)
@@ -180,13 +186,8 @@
                    'int $(PYPY_MAIN_FUNCTION)(int, char*[]); '
                    'int main(int argc, char* argv[]) '
                    '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); }" > $@')
-            if sys.platform == 'darwin':
-                m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'],
-                       ['$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@',
-                        'install_name_tool -add_rpath @executable_path $@'])
-            else:
-                m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'],
-                   '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ -Wl,-rpath=\'$$ORIGIN/\'')
+            m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'],
+                   '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ $(RPATH_FLAGS)')
 
         return m
 

From noreply at buildbot.pypy.org  Wed Dec 31 07:59:49 2014
From: noreply at buildbot.pypy.org (larstiq)
Date: Wed, 31 Dec 2014 07:59:49 +0100 (CET)
Subject: [pypy-commit] pypy osx-shared: Note in the embedding tutorial that
 OSX requires different linking.
Message-ID: <20141231065949.58E641C243B@cobra.cs.uni-duesseldorf.de>

Author: Wouter van Heyst <wouter.pypy at richtlijn.be>
Branch: osx-shared
Changeset: r75179:0418516e0931
Date: 2014-12-22 20:35 +0200
http://bitbucket.org/pypy/pypy/changeset/0418516e0931/

Log:	Note in the embedding tutorial that OSX requires different linking.

diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -97,12 +97,18 @@
       return res;
     }
 
-If we save it as ``x.c`` now, compile it and run it with::
+If we save it as ``x.c`` now, compile it and run it (on linux) with::
 
     fijal at hermann:/opt/pypy$ gcc -o x x.c -lpypy-c -L.
     fijal at hermann:/opt/pypy$ LD_LIBRARY_PATH=. ./x
     hello from pypy
 
+on OSX it is necessary to set the rpath of the binary if one wants to link to it::
+
+    gcc -o x x.c -lpypy-c -L. -Wl,-rpath -Wl, at executable_path
+    ./x
+    hello from pypy
+
 Worked!
 
 .. note:: If the compilation fails because of missing PyPy.h header file,

From noreply at buildbot.pypy.org  Wed Dec 31 07:59:50 2014
From: noreply at buildbot.pypy.org (larstiq)
Date: Wed, 31 Dec 2014 07:59:50 +0100 (CET)
Subject: [pypy-commit] pypy osx-shared: Fix typo: s/os/sys/.
Message-ID: <20141231065950.82F121C243B@cobra.cs.uni-duesseldorf.de>

Author: Wouter van Heyst <wouter.pypy at richtlijn.be>
Branch: osx-shared
Changeset: r75180:8d31fd6373c2
Date: 2014-12-22 22:33 +0200
http://bitbucket.org/pypy/pypy/changeset/8d31fd6373c2/

Log:	Fix typo: s/os/sys/.

diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -140,7 +140,7 @@
         rel_libdirs = [rpyrel(libdir) for libdir in
                        self.preprocess_library_dirs(eci.library_dirs)]
 
-        if os.platform == 'darwin':
+        if sys.platform == 'darwin':
             rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path']
         else:
             rpath_flags = ['-Wl,-rpath=\'$$ORIGIN/\'']

From noreply at buildbot.pypy.org  Wed Dec 31 07:59:51 2014
From: noreply at buildbot.pypy.org (larstiq)
Date: Wed, 31 Dec 2014 07:59:51 +0100 (CET)
Subject: [pypy-commit] pypy osx-shared: Eliminate platform dependence in
 BasePosix.gen_makefile by moving rpath_flags onto BasePosix and overriding
 in Darwin.
Message-ID: <20141231065951.BB3671C243B@cobra.cs.uni-duesseldorf.de>

Author: Wouter van Heyst <wouter.pypy at richtlijn.be>
Branch: osx-shared
Changeset: r75181:ae514a359f3e
Date: 2014-12-29 21:19 +0200
http://bitbucket.org/pypy/pypy/changeset/ae514a359f3e/

Log:	Eliminate platform dependence in BasePosix.gen_makefile by moving
	rpath_flags onto BasePosix and overriding in Darwin.

diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -11,6 +11,10 @@
     so_ext = 'dylib'
     DEFAULT_CC = 'clang'
 
+    def __init__(self, cc=None):
+        super(Darwin, self).__init__(cc)
+        self.rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path']
+
     def _args_for_shared(self, args):
         return (list(self.shared_only)
                 + ['-dynamiclib', '-install_name', '@rpath/$(TARGET)', '-undefined', 'dynamic_lookup']
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -17,6 +17,7 @@
 
     def __init__(self, cc=None):
         self.cc = cc or os.environ.get('CC', self.DEFAULT_CC)
+        self.rpath_flags = ['-Wl,-rpath=\'$$ORIGIN/\'']
 
     def _libs(self, libraries):
         return ['-l%s' % lib for lib in libraries]
@@ -140,11 +141,6 @@
         rel_libdirs = [rpyrel(libdir) for libdir in
                        self.preprocess_library_dirs(eci.library_dirs)]
 
-        if sys.platform == 'darwin':
-            rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path']
-        else:
-            rpath_flags = ['-Wl,-rpath=\'$$ORIGIN/\'']
-
         m.comment('automatically generated makefile')
         definitions = [
             ('RPYDIR', '"%s"' % rpydir),
@@ -163,7 +159,7 @@
             ('CC', self.cc),
             ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'),
             ('LINKFILES', eci.link_files),
-            ('RPATH_FLAGS', rpath_flags),
+            ('RPATH_FLAGS', self.rpath_flags),
             ]
         for args in definitions:
             m.definition(*args)

From noreply at buildbot.pypy.org  Wed Dec 31 07:59:52 2014
From: noreply at buildbot.pypy.org (larstiq)
Date: Wed, 31 Dec 2014 07:59:52 +0100 (CET)
Subject: [pypy-commit] pypy osx-shared: Fix TestShared.test_entrypoint on
	OSX.
Message-ID: <20141231065952.E57B51C243B@cobra.cs.uni-duesseldorf.de>

Author: Wouter van Heyst <wouter.pypy at richtlijn.be>
Branch: osx-shared
Changeset: r75182:79717888c2b6
Date: 2014-12-30 20:12 +0200
http://bitbucket.org/pypy/pypy/changeset/79717888c2b6/

Log:	Fix TestShared.test_entrypoint on OSX.

diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1386,6 +1386,8 @@
         ext_suffix = '.so'
         if cbuilder.eci.platform.name == 'msvc':
             ext_suffix = '.dll'
+        elif cbuilder.eci.platform.name.startswith('darwin'):
+            ext_suffix = '.dylib'
         libname = cbuilder.executable_name.join('..', 'lib' +
                                       cbuilder.modulename + ext_suffix)
         lib = ctypes.CDLL(str(libname))

From noreply at buildbot.pypy.org  Wed Dec 31 07:59:54 2014
From: noreply at buildbot.pypy.org (larstiq)
Date: Wed, 31 Dec 2014 07:59:54 +0100 (CET)
Subject: [pypy-commit] pypy osx-shared: Move rpath_flags from Posix platform
 instance to the class.
Message-ID: <20141231065954.16DFB1C243B@cobra.cs.uni-duesseldorf.de>

Author: Wouter van Heyst <wouter.pypy at richtlijn.be>
Branch: osx-shared
Changeset: r75183:47c7b39c1da6
Date: 2014-12-31 08:56 +0200
http://bitbucket.org/pypy/pypy/changeset/47c7b39c1da6/

Log:	Move rpath_flags from Posix platform instance to the class.

diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -10,10 +10,7 @@
 
     so_ext = 'dylib'
     DEFAULT_CC = 'clang'
-
-    def __init__(self, cc=None):
-        super(Darwin, self).__init__(cc)
-        self.rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path']
+    rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path']
 
     def _args_for_shared(self, args):
         return (list(self.shared_only)
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -14,10 +14,10 @@
     relevant_environ = ('CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH')
 
     DEFAULT_CC = 'gcc'
+    rpath_flags = ['-Wl,-rpath=\'$$ORIGIN/\'']
 
     def __init__(self, cc=None):
         self.cc = cc or os.environ.get('CC', self.DEFAULT_CC)
-        self.rpath_flags = ['-Wl,-rpath=\'$$ORIGIN/\'']
 
     def _libs(self, libraries):
         return ['-l%s' % lib for lib in libraries]

From noreply at buildbot.pypy.org  Wed Dec 31 07:59:55 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 31 Dec 2014 07:59:55 +0100 (CET)
Subject: [pypy-commit] pypy default: Merged in larstiq/pypy/osx-shared (pull
	request #293)
Message-ID: <20141231065955.3C5381C243B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75184:3f1e8d12f774
Date: 2014-12-31 08:59 +0200
http://bitbucket.org/pypy/pypy/changeset/3f1e8d12f774/

Log:	Merged in larstiq/pypy/osx-shared (pull request #293)

	--shared support on OSX

diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -97,12 +97,18 @@
       return res;
     }
 
-If we save it as ``x.c`` now, compile it and run it with::
+If we save it as ``x.c`` now, compile it and run it (on linux) with::
 
     fijal at hermann:/opt/pypy$ gcc -o x x.c -lpypy-c -L.
     fijal at hermann:/opt/pypy$ LD_LIBRARY_PATH=. ./x
     hello from pypy
 
+on OSX it is necessary to set the rpath of the binary if one wants to link to it::
+
+    gcc -o x x.c -lpypy-c -L. -Wl,-rpath -Wl, at executable_path
+    ./x
+    hello from pypy
+
 Worked!
 
 .. note:: If the compilation fails because of missing PyPy.h header file,
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -1386,6 +1386,8 @@
         ext_suffix = '.so'
         if cbuilder.eci.platform.name == 'msvc':
             ext_suffix = '.dll'
+        elif cbuilder.eci.platform.name.startswith('darwin'):
+            ext_suffix = '.dylib'
         libname = cbuilder.executable_name.join('..', 'lib' +
                                       cbuilder.modulename + ext_suffix)
         lib = ctypes.CDLL(str(libname))
diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -10,10 +10,11 @@
 
     so_ext = 'dylib'
     DEFAULT_CC = 'clang'
+    rpath_flags = ['-Wl,-rpath', '-Wl, at executable_path']
 
     def _args_for_shared(self, args):
         return (list(self.shared_only)
-                + ['-dynamiclib', '-undefined', 'dynamic_lookup']
+                + ['-dynamiclib', '-install_name', '@rpath/$(TARGET)', '-undefined', 'dynamic_lookup']
                 + args)
 
     def _include_dirs_for_libffi(self):
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -14,6 +14,7 @@
     relevant_environ = ('CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH')
 
     DEFAULT_CC = 'gcc'
+    rpath_flags = ['-Wl,-rpath=\'$$ORIGIN/\'']
 
     def __init__(self, cc=None):
         self.cc = cc or os.environ.get('CC', self.DEFAULT_CC)
@@ -158,6 +159,7 @@
             ('CC', self.cc),
             ('CC_LINK', eci.use_cpp_linker and 'g++' or '$(CC)'),
             ('LINKFILES', eci.link_files),
+            ('RPATH_FLAGS', self.rpath_flags),
             ]
         for args in definitions:
             m.definition(*args)
@@ -181,7 +183,7 @@
                    'int main(int argc, char* argv[]) '
                    '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); }" > $@')
             m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'],
-                   '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ -Wl,-rpath=\'$$ORIGIN/\'')
+                   '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ $(RPATH_FLAGS)')
 
         return m
 

From noreply at buildbot.pypy.org  Wed Dec 31 08:14:02 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 31 Dec 2014 08:14:02 +0100 (CET)
Subject: [pypy-commit] pypy default: Make this default on all platforms
Message-ID: <20141231071402.22DCF1C00BB@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75185:f257da7dc012
Date: 2014-12-31 09:13 +0200
http://bitbucket.org/pypy/pypy/changeset/f257da7dc012/

Log:	Make this default on all platforms

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -234,8 +234,7 @@
             enable_translationmodules(config)
 
         config.translation.suggest(check_str_without_nul=True)
-        if sys.platform.startswith('linux'):
-            config.translation.suggest(shared=True)
+        config.translation.suggest(shared=True)
 
         if config.translation.thread:
             config.objspace.usemodules.thread = True

From noreply at buildbot.pypy.org  Wed Dec 31 08:14:03 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 31 Dec 2014 08:14:03 +0100 (CET)
Subject: [pypy-commit] pypy default: This is a spooky check (other people
 enforce that inputargs are unique anyway)
Message-ID: <20141231071403.6843E1C00BB@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r75186:764546158178
Date: 2014-12-31 09:13 +0200
http://bitbucket.org/pypy/pypy/changeset/764546158178/

Log:	This is a spooky check (other people enforce that inputargs are
	unique anyway)

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -234,11 +234,7 @@
         self.short_boxes = exported_state.short_boxes
         self.initial_virtual_state = target_token.virtual_state
 
-        seen = {}
         for box in self.inputargs:
-            if box in seen:
-                continue
-            seen[box] = None
             preamble_value = exported_state.exported_values[box]
             value = self.optimizer.getvalue(box)
             value.import_from(preamble_value, self.optimizer)

From noreply at buildbot.pypy.org  Wed Dec 31 08:42:13 2014
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 31 Dec 2014 08:42:13 +0100 (CET)
Subject: [pypy-commit] pypy optresult: merge default (badly)
Message-ID: <20141231074213.5689F1C243B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: optresult
Changeset: r75187:7bdcf543fd1c
Date: 2014-12-31 09:42 +0200
http://bitbucket.org/pypy/pypy/changeset/7bdcf543fd1c/

Log:	merge default (badly)

diff too long, truncating to 2000 out of 5605 lines

diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -225,9 +225,9 @@
     >>> element.remove(subelement)
     >>> serialize(element) # 5
     '<tag key="value" />'
-    >>> element.remove(subelement)
+    >>> element.remove(subelement)    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ValueError: list.remove(x): x not in list
+    ValueError: list.remove(...
     >>> serialize(element) # 6
     '<tag key="value" />'
     >>> element[0:0] = [subelement, subelement, subelement]
diff --git a/lib_pypy/readline.py b/lib_pypy/readline.py
--- a/lib_pypy/readline.py
+++ b/lib_pypy/readline.py
@@ -6,4 +6,11 @@
 are only stubs at the moment.
 """
 
-from pyrepl.readline import *
+try:
+    from pyrepl.readline import *
+except ImportError:
+    import sys
+    if sys.platform == 'win32':
+        raise ImportError("the 'readline' module is not available on Windows"
+                          " (on either PyPy or CPython)")
+    raise
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -83,17 +83,16 @@
 
 class __extend__(ast.BoolOp):
 
-    def _accept_jump_if_any_is(self, gen, condition, target):
-        self.values[0].accept_jump_if(gen, condition, target)
-        for i in range(1, len(self.values)):
+    def _accept_jump_if_any_is(self, gen, condition, target, skip_last=0):
+        for i in range(len(self.values) - skip_last):
             self.values[i].accept_jump_if(gen, condition, target)
 
     def accept_jump_if(self, gen, condition, target):
         if condition and self.op == ast.And or \
                 (not condition and self.op == ast.Or):
             end = gen.new_block()
-            self._accept_jump_if_any_is(gen, not condition, end)
-            gen.emit_jump(ops.JUMP_FORWARD, target)
+            self._accept_jump_if_any_is(gen, not condition, end, skip_last=1)
+            self.values[-1].accept_jump_if(gen, condition, target)
             gen.use_next_block(end)
         else:
             self._accept_jump_if_any_is(gen, condition, target)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -62,10 +62,54 @@
 
 eptype("intptr_t",  rffi.INTPTR_T,  ctypeprim.W_CTypePrimitiveSigned)
 eptype("uintptr_t", rffi.UINTPTR_T, ctypeprim.W_CTypePrimitiveUnsigned)
-eptype("ptrdiff_t", rffi.INTPTR_T,  ctypeprim.W_CTypePrimitiveSigned) # <-xxx
 eptype("size_t",    rffi.SIZE_T,    ctypeprim.W_CTypePrimitiveUnsigned)
 eptype("ssize_t",   rffi.SSIZE_T,   ctypeprim.W_CTypePrimitiveSigned)
 
+_WCTSigned = ctypeprim.W_CTypePrimitiveSigned
+_WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned
+
+eptype("ptrdiff_t", getattr(rffi, 'PTRDIFF_T', rffi.INTPTR_T), _WCTSigned)
+eptype("intmax_t",  getattr(rffi, 'INTMAX_T',  rffi.LONGLONG), _WCTSigned)
+eptype("uintmax_t", getattr(rffi, 'UINTMAX_T', rffi.LONGLONG), _WCTUnsign)
+
+if hasattr(rffi, 'INT_LEAST8_T'):
+    eptype("int_least8_t",  rffi.INT_LEAST8_T,  _WCTSigned)
+    eptype("int_least16_t", rffi.INT_LEAST16_T, _WCTSigned)
+    eptype("int_least32_t", rffi.INT_LEAST32_T, _WCTSigned)
+    eptype("int_least64_t", rffi.INT_LEAST64_T, _WCTSigned)
+    eptype("uint_least8_t", rffi.UINT_LEAST8_T,  _WCTUnsign)
+    eptype("uint_least16_t",rffi.UINT_LEAST16_T, _WCTUnsign)
+    eptype("uint_least32_t",rffi.UINT_LEAST32_T, _WCTUnsign)
+    eptype("uint_least64_t",rffi.UINT_LEAST64_T, _WCTUnsign)
+else:
+    eptypesize("int_least8_t",   1, _WCTSigned)
+    eptypesize("uint_least8_t",  1, _WCTUnsign)
+    eptypesize("int_least16_t",  2, _WCTSigned)
+    eptypesize("uint_least16_t", 2, _WCTUnsign)
+    eptypesize("int_least32_t",  4, _WCTSigned)
+    eptypesize("uint_least32_t", 4, _WCTUnsign)
+    eptypesize("int_least64_t",  8, _WCTSigned)
+    eptypesize("uint_least64_t", 8, _WCTUnsign)
+
+if hasattr(rffi, 'INT_FAST8_T'):
+    eptype("int_fast8_t",  rffi.INT_FAST8_T,  _WCTSigned)
+    eptype("int_fast16_t", rffi.INT_FAST16_T, _WCTSigned)
+    eptype("int_fast32_t", rffi.INT_FAST32_T, _WCTSigned)
+    eptype("int_fast64_t", rffi.INT_FAST64_T, _WCTSigned)
+    eptype("uint_fast8_t", rffi.UINT_FAST8_T,  _WCTUnsign)
+    eptype("uint_fast16_t",rffi.UINT_FAST16_T, _WCTUnsign)
+    eptype("uint_fast32_t",rffi.UINT_FAST32_T, _WCTUnsign)
+    eptype("uint_fast64_t",rffi.UINT_FAST64_T, _WCTUnsign)
+else:
+    eptypesize("int_fast8_t",   1, _WCTSigned)
+    eptypesize("uint_fast8_t",  1, _WCTUnsign)
+    eptypesize("int_fast16_t",  2, _WCTSigned)
+    eptypesize("uint_fast16_t", 2, _WCTUnsign)
+    eptypesize("int_fast32_t",  4, _WCTSigned)
+    eptypesize("uint_fast32_t", 4, _WCTUnsign)
+    eptypesize("int_fast64_t",  8, _WCTSigned)
+    eptypesize("uint_fast64_t", 8, _WCTUnsign)
+
 @unwrap_spec(name=str)
 def new_primitive_type(space, name):
     try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -397,7 +397,7 @@
 def test_invalid_indexing():
     p = new_primitive_type("int")
     x = cast(p, 42)
-    py.test.raises(TypeError, "p[0]")
+    py.test.raises(TypeError, "x[0]")
 
 def test_default_str():
     BChar = new_primitive_type("char")
@@ -2718,7 +2718,16 @@
 def test_nonstandard_integer_types():
     for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t',
                      'uint32_t', 'int64_t', 'uint64_t', 'intptr_t',
-                     'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t']:
+                     'uintptr_t', 'ptrdiff_t', 'size_t', 'ssize_t',
+                     'int_least8_t',  'uint_least8_t',
+                     'int_least16_t', 'uint_least16_t',
+                     'int_least32_t', 'uint_least32_t',
+                     'int_least64_t', 'uint_least64_t',
+                     'int_fast8_t',  'uint_fast8_t',
+                     'int_fast16_t', 'uint_fast16_t',
+                     'int_fast32_t', 'uint_fast32_t',
+                     'int_fast64_t', 'uint_fast64_t',
+                     'intmax_t', 'uintmax_t']:
         new_primitive_type(typename)    # works
 
 def test_cannot_convert_unicode_to_charp():
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,4 +1,5 @@
 from rpython.rlib.buffer import Buffer
+from rpython.rtyper.lltypesystem import rffi
 
 # XXX not the most efficient implementation
 
@@ -20,3 +21,7 @@
     def setitem(self, index, char):
         ll_buffer = self.datainstance.ll_buffer
         ll_buffer[index] = char
+
+    def get_raw_address(self):
+        ll_buffer = self.datainstance.ll_buffer
+        return rffi.cast(rffi.CCHARP, ll_buffer)
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1144,6 +1144,15 @@
         b[3] = b'x'
         assert b[3] == b'x'
 
+    def test_pypy_raw_address(self):
+        import _rawffi
+        S = _rawffi.Structure((40, 1))
+        s = S(autofree=True)
+        addr = buffer(s)._pypy_raw_address()
+        assert type(addr) is int
+        assert buffer(s)._pypy_raw_address() == addr
+        assert buffer(s, 10)._pypy_raw_address() == addr + 10
+
     def test_union(self):
         import _rawffi
         longsize = _rawffi.sizeof('l')
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -244,6 +244,9 @@
     def getitem(self, index):
         return self.ptr[index]
 
+    def get_raw_address(self):
+        return rffi.cast(rffi.CCHARP, self.ptr)
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -30,6 +30,7 @@
                 'get_referrers': 'referents.get_referrers',
                 '_dump_rpy_heap': 'referents._dump_rpy_heap',
                 'get_typeids_z': 'referents.get_typeids_z',
+                'get_typeids_list': 'referents.get_typeids_list',
                 'GcRef': 'referents.W_GcRef',
                 })
         MixedModule.__init__(self, space, w_name)
diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -16,7 +16,8 @@
     [0][0][0][-1] inserted after all GC roots, before all non-roots.
 
     If the argument is a filename and the 'zlib' module is available,
-    we also write a 'typeids.txt' in the same directory, if none exists.
+    we also write 'typeids.txt' and 'typeids.lst' in the same directory,
+    if they don't already exist.
     """
     if isinstance(file, str):
         f = open(file, 'wb')
@@ -30,7 +31,13 @@
             filename2 = os.path.join(os.path.dirname(file), 'typeids.txt')
             if not os.path.exists(filename2):
                 data = zlib.decompress(gc.get_typeids_z())
-                f = open(filename2, 'wb')
+                f = open(filename2, 'w')
+                f.write(data)
+                f.close()
+            filename2 = os.path.join(os.path.dirname(file), 'typeids.lst')
+            if not os.path.exists(filename2):
+                data = ''.join(['%d\n' % n for n in gc.get_typeids_list()])
+                f = open(filename2, 'w')
                 f.write(data)
                 f.close()
     else:
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -228,3 +228,8 @@
     a = rgc.get_typeids_z()
     s = ''.join([a[i] for i in range(len(a))])
     return space.wrap(s)
+
+def get_typeids_list(space):
+    l = rgc.get_typeids_list()
+    list_w = [space.wrap(l[i]) for i in range(len(l))]
+    return space.newlist(list_w)
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -392,7 +392,7 @@
         alignment = space.int_w(space.getitem(w_data, space.wrap(6)))
 
         if (w_names == space.w_None) != (w_fields == space.w_None):
-            raise oefmt(space.w_ValueError, "inconsistent fields and names")
+            raise oefmt(space.w_ValueError, "inconsistent fields and names in Numpy dtype unpickling")
 
         self.byteorder = endian
         self.shape = []
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -478,6 +478,15 @@
         for i in range(4):
             assert c[i] == max(a[i], b[i])
 
+
+    def test_abs_overflow(self):
+        from numpy import array, absolute, isinf
+        a = array(complex(1.5e308,1.5e308))
+        # Prints a RuntimeWarning, but does not raise
+        b = absolute(a)
+        assert isinf(b)
+        
+
     def test_basic(self):
         import sys
         from numpy import (dtype, add, array, dtype,
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -665,6 +665,7 @@
 
         assert numpy.int64(9223372036854775807) == 9223372036854775807
         assert numpy.int64(9223372036854775807) == 9223372036854775807
+        assert numpy.int64(-9223372036854775807) == -9223372036854775807
         raises(OverflowError, numpy.int64, 9223372036854775808)
         raises(OverflowError, numpy.int64, 9223372036854775808L)
 
@@ -1233,7 +1234,8 @@
 
         d = np.dtype(('<f8', 2))
         exc = raises(ValueError, "d.__setstate__((3, '|', None, ('f0', 'f1'), None, 16, 1, 0))")
-        assert exc.value[0] == 'inconsistent fields and names'
+        inconsistent = 'inconsistent fields and names in Numpy dtype unpickling'
+        assert exc.value[0] == inconsistent
         assert d.fields is None
         assert d.shape == (2,)
         assert d.subdtype is not None
@@ -1241,7 +1243,7 @@
 
         d = np.dtype(('<f8', 2))
         exc = raises(ValueError, "d.__setstate__((3, '|', None, None, {'f0': (np.dtype('float64'), 0), 'f1': (np.dtype('float64'), 8)}, 16, 1, 0))")
-        assert exc.value[0] == 'inconsistent fields and names'
+        assert exc.value[0] == inconsistent
         assert d.fields is None
         assert d.shape == (2,)
         assert d.subdtype is not None
@@ -1282,7 +1284,11 @@
         from cPickle import loads, dumps
 
         d = dtype([("x", "int32"), ("y", "int32"), ("z", "int32"), ("value", float)])
-        assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '|', None, ('x', 'y', 'z', 'value'), {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0), 'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12)}, 20, 1, 0))
+        assert d.__reduce__() == (dtype, ('V20', 0, 1), (3, '|', None, 
+                     ('x', 'y', 'z', 'value'),
+                     {'y': (dtype('int32'), 4), 'x': (dtype('int32'), 0),
+                      'z': (dtype('int32'), 8), 'value': (dtype('float64'), 12),
+                      }, 20, 1, 0))
 
         new_d = loads(dumps(d))
 
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1195,7 +1195,11 @@
 
     @complex_to_real_unary_op
     def abs(self, v):
-        return rcomplex.c_abs(v[0], v[1])
+        try:
+            return rcomplex.c_abs(v[0], v[1])
+        except OverflowError:
+            # warning ...
+            return rfloat.INFINITY
 
     @raw_unary_op
     def isnan(self, v):
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -619,6 +619,9 @@
             space.int_w(w_obj)
         except OperationError, e:
             if e.match(space, space.w_OverflowError):
+                if space.is_true(space.le(w_obj, space.wrap(0))):
+                    return find_binop_result_dtype(space, int64_dtype,
+                                               current_guess)
                 return find_binop_result_dtype(space, uint64_dtype,
                                                current_guess)
             raise
diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -286,7 +286,7 @@
 
             t = thread.start_new_thread(pollster.poll, ())
             try:
-                time.sleep(0.1)
+                time.sleep(0.3)
                 for i in range(5): print '',  # to release GIL untranslated
                 # trigger ufds array reallocation
                 for fd in rfds:
@@ -297,7 +297,7 @@
             finally:
                 # and make the call to poll() from the thread return
                 os.write(w, b'spam')
-                time.sleep(0.1)
+                time.sleep(0.3)
                 for i in range(5): print '',  # to release GIL untranslated
         finally:
             os.close(r)
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -7,7 +7,7 @@
 # all but one will be blocked.  The other threads get a chance to run
 # from time to time, using the periodic action GILReleaseAction.
 
-from rpython.rlib import rthread, rgil
+from rpython.rlib import rthread, rgil, rwin32
 from pypy.module.thread.error import wrap_thread_error
 from pypy.interpreter.executioncontext import PeriodicAsyncAction
 from pypy.module.thread.threadlocals import OSThreadLocals
@@ -76,9 +76,14 @@
 
 def after_external_call():
     e = get_errno()
+    e2 = 0
+    if rwin32.WIN32:
+        e2 = rwin32.GetLastError()
     rgil.gil_acquire()
     rthread.gc_thread_run()
     after_thread_switch()
+    if rwin32.WIN32:
+        rwin32.SetLastError(e2)
     set_errno(e)
 after_external_call._gctransformer_hint_cannot_collect_ = True
 after_external_call._dont_reach_me_in_del_ = True
diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -5,7 +5,7 @@
 from rpython.rlib.objectmodel import compute_hash
 
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import oefmt
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
 
@@ -123,6 +123,17 @@
         return space.wrap("<%s for 0x%s, size %d>" %
                           (info, addrstring, self.buf.getlength()))
 
+    def descr_pypy_raw_address(self, space):
+        from rpython.rtyper.lltypesystem import lltype, rffi
+        try:
+            ptr = self.buf.get_raw_address()
+        except ValueError:
+            # report the error using the RPython-level internal repr of self.buf
+            msg = ("cannot find the underlying address of buffer that "
+                   "is internally %r" % (self.buf,))
+            raise OperationError(space.w_ValueError, space.wrap(msg))
+        return space.wrap(rffi.cast(lltype.Signed, ptr))
+
 W_Buffer.typedef = TypeDef(
     "buffer",
     __doc__ = """\
@@ -149,5 +160,6 @@
     __mul__ = interp2app(W_Buffer.descr_mul),
     __rmul__ = interp2app(W_Buffer.descr_mul),
     __repr__ = interp2app(W_Buffer.descr_repr),
+    _pypy_raw_address = interp2app(W_Buffer.descr_pypy_raw_address),
 )
 W_Buffer.typedef.acceptable_as_base_class = False
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -618,8 +618,8 @@
         try:
             i = self.find(w_value, 0, sys.maxint)
         except ValueError:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("list.remove(x): x not in list"))
+            raise oefmt(space.w_ValueError,
+                        "list.remove(): %R is not in list", w_value)
         if i < self.length():  # otherwise list was mutated
             self.pop(i)
 
@@ -633,8 +633,7 @@
         try:
             i = self.find(w_value, i, stop)
         except ValueError:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("list.index(x): x not in list"))
+            raise oefmt(space.w_ValueError, "%R is not in list", w_value)
         return space.wrap(i)
 
     @unwrap_spec(reverse=bool)
diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py
--- a/pypy/objspace/std/test/test_bufferobject.py
+++ b/pypy/objspace/std/test/test_bufferobject.py
@@ -197,3 +197,9 @@
         buf = buffer('hello world')
         raises(TypeError, "buf[MyInt(0)]")
         raises(TypeError, "buf[MyInt(0):MyInt(5)]")
+
+    def test_pypy_raw_address_base(self):
+        raises(ValueError, buffer("foobar")._pypy_raw_address)
+        raises(ValueError, buffer(u"foobar")._pypy_raw_address)
+        e = raises(ValueError, buffer(bytearray("foobar"))._pypy_raw_address)
+        assert 'BytearrayBuffer' in str(e.value)
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -15,14 +15,15 @@
         assert NotEmpty.__dict__.get("b") is None
         raises(TypeError, 'NotEmpty.__dict__[15] = "y"')
         raises(KeyError, 'del NotEmpty.__dict__[15]')
+
+        key, value = NotEmpty.__dict__.popitem()
+        assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
+
         assert NotEmpty.__dict__.setdefault("string", 1) == 1
         assert NotEmpty.__dict__.setdefault("string", 2) == 1
         assert NotEmpty.string == 1
         raises(TypeError, 'NotEmpty.__dict__.setdefault(15, 1)')
 
-        key, value = NotEmpty.__dict__.popitem()
-        assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
-
     def test_dictproxy_getitem(self):
         class NotEmpty(object):
             a = 1
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -976,7 +976,10 @@
 
         c = [0.0, 2.2, 4.4]
         assert c.index(0) == 0.0
-        raises(ValueError, c.index, 3)
+        e = raises(ValueError, c.index, 3)
+        import sys
+        if sys.version_info[:2] == (2, 7):     # CPython 2.7, PyPy
+            assert str(e.value) == '3 is not in list'
 
     def test_index_cpython_bug(self):
         if self.on_cpython:
@@ -1228,7 +1231,9 @@
         assert l == [0.0, 1.1, 3.3, 4.4]
         l = [0.0, 3.3, 5.5]
         raises(ValueError, c.remove, 2)
-        raises(ValueError, c.remove, 2.2)
+        e = raises(ValueError, c.remove, 2.2)
+        if not self.on_cpython:
+            assert str(e.value) == 'list.remove(): 2.2 is not in list'
 
     def test_reverse(self):
         c = list('hello world')
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -254,7 +254,10 @@
         return self.__class__(can_be_None=False, no_nul=self.no_nul)
 
     def nonnulify(self):
-        return self.__class__(can_be_None=self.can_be_None, no_nul=True)
+        if self.can_be_None:
+            return self.__class__(can_be_None=True, no_nul=True)
+        else:
+            return self.__class__(no_nul=True)
 
 
 class SomeString(SomeStringOrUnicode):
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4326,6 +4326,13 @@
         assert isinstance(s, annmodel.SomeString)
         assert not s.can_be_none()
 
+    def test_nonnulify(self):
+        s = annmodel.SomeString(can_be_None=True).nonnulify()
+        assert s.can_be_None is True
+        assert s.no_nul is True
+        s = annmodel.SomeChar().nonnulify()
+        assert s.no_nul is True
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -901,7 +901,7 @@
             descr = tok.faildescr
             assert isinstance(descr, AbstractFailDescr)
             failure_recovery_pos = block_start + tok.pos_recovery_stub
-            descr._arm_failure_recovery_block = failure_recovery_pos
+            descr.adr_jump_offset = failure_recovery_pos
             relative_offset = tok.pos_recovery_stub - tok.offset
             guard_pos = block_start + tok.offset
             if not tok.is_guard_not_invalidated:
@@ -968,11 +968,11 @@
 
     def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
         b = InstrBuilder(self.cpu.cpuinfo.arch_version)
-        patch_addr = faildescr._arm_failure_recovery_block
+        patch_addr = faildescr.adr_jump_offset
         assert patch_addr != 0
         b.B(bridge_addr)
         b.copy_to_raw_memory(patch_addr)
-        faildescr._arm_failure_recovery_block = 0
+        faildescr.adr_jump_offset = 0
 
     # regalloc support
     def load(self, loc, value):
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -192,8 +192,6 @@
             positions[i] = rffi.cast(rffi.USHORT, position)
         # write down the positions of locs
         guardtok.faildescr.rd_locs = positions
-        # we want the descr to keep alive
-        guardtok.faildescr.rd_loop_token = self.current_clt
         return fail_descr, target
 
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -572,13 +572,13 @@
 
     def patch_pending_failure_recoveries(self, rawstart):
         # after we wrote the assembler to raw memory, set up
-        # tok.faildescr._x86_adr_jump_offset to contain the raw address of
+        # tok.faildescr.adr_jump_offset to contain the raw address of
         # the 4-byte target field in the JMP/Jcond instruction, and patch
         # the field in question to point (initially) to the recovery stub
         clt = self.current_clt
         for tok in self.pending_guard_tokens:
             addr = rawstart + tok.pos_jump_offset
-            tok.faildescr._x86_adr_jump_offset = addr
+            tok.faildescr.adr_jump_offset = addr
             relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4)
             assert rx86.fits_in_32bits(relative_target)
             #
@@ -685,7 +685,7 @@
                                    self.cpu.gc_ll_descr.gcrootmap)
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
-        adr_jump_offset = faildescr._x86_adr_jump_offset
+        adr_jump_offset = faildescr.adr_jump_offset
         assert adr_jump_offset != 0
         offset = adr_new_target - (adr_jump_offset + 4)
         # If the new target fits within a rel32 of the jump, just patch
@@ -705,7 +705,7 @@
             p = rffi.cast(rffi.INTP, adr_jump_offset)
             adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0])
             mc.copy_to_raw_memory(adr_target)
-        faildescr._x86_adr_jump_offset = 0    # means "patched"
+        faildescr.adr_jump_offset = 0    # means "patched"
 
     def fixup_target_tokens(self, rawstart):
         for targettoken in self.target_tokens_currently_compiling:
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1479,15 +1479,15 @@
             assert kind == 'v'
         return lltype.nullptr(rclass.OBJECTPTR.TO)
 
-    def _prepare_resume_from_failure(self, opnum, dont_change_position,
-                                     deadframe):
+    def _prepare_resume_from_failure(self, opnum, deadframe):
         from rpython.jit.metainterp.resoperation import rop
         #
-        if opnum == rop.GUARD_TRUE:
+        if opnum == rop.GUARD_FUTURE_CONDITION:
+            pass
+        elif opnum == rop.GUARD_TRUE:
             # Produced directly by some goto_if_not_xxx() opcode that did not
             # jump, but which must now jump.  The pc is just after the opcode.
-            if not dont_change_position:
-                self.position = self.jitcode.follow_jump(self.position)
+            self.position = self.jitcode.follow_jump(self.position)
         #
         elif opnum == rop.GUARD_FALSE:
             # Produced directly by some goto_if_not_xxx() opcode that jumped,
@@ -1517,8 +1517,7 @@
         elif opnum == rop.GUARD_NO_OVERFLOW:
             # Produced by int_xxx_ovf().  The pc is just after the opcode.
             # We get here because it did not used to overflow, but now it does.
-            if not dont_change_position:
-                return get_llexception(self.cpu, OverflowError())
+            return get_llexception(self.cpu, OverflowError())
         #
         elif opnum == rop.GUARD_OVERFLOW:
             # Produced by int_xxx_ovf().  The pc is just after the opcode.
@@ -1649,13 +1648,9 @@
         resumedescr,
         deadframe,
         all_virtuals)
-    if isinstance(resumedescr, ResumeAtPositionDescr):
-        dont_change_position = True
-    else:
-        dont_change_position = False
 
     current_exc = blackholeinterp._prepare_resume_from_failure(
-        resumedescr.guard_opnum, dont_change_position, deadframe)
+        resumedescr.guard_opnum, deadframe)
 
     _run_forever(blackholeinterp, current_exc)
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -61,14 +61,18 @@
     if metainterp_sd.warmrunnerdesc is not None:    # for tests
         assert original_jitcell_token.generation > 0     # has been registered with memmgr
     wref = weakref.ref(original_jitcell_token)
+    clt = original_jitcell_token.compiled_loop_token
+    clt.loop_token_wref = wref
     for op in loop.operations:
         descr = op.getdescr()
+        # not sure what descr.index is about
         if isinstance(descr, ResumeDescr):
-            descr.wref_original_loop_token = wref   # stick it there
-            n = descr.index
-            if n >= 0:       # we also record the resumedescr number
-                original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
-        elif isinstance(descr, JitCellToken):
+            descr.rd_loop_token = clt   # stick it there
+            #n = descr.index
+            #if n >= 0:       # we also record the resumedescr number
+            #    original_jitcell_token.compiled_loop_token.record_faildescr_index(n)
+        #    pass
+        if isinstance(descr, JitCellToken):
             # for a CALL_ASSEMBLER: record it as a potential jump.
             if descr is not original_jitcell_token:
                 original_jitcell_token.record_jump_to(descr)
@@ -135,15 +139,14 @@
     part = create_empty_loop(metainterp)
     part.inputargs = inputargs[:]
     h_ops = history.operations
-    memo = Memo()
-    part.operations = [ResOperation(rop.LABEL, inputargs, descr=TargetToken(jitcell_token))] + \
-                      [h_ops[i].clone(memo) for i in range(start, len(h_ops))]
-    jumpargs = [memo.get(box, box) for box in jumpargs]
-    part.operations.append(ResOperation(rop.LABEL, jumpargs, descr=jitcell_token))
+    label = ResOperation(rop.LABEL, inputargs, None,
+                         descr=TargetToken(jitcell_token))
+    end_label = ResOperation(rop.LABEL, jumpargs, None, descr=jitcell_token)
+    part.operations = [label] + h_ops[start:] + [end_label]
 
     try:
-        start_state = optimize_trace(metainterp_sd, part, enable_opts,
-                                     export_state=True)
+        start_state = optimize_trace(metainterp_sd, jitdriver_sd, part,
+                                     enable_opts, export_state=True)
     except InvalidLoop:
         return None
     target_token = part.operations[0].getdescr()
@@ -170,7 +173,7 @@
         jumpargs = part.operations[-1].getarglist()
 
         try:
-            optimize_trace(metainterp_sd, part, enable_opts,
+            optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
                            start_state=start_state, export_state=False)
         except InvalidLoop:
             return None
@@ -218,13 +221,14 @@
     h_ops = history.operations
 
     part.operations = [partial_trace.operations[-1]] + \
-                      [h_ops[i].clone() for i in range(start, len(h_ops))] + \
+                      h_ops[start:] + \
                       [ResOperation(rop.JUMP, jumpargs, None, descr=loop_jitcell_token)]
     label = part.operations[0]
     orignial_label = label.clone()
     assert label.getopnum() == rop.LABEL
     try:
-        optimize_trace(metainterp_sd, part, jitdriver_sd.warmstate.enable_opts,
+        optimize_trace(metainterp_sd, jitdriver_sd, part,
+                       jitdriver_sd.warmstate.enable_opts,
                        start_state=start_state, export_state=False)
     except InvalidLoop:
         # Fall back on jumping to preamble
@@ -234,7 +238,7 @@
                           [ResOperation(rop.JUMP, inputargs[:],
                                         None, descr=loop_jitcell_token)]
         try:
-            optimize_trace(metainterp_sd, part,
+            optimize_trace(metainterp_sd, jitdriver_sd, part,
                            jitdriver_sd.warmstate.enable_opts,
                            inline_short_preamble=False, start_state=start_state,
                            export_state=False)
@@ -494,22 +498,31 @@
     return d
 
 class ResumeDescr(AbstractFailDescr):
-    pass
+    _attrs_ = ()
 
 class ResumeGuardDescr(ResumeDescr):
-    # this class also gets the following attributes stored by resume.py code
-    # XXX move all of unused stuff to guard_op, now that we can have
-    #     a separate class, so it does not survive that long
-    rd_snapshot = None
-    rd_frame_info_list = None
+    _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
+               'rd_frame_info_list', 'rd_pendingfields', 'status')
+    
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
     rd_consts = None
     rd_virtuals = None
+    rd_frame_info_list = None
     rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
 
     status = r_uint(0)
 
+    def copy_all_attributes_from(self, other):
+        assert isinstance(other, ResumeGuardDescr)
+        self.rd_count = other.rd_count
+        self.rd_consts = other.rd_consts
+        self.rd_frame_info_list = other.rd_frame_info_list
+        self.rd_pendingfields = other.rd_pendingfields
+        self.rd_virtuals = other.rd_virtuals
+        self.rd_numb = other.rd_numb
+        # we don't copy status
+
     ST_BUSY_FLAG    = 0x01     # if set, busy tracing from the guard
     ST_TYPE_MASK    = 0x06     # mask for the type (TY_xxx)
     ST_SHIFT        = 3        # in "status >> ST_SHIFT" is stored:
@@ -524,31 +537,12 @@
     def store_final_boxes(self, guard_op, boxes, metainterp_sd):
         guard_op.setfailargs(boxes)
         self.rd_count = len(boxes)
-        self.guard_opnum = guard_op.getopnum()
         #
         if metainterp_sd.warmrunnerdesc is not None:   # for tests
             jitcounter = metainterp_sd.warmrunnerdesc.jitcounter
             hash = jitcounter.fetch_next_hash()
             self.status = hash & self.ST_SHIFT_MASK
 
-    def make_a_counter_per_value(self, guard_value_op):
-        assert guard_value_op.getopnum() == rop.GUARD_VALUE
-        box = guard_value_op.getarg(0)
-        try:
-            i = guard_value_op.getfailargs().index(box)
-        except ValueError:
-            return     # xxx probably very rare
-        else:
-            if box.type == history.INT:
-                ty = self.TY_INT
-            elif box.type == history.REF:
-                ty = self.TY_REF
-            elif box.type == history.FLOAT:
-                ty = self.TY_FLOAT
-            else:
-                assert 0, box.type
-            self.status = ty | (r_uint(i) << self.ST_SHIFT)
-
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         if self.must_compile(deadframe, metainterp_sd, jitdriver_sd):
             self.start_compiling()
@@ -647,32 +641,62 @@
                                self, inputargs, new_loop.operations,
                                new_loop.original_jitcell_token)
 
-    def copy_all_attributes_into(self, res, memo):
-        # XXX a bit ugly to have to list them all here
-        res.rd_snapshot = self.rd_snapshot.copy(memo)
-        res.rd_frame_info_list = self.rd_frame_info_list
-        res.rd_numb = self.rd_numb
-        res.rd_consts = self.rd_consts
-        res.rd_virtuals = self.rd_virtuals
-        res.rd_pendingfields = self.rd_pendingfields
-        res.rd_count = self.rd_count
+    def make_a_counter_per_value(self, guard_value_op):
+        assert guard_value_op.getopnum() == rop.GUARD_VALUE
+        box = guard_value_op.getarg(0)
+        try:
+            i = guard_value_op.getfailargs().index(box)
+        except ValueError:
+            return     # xxx probably very rare
+        else:
+            if box.type == history.INT:
+                ty = self.TY_INT
+            elif box.type == history.REF:
+                ty = self.TY_REF
+            elif box.type == history.FLOAT:
+                ty = self.TY_FLOAT
+            else:
+                assert 0, box.type
+            self.status = ty | (r_uint(i) << self.ST_SHIFT)
 
-    def _clone_if_mutable(self, memo):
-        res = ResumeGuardDescr()
-        self.copy_all_attributes_into(res, memo)
-        return res
+class ResumeGuardNonnullDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NONNULL
+
+class ResumeGuardIsnullDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_ISNULL
+
+class ResumeGuardClassDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_CLASS
+
+class ResumeGuardTrueDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_TRUE
+
+class ResumeGuardFalseDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_FALSE
+
+class ResumeGuardNonnullClassDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NONNULL_CLASS
+
+class ResumeGuardExceptionDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_EXCEPTION
+
+class ResumeGuardNoExceptionDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NO_EXCEPTION
+
+class ResumeGuardOverflowDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_OVERFLOW
+
+class ResumeGuardNoOverflowDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NO_OVERFLOW
+
+class ResumeGuardValueDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_VALUE
 
 class ResumeGuardNotInvalidated(ResumeGuardDescr):
-    def _clone_if_mutable(self, memo):
-        res = ResumeGuardNotInvalidated()
-        self.copy_all_attributes_into(res, memo)
-        return res
+    guard_opnum = rop.GUARD_NOT_INVALIDATED
 
 class ResumeAtPositionDescr(ResumeGuardDescr):
-    def _clone_if_mutable(self, memo):
-        res = ResumeAtPositionDescr()
-        self.copy_all_attributes_into(res, memo)
-        return res
+    guard_opnum = rop.GUARD_FUTURE_CONDITION
 
 class AllVirtuals:
     llopaque = True
@@ -693,8 +717,10 @@
 
 
 class ResumeGuardForcedDescr(ResumeGuardDescr):
+    guard_opnum = rop.GUARD_NOT_FORCED
 
-    def __init__(self, metainterp_sd, jitdriver_sd):
+    def _init(self, metainterp_sd, jitdriver_sd):
+        # to please the annotator
         self.metainterp_sd = metainterp_sd
         self.jitdriver_sd = jitdriver_sd
 
@@ -755,12 +781,39 @@
         hidden_all_virtuals = obj.hide(metainterp_sd.cpu)
         metainterp_sd.cpu.set_savedata_ref(deadframe, hidden_all_virtuals)
 
-    def _clone_if_mutable(self, memo):
-        res = ResumeGuardForcedDescr(self.metainterp_sd,
-                                     self.jitdriver_sd)
-        self.copy_all_attributes_into(res, memo)
-        return res
-
+def invent_fail_descr_for_op(opnum, optimizer):
+    if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
+        resumedescr = ResumeGuardForcedDescr()
+        resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd)
+    elif opnum == rop.GUARD_NOT_INVALIDATED:
+        resumedescr = ResumeGuardNotInvalidated()
+    elif opnum == rop.GUARD_FUTURE_CONDITION:
+        resumedescr = ResumeAtPositionDescr()
+    elif opnum == rop.GUARD_VALUE:
+        resumedescr = ResumeGuardValueDescr()
+    elif opnum == rop.GUARD_NONNULL:
+        resumedescr = ResumeGuardNonnullDescr()
+    elif opnum == rop.GUARD_ISNULL:
+        resumedescr = ResumeGuardIsnullDescr()
+    elif opnum == rop.GUARD_NONNULL_CLASS:
+        resumedescr = ResumeGuardNonnullClassDescr()
+    elif opnum == rop.GUARD_CLASS:
+        resumedescr = ResumeGuardClassDescr()
+    elif opnum == rop.GUARD_TRUE:
+        resumedescr = ResumeGuardTrueDescr()
+    elif opnum == rop.GUARD_FALSE:
+        resumedescr = ResumeGuardFalseDescr()
+    elif opnum == rop.GUARD_EXCEPTION:
+        resumedescr = ResumeGuardExceptionDescr()
+    elif opnum == rop.GUARD_NO_EXCEPTION:
+        resumedescr = ResumeGuardNoExceptionDescr()
+    elif opnum == rop.GUARD_OVERFLOW:
+        resumedescr = ResumeGuardOverflowDescr()
+    elif opnum == rop.GUARD_NO_OVERFLOW:
+        resumedescr = ResumeGuardNoOverflowDescr()
+    else:
+        assert False
+    return resumedescr
 
 class ResumeFromInterpDescr(ResumeDescr):
     def __init__(self, original_greenkey):
@@ -796,19 +849,18 @@
     # it does not work -- i.e. none of the existing old_loop_tokens match.
     new_trace = create_empty_loop(metainterp)
     new_trace.inputargs = metainterp.history.inputargs[:]
-    # clone ops, as optimize_bridge can mutate the ops
 
-    memo = Memo()
-    new_trace.operations = [op.clone(memo) for op in
-                            metainterp.history.operations]
+    new_trace.operations = metainterp.history.operations[:]
     metainterp_sd = metainterp.staticdata
-    state = metainterp.jitdriver_sd.warmstate
+    jitdriver_sd = metainterp.jitdriver_sd
+    state = jitdriver_sd.warmstate
     if isinstance(resumekey, ResumeAtPositionDescr):
         inline_short_preamble = False
     else:
         inline_short_preamble = True
     try:
-        state = optimize_trace(metainterp_sd, new_trace, state.enable_opts,
+        state = optimize_trace(metainterp_sd, jitdriver_sd, new_trace,
+                               state.enable_opts,
                                inline_short_preamble, export_state=True)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -124,14 +124,6 @@
     def repr_of_descr(self):
         return '%r' % (self,)
 
-    def _clone_if_mutable(self, memo):
-        return self
-    def clone_if_mutable(self, memo):
-        clone = self._clone_if_mutable(memo)
-        if not we_are_translated():
-            assert clone.__class__ is self.__class__
-        return clone
-
     def hide(self, cpu):
         descr_ptr = cpu.ts.cast_instance_to_base_ref(self)
         return cpu.ts.cast_to_ref(descr_ptr)
@@ -149,6 +141,8 @@
     index = -1
     final_descr = False
 
+    _attrs_ = ('adr_jump_offset', 'rd_locs', 'rd_loop_token')
+
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         raise NotImplementedError
     def compile_and_attach(self, metainterp, new_loop):
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -47,7 +47,7 @@
 
     return optimizations, unroll
 
-def optimize_trace(metainterp_sd, loop, enable_opts,
+def optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts,
                    inline_short_preamble=True, start_state=None,
                    export_state=True):
     """Optimize loop.operations to remove internal overheadish operations.
@@ -59,11 +59,13 @@
                                                           loop.operations)
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
-            return optimize_unroll(metainterp_sd, loop, optimizations,
+            return optimize_unroll(metainterp_sd, jitdriver_sd, loop,
+                                   optimizations,
                                    inline_short_preamble, start_state,
                                    export_state)
         else:
-            optimizer = Optimizer(metainterp_sd, loop, optimizations)
+            optimizer = Optimizer(metainterp_sd, jitdriver_sd, loop,
+                                  optimizations)
             optimizer.propagate_all_forward()
     finally:
         debug_stop("jit-optimize")
diff --git a/rpython/jit/metainterp/optimizeopt/generalize.py b/rpython/jit/metainterp/optimizeopt/generalize.py
--- a/rpython/jit/metainterp/optimizeopt/generalize.py
+++ b/rpython/jit/metainterp/optimizeopt/generalize.py
@@ -1,4 +1,5 @@
-from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT
+from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT,\
+     IntOptValue
 
 
 class GeneralizationStrategy(object):
@@ -14,7 +15,8 @@
         for v in self.optimizer.values.values():
             if v.is_constant():
                 continue
-            if v.intbound.lower < MININT / 2:
-                v.intbound.lower = MININT
-            if v.intbound.upper > MAXINT / 2:
-                v.intbound.upper = MAXINT
+            if isinstance(v, IntOptValue):
+                if v.intbound.lower < MININT / 2:
+                    v.intbound.lower = MININT
+                if v.intbound.upper > MAXINT / 2:
+                    v.intbound.upper = MAXINT
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -4,7 +4,8 @@
 from rpython.jit.metainterp.optimizeopt.util import args_dict
 from rpython.jit.metainterp.history import Const
 from rpython.jit.metainterp.jitexc import JitException
-from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED
+from rpython.jit.metainterp.optimizeopt.optimizer import Optimization,\
+     MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers
@@ -63,6 +64,17 @@
             # cancelling its previous effects with no side effect.
             self._lazy_setfield = None
 
+    def value_updated(self, oldvalue, newvalue):
+        try:
+            fieldvalue = self._cached_fields[oldvalue]
+        except KeyError:
+            pass
+        else:
+            self._cached_fields[newvalue] = fieldvalue
+            op = self._cached_fields_getfield_op[oldvalue].clone()
+            op.setarg(0, newvalue.box)
+            self._cached_fields_getfield_op[newvalue] = op
+
     def possible_aliasing(self, optheap, structvalue):
         # If lazy_setfield is set and contains a setfield on a different
         # structvalue, then we are annoyed, because it may point to either
@@ -82,10 +94,12 @@
         else:
             return self._cached_fields.get(structvalue, None)
 
-    def remember_field_value(self, structvalue, fieldvalue, getfield_op=None):
+    def remember_field_value(self, structvalue, fieldvalue, op=None,
+                             optimizer=None):
         assert self._lazy_setfield is None
         self._cached_fields[structvalue] = fieldvalue
-        self._cached_fields_getfield_op[structvalue] = getfield_op
+        op = optimizer.get_op_replacement(op)
+        self._cached_fields_getfield_op[structvalue] = op
 
     def force_lazy_setfield(self, optheap, can_cache=True):
         op = self._lazy_setfield
@@ -109,7 +123,8 @@
             # field.
             structvalue = optheap.getvalue(op.getarg(0))
             fieldvalue = optheap.getvalue(op.getarglist()[-1])
-            self.remember_field_value(structvalue, fieldvalue, op)
+            self.remember_field_value(structvalue, fieldvalue, op,
+                                      optheap.optimizer)
         elif not can_cache:
             self.clear()
 
@@ -117,18 +132,6 @@
         self._cached_fields.clear()
         self._cached_fields_getfield_op.clear()
 
-    def turned_constant(self, newvalue, value):
-        if newvalue not in self._cached_fields and value in self._cached_fields:
-            self._cached_fields[newvalue] = self._cached_fields[value]
-            op = self._cached_fields_getfield_op[value].clone()
-            constbox = value.box
-            assert isinstance(constbox, Const)
-            op.setarg(0, constbox)
-            self._cached_fields_getfield_op[newvalue] = op
-        for structvalue in self._cached_fields.keys():
-            if self._cached_fields[structvalue] is value:
-                self._cached_fields[structvalue] = newvalue
-
     def produce_potential_short_preamble_ops(self, optimizer, shortboxes, descr):
         if self._lazy_setfield is not None:
             return
@@ -138,7 +141,7 @@
                 continue
             value = optimizer.getvalue(op.getarg(0))
             if value in optimizer.opaque_pointers:
-                if value.level < LEVEL_KNOWNCLASS:
+                if value.getlevel() < LEVEL_KNOWNCLASS:
                     continue
                 if op.getopnum() != rop.SETFIELD_GC and op.getopnum() != rop.GETFIELD_GC:
                     continue
@@ -191,6 +194,17 @@
         self._seen_guard_not_invalidated = False
         self.postponed_op = None
 
+    def setup(self):
+        self.optimizer.optheap = self
+
+    def value_updated(self, oldvalue, newvalue):
+        # XXXX very unhappy about that
+        for cf in self.cached_fields.itervalues():
+            cf.value_updated(oldvalue, newvalue)
+        for submap in self.cached_arrayitems.itervalues():
+            for cf in submap.itervalues():
+                cf.value_updated(oldvalue, newvalue)
+
     def force_at_end_of_preamble(self):
         self.cached_dict_reads.clear()
         self.corresponding_array_descrs.clear()
@@ -363,16 +377,6 @@
             # ^^^ we only need to force this field; the other fields
             # of virtualref_info and virtualizable_info are not gcptrs.
 
-    def turned_constant(self, value):
-        assert value.is_constant()
-        newvalue = self.getvalue(value.box)
-        if value is not newvalue:
-            for cf in self.cached_fields.itervalues():
-                cf.turned_constant(newvalue, value)
-            for submap in self.cached_arrayitems.itervalues():
-                for cf in submap.itervalues():
-                    cf.turned_constant(newvalue, value)
-
     def force_lazy_setfield(self, descr, can_cache=True):
         try:
             cf = self.cached_fields[descr]
@@ -386,7 +390,7 @@
         except KeyError:
             return
         for idx, cf in submap.iteritems():
-            if indexvalue is None or indexvalue.intbound.contains(idx):
+            if indexvalue is None or indexvalue.getintbound().contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
     def _assert_valid_cf(self, cf):
@@ -448,7 +452,7 @@
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op)
-        cf.remember_field_value(structvalue, fieldvalue, op)
+        cf.remember_field_value(structvalue, fieldvalue, op, self.optimizer)
     optimize_GETFIELD_GC_R = optimize_GETFIELD_GC_I
     optimize_GETFIELD_GC_F = optimize_GETFIELD_GC_I
 
@@ -497,7 +501,7 @@
         # the remember the result of reading the array item
         if cf is not None:
             fieldvalue = self.getvalue(op)
-            cf.remember_field_value(arrayvalue, fieldvalue, op)
+            cf.remember_field_value(arrayvalue, fieldvalue, op, self.optimizer)
     optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_GC_I
     optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_GC_I
 
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt.intutils import (IntBound, IntLowerBound,
     IntUpperBound)
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1,
-    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE)
+    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE, IntOptValue)
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, AbstractResOp
 from rpython.jit.backend.llsupport import symbolic
@@ -53,7 +53,9 @@
         # FIXME: This takes care of the instruction where box is the reuslt
         #        but the bounds produced by all instructions where box is
         #        an argument might also be tighten
-        b = v.intbound
+        b = v.getintbound()
+        if b is None:
+            return # pointer
         if b.has_lower and b.has_upper and b.lower == b.upper:
             v.make_constant(ConstInt(b.lower))
 
@@ -78,11 +80,13 @@
                 self.make_constant_int(op, 0)
             return
         self.emit_operation(op)
-        if v1.intbound.known_ge(IntBound(0, 0)) and \
-           v2.intbound.known_ge(IntBound(0, 0)):
-            r = self.getvalue(op)
-            mostsignificant = v1.intbound.upper | v2.intbound.upper
-            r.intbound.intersect(IntBound(0, next_pow2_m1(mostsignificant)))
+        bound1 = v1.getintbound()
+        bound2 = v2.getintbound()
+        if bound1.known_ge(IntBound(0, 0)) and \
+           bound2.known_ge(IntBound(0, 0)):
+            r = self.getvalue(op).getintbound()
+            mostsignificant = bound1.upper | bound2.upper
+            r.intersect(IntBound(0, next_pow2_m1(mostsignificant)))
 
     optimize_INT_OR = optimize_INT_OR_or_XOR
     optimize_INT_XOR = optimize_INT_OR_or_XOR
@@ -96,55 +100,55 @@
         if v2.is_constant():
             val = v2.box.getint()
             if val >= 0:
-                r.intbound.intersect(IntBound(0, val))
+                r.getintbound().intersect(IntBound(0, val))
         elif v1.is_constant():
             val = v1.box.getint()
             if val >= 0:
-                r.intbound.intersect(IntBound(0, val))
-        elif v1.intbound.known_ge(IntBound(0, 0)) and \
-          v2.intbound.known_ge(IntBound(0, 0)):
-            lesser = min(v1.intbound.upper, v2.intbound.upper)
-            r.intbound.intersect(IntBound(0, next_pow2_m1(lesser)))
+                r.getintbound().intersect(IntBound(0, val))
+        elif v1.getintbound().known_ge(IntBound(0, 0)) and \
+          v2.getintbound().known_ge(IntBound(0, 0)):
+            lesser = min(v1.getintbound().upper, v2.getintbound().upper)
+            r.getintbound().intersect(IntBound(0, next_pow2_m1(lesser)))
 
     def optimize_INT_SUB(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op)
-        b = v1.intbound.sub_bound(v2.intbound)
+        b = v1.getintbound().sub_bound(v2.getintbound())
         if b.bounded():
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op)
-        b = v1.intbound.add_bound(v2.intbound)
+        b = v1.getintbound().add_bound(v2.getintbound())
         if b.bounded():
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op)
-        b = v1.intbound.mul_bound(v2.intbound)
+        b = v1.getintbound().mul_bound(v2.getintbound())
         if b.bounded():
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op)
-        r.intbound.intersect(v1.intbound.div_bound(v2.intbound))
+        r.getintbound().intersect(v1.getintbound().div_bound(v2.getintbound()))
 
     def optimize_INT_MOD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        known_nonneg = (v1.intbound.known_ge(IntBound(0, 0)) and
-                        v2.intbound.known_ge(IntBound(0, 0)))
+        known_nonneg = (v1.getintbound().known_ge(IntBound(0, 0)) and
+                        v2.getintbound().known_ge(IntBound(0, 0)))
         if known_nonneg and v2.is_constant():
             val = v2.box.getint()
             if (val & (val-1)) == 0:
@@ -162,18 +166,18 @@
                     return     # give up
                 val = -val
             if known_nonneg:
-                r.intbound.make_ge(IntBound(0, 0))
+                r.getintbound().make_ge(IntBound(0, 0))
             else:
-                r.intbound.make_gt(IntBound(-val, -val))
-            r.intbound.make_lt(IntBound(val, val))
+                r.getintbound().make_gt(IntBound(-val, -val))
+            r.getintbound().make_lt(IntBound(val, val))
 
     def optimize_INT_LSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
         r = self.getvalue(op)
-        b = v1.intbound.lshift_bound(v2.intbound)
-        r.intbound.intersect(b)
+        b = v1.getintbound().lshift_bound(v2.getintbound())
+        r.getintbound().intersect(b)
         # intbound.lshift_bound checks for an overflow and if the
         # lshift can be proven not to overflow sets b.has_upper and
         # b.has_lower
@@ -184,14 +188,14 @@
     def optimize_INT_RSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        b = v1.intbound.rshift_bound(v2.intbound)
+        b = v1.getintbound().rshift_bound(v2.getintbound())
         if b.has_lower and b.has_upper and b.lower == b.upper:
             # constant result (likely 0, for rshifts that kill all bits)
             self.make_constant_int(op, b.lower)
         else:
             self.emit_operation(op)
             r = self.getvalue(op)
-            r.intbound.intersect(b)
+            r.getintbound().intersect(b)
 
     def optimize_GUARD_NO_OVERFLOW(self, op):
         lastop = self.last_emitted_operation
@@ -236,17 +240,16 @@
     def optimize_INT_ADD_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        resbound = v1.intbound.add_bound(v2.intbound)
+        resbound = v1.getintbound().add_bound(v2.getintbound())
         if resbound.bounded():
             # Transform into INT_ADD.  The following guard will be killed
             # by optimize_GUARD_NO_OVERFLOW; if we see instead an
             # optimize_GUARD_OVERFLOW, then InvalidLoop.
-            newop = self.replace_op_with(op, rop.INT_ADD)
-        else:
-            newop = op
-        self.emit_operation(newop) # emit the op
+            xxx
+            op = op.copy_and_change(rop.INT_ADD)
+        self.emit_operation(op) # emit the op
         r = self.getvalue(op)
-        r.intbound.intersect(resbound)
+        r.getintbound().intersect(resbound)
 
     def optimize_INT_SUB_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -254,31 +257,30 @@
         if v1 is v2:
             self.make_constant_int(op, 0)
             return
-        resbound = v1.intbound.sub_bound(v2.intbound)
+        resbound = v1.getintbound().sub_bound(v2.getintbound())
         if resbound.bounded():
-            newop = self.replace_op_with(op, rop.INT_SUB)
-        else:
-            newop = op
-        self.emit_operation(newop) # emit the op
+            xxx
+            op = op.copy_and_change(rop.INT_SUB)
+        self.emit_operation(op) # emit the op
         r = self.getvalue(op)
-        r.intbound.intersect(resbound)
+        r.getintbound().intersect(resbound)
 
     def optimize_INT_MUL_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        resbound = v1.intbound.mul_bound(v2.intbound)
+        resbound = v1.getintbound().mul_bound(v2.getintbound())
         if resbound.bounded():
             op = self.replace_op_with(op, rop.INT_MUL)
         self.emit_operation(op)
         r = self.getvalue(op)
-        r.intbound.intersect(resbound)
+        r.getintbound().intersect(resbound)
 
     def optimize_INT_LT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_lt(v2.intbound):
+        if v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op, 1)
-        elif v1.intbound.known_ge(v2.intbound) or v1 is v2:
+        elif v1.getintbound().known_ge(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
@@ -286,9 +288,9 @@
     def optimize_INT_GT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_gt(v2.intbound):
+        if v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op, 1)
-        elif v1.intbound.known_le(v2.intbound) or v1 is v2:
+        elif v1.getintbound().known_le(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
@@ -296,9 +298,9 @@
     def optimize_INT_LE(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_le(v2.intbound) or v1 is v2:
+        if v1.getintbound().known_le(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op, 1)
-        elif v1.intbound.known_gt(v2.intbound):
+        elif v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
@@ -306,9 +308,9 @@
     def optimize_INT_GE(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_ge(v2.intbound) or v1 is v2:
+        if v1.getintbound().known_ge(v2.getintbound()) or v1 is v2:
             self.make_constant_int(op, 1)
-        elif v1.intbound.known_lt(v2.intbound):
+        elif v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
@@ -316,9 +318,9 @@
     def optimize_INT_EQ(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_gt(v2.intbound):
+        if v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op, 0)
-        elif v1.intbound.known_lt(v2.intbound):
+        elif v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op, 0)
         elif v1 is v2:
             self.make_constant_int(op, 1)
@@ -328,9 +330,9 @@
     def optimize_INT_NE(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        if v1.intbound.known_gt(v2.intbound):
+        if v1.getintbound().known_gt(v2.getintbound()):
             self.make_constant_int(op, 1)
-        elif v1.intbound.known_lt(v2.intbound):
+        elif v1.getintbound().known_lt(v2.getintbound()):
             self.make_constant_int(op, 1)
         elif v1 is v2:
             self.make_constant_int(op, 0)
@@ -339,7 +341,7 @@
 
     def optimize_INT_FORCE_GE_ZERO(self, op):
         value = self.getvalue(op.getarg(0))
-        if value.intbound.known_ge(IntBound(0, 0)):
+        if value.getintbound().known_ge(IntBound(0, 0)):
             self.make_equal_to(op, value)
         else:
             self.emit_operation(op)
@@ -350,50 +352,53 @@
         start = -(1 << (numbits - 1))
         stop = 1 << (numbits - 1)
         bounds = IntBound(start, stop - 1)
-        if bounds.contains_bound(value.intbound):
+        if bounds.contains_bound(value.getintbound()):
             self.make_equal_to(op, value)
         else:
             self.emit_operation(op)
             vres = self.getvalue(op)
-            vres.intbound.intersect(bounds)
+            vres.getintbound().intersect(bounds)
 
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
         result = self.getvalue(op)
         array.make_len_gt(MODE_ARRAY, op.getdescr(), -1)
-        array.lenbound.bound.intersect(result.intbound)
-        result.intbound = array.lenbound.bound
+        array.getlenbound().bound.intersect(result.getintbound())
+        assert isinstance(result, IntOptValue)
+        result.intbound = array.getlenbound().bound
 
     def optimize_STRLEN(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
         result = self.getvalue(op)
         array.make_len_gt(MODE_STR, op.getdescr(), -1)
-        array.lenbound.bound.intersect(result.intbound)
-        result.intbound = array.lenbound.bound
+        array.getlenbound().bound.intersect(result.getintbound())
+        assert isinstance(result, IntOptValue)
+        result.intbound = array.getlenbound().bound
 
     def optimize_UNICODELEN(self, op):
         self.emit_operation(op)
         array = self.getvalue(op.getarg(0))
         result = self.getvalue(op)
         array.make_len_gt(MODE_UNICODE, op.getdescr(), -1)
-        array.lenbound.bound.intersect(result.intbound)
-        result.intbound = array.lenbound.bound
+        array.getlenbound().bound.intersect(result.getintbound())
+        assert isinstance(result, IntOptValue)        
+        result.intbound = array.getlenbound().bound
 
     def optimize_STRGETITEM(self, op):
         self.emit_operation(op)
         v1 = self.getvalue(op)
-        v1.intbound.make_ge(IntLowerBound(0))
-        v1.intbound.make_lt(IntUpperBound(256))
+        v1.getintbound().make_ge(IntLowerBound(0))
+        v1.getintbound().make_lt(IntUpperBound(256))
 
     def optimize_GETFIELD_RAW_I(self, op):
         self.emit_operation(op)
         descr = op.getdescr()
         if descr.is_integer_bounded():
             v1 = self.getvalue(op)
-            v1.intbound.make_ge(IntLowerBound(descr.get_integer_min()))
-            v1.intbound.make_le(IntUpperBound(descr.get_integer_max()))
+            v1.getintbound().make_ge(IntLowerBound(descr.get_integer_min()))
+            v1.getintbound().make_le(IntUpperBound(descr.get_integer_max()))
 
     optimize_GETFIELD_RAW_F = optimize_GETFIELD_RAW_I
     optimize_GETFIELD_GC_I = optimize_GETFIELD_RAW_I
@@ -408,9 +413,9 @@
         self.emit_operation(op)
         descr = op.getdescr()
         if descr and descr.is_item_integer_bounded():
-            v1 = self.getvalue(op)
-            v1.intbound.make_ge(IntLowerBound(descr.get_item_integer_min()))
-            v1.intbound.make_le(IntUpperBound(descr.get_item_integer_max()))
+            intbound = self.getvalue(op).getintbound()
+            intbound.make_ge(IntLowerBound(descr.get_item_integer_min()))
+            intbound.make_le(IntUpperBound(descr.get_item_integer_max()))
 
     optimize_GETARRAYITEM_RAW_F = optimize_GETARRAYITEM_RAW_I
     optimize_GETARRAYITEM_GC_I = optimize_GETARRAYITEM_RAW_I
@@ -420,22 +425,22 @@
     def optimize_UNICODEGETITEM(self, op):
         self.emit_operation(op)
         v1 = self.getvalue(op)
-        v1.intbound.make_ge(IntLowerBound(0))
+        v1.getintbound().make_ge(IntLowerBound(0))
 
     def make_int_lt(self, box1, box2):
         v1 = self.getvalue(box1)
         v2 = self.getvalue(box2)
-        if v1.intbound.make_lt(v2.intbound):
+        if v1.getintbound().make_lt(v2.getintbound()):
             self.propagate_bounds_backward(box1, v1)
-        if v2.intbound.make_gt(v1.intbound):
+        if v2.getintbound().make_gt(v1.getintbound()):
             self.propagate_bounds_backward(box2, v2)
 
     def make_int_le(self, box1, box2):
         v1 = self.getvalue(box1)
         v2 = self.getvalue(box2)
-        if v1.intbound.make_le(v2.intbound):
+        if v1.getintbound().make_le(v2.getintbound()):
             self.propagate_bounds_backward(box1, v1)
-        if v2.intbound.make_ge(v1.intbound):
+        if v2.getintbound().make_ge(v1.getintbound()):
             self.propagate_bounds_backward(box2, v2)
 
     def make_int_gt(self, box1, box2):
@@ -482,9 +487,9 @@
             if r.box.same_constant(CONST_1):
                 v1 = self.getvalue(op.getarg(0))
                 v2 = self.getvalue(op.getarg(1))
-                if v1.intbound.intersect(v2.intbound):
+                if v1.getintbound().intersect(v2.getintbound()):
                     self.propagate_bounds_backward(op.getarg(0), v1)
-                if v2.intbound.intersect(v1.intbound):
+                if v2.getintbound().intersect(v1.getintbound()):
                     self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_NE(self, op):
@@ -493,9 +498,9 @@
             if r.box.same_constant(CONST_0):
                 v1 = self.getvalue(op.getarg(0))
                 v2 = self.getvalue(op.getarg(1))
-                if v1.intbound.intersect(v2.intbound):
+                if v1.getintbound().intersect(v2.getintbound()):
                     self.propagate_bounds_backward(op.getarg(0), v1)
-                if v2.intbound.intersect(v1.intbound):
+                if v2.getintbound().intersect(v1.getintbound()):
                     self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_IS_TRUE(self, op):
@@ -503,8 +508,8 @@
         if r.is_constant():
             if r.box.same_constant(CONST_1):
                 v1 = self.getvalue(op.getarg(0))
-                if v1.intbound.known_ge(IntBound(0, 0)):
-                    v1.intbound.make_gt(IntBound(0, 0))
+                if v1.getintbound().known_ge(IntBound(0, 0)):
+                    v1.getintbound().make_gt(IntBound(0, 0))
                     self.propagate_bounds_backward(op.getarg(0), v1)
 
     def propagate_bounds_INT_IS_ZERO(self, op):
@@ -515,50 +520,50 @@
                 # Clever hack, we can't use self.make_constant_int yet because
                 # the args aren't in the values dictionary yet so it runs into
                 # an assert, this is a clever way of expressing the same thing.
-                v1.intbound.make_ge(IntBound(0, 0))
-                v1.intbound.make_lt(IntBound(1, 1))
+                v1.getintbound().make_ge(IntBound(0, 0))
+                v1.getintbound().make_lt(IntBound(1, 1))
                 self.propagate_bounds_backward(op.getarg(0), v1)
 
     def propagate_bounds_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op)
-        b = r.intbound.sub_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        r = self.getvalue(op.result)
+        b = r.getintbound().sub_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0), v1)
-        b = r.intbound.sub_bound(v1.intbound)
-        if v2.intbound.intersect(b):
+        b = r.getintbound().sub_bound(v1.getintbound())
+        if v2.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_SUB(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op)
-        b = r.intbound.add_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        r = self.getvalue(op.result)
+        b = r.getintbound().add_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0), v1)
-        b = r.intbound.sub_bound(v1.intbound).mul(-1)
-        if v2.intbound.intersect(b):
+        b = r.getintbound().sub_bound(v1.getintbound()).mul(-1)
+        if v2.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op)
-        b = r.intbound.div_bound(v2.intbound)
-        if v1.intbound.intersect(b):
+        r = self.getvalue(op.result)
+        b = r.getintbound().div_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(0), v1)
-        b = r.intbound.div_bound(v1.intbound)
-        if v2.intbound.intersect(b):
+        b = r.getintbound().div_bound(v1.getintbound())
+        if v2.getintbound().intersect(b):
             self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_LSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
         r = self.getvalue(op)
-        b = r.intbound.rshift_bound(v2.intbound)
-        if v1.intbound.intersect(b):
-            self.propagate_bounds_backward(op.getarg(0))
+        b = r.getintbound().rshift_bound(v2.getintbound())
+        if v1.getintbound().intersect(b):
+            self.propagate_bounds_backward(op.getarg(0), v1)
 
     propagate_bounds_INT_ADD_OVF = propagate_bounds_INT_ADD
     propagate_bounds_INT_SUB_OVF = propagate_bounds_INT_SUB
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -4,20 +4,26 @@
 from rpython.jit.metainterp.history import Const, ConstInt, REF
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded, \
                                                      ImmutableIntUnbounded, \
-                                                     IntLowerBound, MININT, MAXINT
+                                                     IntLowerBound, MININT,\
+                                                     MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
-     AbstractResOp, AbstractInputArg, DONT_CHANGE
+     AbstractResOp, AbstractInputArg, DONT_CHANGE, GuardResOp
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.tool.pairtype import extendabletype
 from rpython.rlib.debug import debug_print
 from rpython.rlib.objectmodel import specialize
 
+""" The tag field on OptValue has a following meaning:
 
-LEVEL_UNKNOWN    = '\x00'
-LEVEL_NONNULL    = '\x01'
-LEVEL_KNOWNCLASS = '\x02'     # might also mean KNOWNARRAYDESCR, for arrays
-LEVEL_CONSTANT   = '\x03'
+lower two bits are LEVEL
+next 16 bits is the position in the original list, 0 if unknown or a constant
+"""
+
+LEVEL_UNKNOWN    = 0
+LEVEL_NONNULL    = 1
+LEVEL_KNOWNCLASS = 2     # might also mean KNOWNARRAYDESCR, for arrays
+LEVEL_CONSTANT   = 3
 
 MODE_ARRAY   = '\x00'
 MODE_STR     = '\x01'
@@ -41,92 +47,49 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
-    last_guard = None
+    _attrs_ = ('box', '_tag')
 
-    level = LEVEL_UNKNOWN
-    known_class = None
-    intbound = ImmutableIntUnbounded()
-    lenbound = None
+    _tag = 0
 
     def __init__(self, box, level=None, known_class=None, intbound=None):
         self.box = box
         if level is not None:
-            self.level = level
-        self.known_class = known_class
-        if intbound:
-            self.intbound = intbound
-        else:
-            if box is not None and box.type == 'i':
-                self.intbound = IntBound(MININT, MAXINT)
-            else:
-                self.intbound = IntUnbounded()
+            self._tag = level
 
         if isinstance(box, Const):
             self.make_constant(box)
         # invariant: box is a Const if and only if level == LEVEL_CONSTANT
 
-    def make_len_gt(self, mode, descr, val):
-        if self.lenbound:
-            assert self.lenbound.mode == mode
-            assert self.lenbound.descr == descr
-            self.lenbound.bound.make_gt(IntBound(val, val))
+    def getlevel(self):
+        return self._tag & 0x3
+
+    def setlevel(self, level):
+        self._tag = (self._tag & (~0x3)) | level
+
+    def import_from(self, other, optimizer):
+        if self.getlevel() == LEVEL_CONSTANT:
+            assert other.getlevel() == LEVEL_CONSTANT
+            assert other.box.same_constant(self.box)
+            return
+        assert self.getlevel() <= LEVEL_NONNULL
+        if other.getlevel() == LEVEL_CONSTANT:
+            self.make_constant(other.get_key_box())
+        elif other.getlevel() == LEVEL_KNOWNCLASS:
+            self.make_constant_class(None, other.get_known_class())
         else:
-            self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))
+            if other.getlevel() == LEVEL_NONNULL:
+                self.ensure_nonnull()
 
     def make_guards(self, box):
-        guards = []
-        if self.level == LEVEL_CONSTANT:
+        if self.getlevel() == LEVEL_CONSTANT:
             op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
-            guards.append(op)
-        elif self.level == LEVEL_KNOWNCLASS:
-            op = ResOperation(rop.GUARD_NONNULL, [box], None)
-            guards.append(op)
-            op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
-            guards.append(op)
-        else:
-            if self.level == LEVEL_NONNULL:
-                op = ResOperation(rop.GUARD_NONNULL, [box], None)
-                guards.append(op)
-            self.intbound.make_guards(box, guards)
-            if self.lenbound:
-                lenbox = BoxInt()
-                if self.lenbound.mode == MODE_ARRAY:
-                    op = ResOperation(rop.ARRAYLEN_GC, [box], lenbox, self.lenbound.descr)
-                elif self.lenbound.mode == MODE_STR:
-                    op = ResOperation(rop.STRLEN, [box], lenbox, self.lenbound.descr)
-                elif self.lenbound.mode == MODE_UNICODE:
-                    op = ResOperation(rop.UNICODELEN, [box], lenbox, self.lenbound.descr)
-                else:
-                    debug_print("Unknown lenbound mode")
-                    assert False
-                guards.append(op)
-                self.lenbound.bound.make_guards(lenbox, guards)
-        return guards
+            return [op]
+        return []
 
-    def import_from(self, other, optimizer):
-        if self.level == LEVEL_CONSTANT:
-            assert other.level == LEVEL_CONSTANT
-            assert other.box.same_constant(self.box)
-            return
-        assert self.level <= LEVEL_NONNULL
-        if other.level == LEVEL_CONSTANT:
-            self.make_constant(other.get_key_box())
-            optimizer.turned_constant(self)
-        elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, None)
-        else:
-            if other.level == LEVEL_NONNULL:
-                self.ensure_nonnull()
-            self.intbound.intersect(other.intbound)
-            if other.lenbound:
-                if self.lenbound:
-                    assert other.lenbound.mode == self.lenbound.mode
-                    assert other.lenbound.descr == self.lenbound.descr
-                    self.lenbound.bound.intersect(other.lenbound.bound)
-                else:
-                    self.lenbound = other.lenbound.clone()
-
+    def copy_from(self, other_value):
+        assert isinstance(other_value, OptValue)
+        self.box = other_value.box
+        self._tag = other_value._tag
 
     def force_box(self, optforce):
         return self.box
@@ -154,7 +117,7 @@
         assert 0, "unreachable"
 
     def is_constant(self):
-        return self.level == LEVEL_CONSTANT
+        return self.getlevel() == LEVEL_CONSTANT
 
     def is_null(self):
         if self.is_constant():
@@ -170,58 +133,20 @@
             return self.box.same_constant(other.box)
         return self is other
 
-    def make_constant(self, constbox):
-        """Replace 'self.box' with a Const box."""
-        assert isinstance(constbox, Const)
-        self.box = constbox
-        self.level = LEVEL_CONSTANT
-
-        if isinstance(constbox, ConstInt):
-            val = constbox.getint()
-            self.intbound = IntBound(val, val)
-        else:
-            self.intbound = IntUnbounded()
-
-    def get_constant_class(self, cpu):
-        level = self.level
-        if level == LEVEL_KNOWNCLASS:
-            return self.known_class
-        elif level == LEVEL_CONSTANT:
-            return cpu.ts.cls_of_box(self.box)
-        else:
-            return None
-
-    def make_constant_class(self, classbox, guardop):
-        assert self.level < LEVEL_KNOWNCLASS
-        self.known_class = classbox
-        self.level = LEVEL_KNOWNCLASS
-        self.last_guard = guardop
-
-    def make_nonnull(self, guardop):
-        assert self.level < LEVEL_NONNULL
-        self.level = LEVEL_NONNULL
-        self.last_guard = guardop
-
     def is_nonnull(self):
-        level = self.level
+        level = self.getlevel()
         if level == LEVEL_NONNULL or level == LEVEL_KNOWNCLASS:
             return True
         elif level == LEVEL_CONSTANT:
             box = self.box
             assert isinstance(box, Const)
             return box.nonnull()
-        elif self.intbound:
-            if self.intbound.known_gt(IntBound(0, 0)) or \
-               self.intbound.known_lt(IntBound(0, 0)):
-                return True
-            else:
-                return False
         else:
             return False
 
     def ensure_nonnull(self):
-        if self.level < LEVEL_NONNULL:
-            self.level = LEVEL_NONNULL
+        if self.getlevel() < LEVEL_NONNULL:
+            self.setlevel(LEVEL_NONNULL)
 
     def is_virtual(self):
         # Don't check this with 'isinstance(_, VirtualValue)'!
@@ -262,8 +187,224 @@
     def get_missing_null_value(self):
         raise NotImplementedError    # only for VArrayValue
 
+    def make_constant(self, constbox):
+        """Replace 'self.box' with a Const box."""
+        assert isinstance(constbox, Const)
+        self.box = constbox
+        self.setlevel(LEVEL_CONSTANT)
 
-class ConstantValue(OptValue):
+    def get_last_guard(self, optimizer):
+        return None
+
+    def get_known_class(self):
+        return None
+
+    def getlenbound(self):
+        return None
+
+    def getintbound(self):
+        return None
+
+    def get_constant_class(self, cpu):
+        return None
+
+class PtrOptValue(OptValue):
+    _attrs_ = ('known_class', 'last_guard_pos', 'lenbound')
+
+    known_class = None
+    last_guard_pos = -1
+    lenbound = None
+
+    def __init__(self, box, level=None, known_class=None, intbound=None):
+        OptValue.__init__(self, box, level, None, intbound)
+        if not isinstance(box, Const):
+            self.known_class = known_class
+
+    def copy_from(self, other_value):
+        assert isinstance(other_value, PtrOptValue)
+        self.box = other_value.box
+        self.known_class = other_value.known_class
+        self._tag = other_value._tag
+        self.last_guard_pos = other_value.last_guard_pos
+        self.lenbound = other_value.lenbound
+
+    def make_len_gt(self, mode, descr, val):
+        if self.lenbound:
+            assert self.lenbound.mode == mode
+            assert self.lenbound.descr == descr
+            self.lenbound.bound.make_gt(IntBound(val, val))
+        else:
+            self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))

From noreply at buildbot.pypy.org  Wed Dec 31 16:34:07 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 31 Dec 2014 16:34:07 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix compiler tests,
 broken when I added the __qualname__ attribute.
Message-ID: <20141231153407.A62FB1C34AB@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75188:b768017bc82f
Date: 2014-12-31 14:29 +0100
http://bitbucket.org/pypy/pypy/changeset/b768017bc82f/

Log:	Fix compiler tests, broken when I added the __qualname__ attribute.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -259,6 +259,20 @@
                 return expr_value
         return None
 
+    def ensure_docstring_constant(self, body):
+        # If there's a docstring, store it as the first constant.
+        if body:
+            doc_expr = self.possible_docstring(body[0])
+        else:
+            doc_expr = None
+        if doc_expr is not None:
+            self.add_const(doc_expr.s)
+            self.scope.doc_removable = True
+            return True
+        else:
+            self.add_const(self.space.w_None)
+            return False
+
     def _get_code_flags(self):
         # Default for everything but module scopes.
         return consts.CO_NEWLOCALS
@@ -1291,18 +1305,8 @@
 
     def _compile(self, func):
         assert isinstance(func, ast.FunctionDef)
-        # If there's a docstring, store it as the first constant.
-        if func.body:
-            doc_expr = self.possible_docstring(func.body[0])
-        else:
-            doc_expr = None
-        if doc_expr is not None:
-            self.add_const(doc_expr.s)
-            self.scope.doc_removable = True
-            start = 1
-        else:
-            self.add_const(self.space.w_None)
-            start = 0
+        has_docstring = self.ensure_docstring_constant(func.body)
+        start = 1 if has_docstring else 0
         args = func.args
         assert isinstance(args, ast.arguments)
         if args.args:
@@ -1357,6 +1361,7 @@
 
     def _compile(self, cls):
         assert isinstance(cls, ast.ClassDef)
+        self.ensure_docstring_constant(cls.body)
         self.lineno = self.first_lineno
         self.argcount = 1
         # load the first argument (__locals__) ...
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -24,7 +24,7 @@
     function_ast = ast.body[0]
     symbols = symtable.SymtableBuilder(space, ast, info)
     generator = codegen.FunctionCodeGenerator(
-        space, 'function', function_ast, 1, symbols, info)
+        space, 'function', function_ast, 1, symbols, info, qualname='function')
     blocks = generator.first_block.post_order()
     generator._resolve_block_targets(blocks)
     return generator, blocks

From noreply at buildbot.pypy.org  Wed Dec 31 16:34:08 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 31 Dec 2014 16:34:08 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix many tests in interpreter/
Message-ID: <20141231153408.D04941C34AB@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75189:21aecd71f599
Date: 2014-12-31 16:24 +0100
http://bitbucket.org/pypy/pypy/changeset/21aecd71f599/

Log:	Fix many tests in interpreter/

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -827,9 +827,14 @@
     del os # make sure that os is not available globally, because this is what
            # happens in "real life" outside the tests
 
+    # when run as __main__, this module is often executed by a Python
+    # interpreter that have a different list of builtin modules.
+    # Make some tests happy by loading them before we clobber sys.path
+    import runpy
     if 'time' not in sys.builtin_module_names:
-        # make some tests happy by loading this before we clobber sys.path
         import time; del time
+    if 'operator' not in sys.builtin_module_names:
+        import operator; del operator
 
     # no one should change to which lists sys.argv and sys.path are bound
     old_argv = sys.argv
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -391,9 +391,6 @@
     def test_tmpnames(self):
         scp = self.mod_scope("with x: pass")
         assert scp.lookup("_[1]") == symtable.SCOPE_LOCAL
-        scp = self.mod_scope("with x as y: pass")
-        assert scp.lookup("_[1]") == symtable.SCOPE_LOCAL
-        assert scp.lookup("_[2]") == symtable.SCOPE_LOCAL
 
     def test_issue13343(self):
         scp = self.mod_scope("lambda *, k1=x, k2: None")
diff --git a/pypy/interpreter/test/test_mixedmodule.py b/pypy/interpreter/test/test_mixedmodule.py
--- a/pypy/interpreter/test/test_mixedmodule.py
+++ b/pypy/interpreter/test/test_mixedmodule.py
@@ -52,6 +52,13 @@
 
         m = Module(space, space.wrap("test_module"))
         m.install()
+        # Python3's importlib relies on sys.builtin_module_names, the
+        # call to m.install() above is not enough because the object
+        # space was already initialized.
+        space.setattr(space.sys, space.wrap('builtin_module_names'),
+                      space.add(space.sys.get('builtin_module_names'),
+                                space.newtuple([
+                                    space.wrap("test_module")])))
 
     def teardown_class(cls):
         from pypy.module.sys.state import get

From noreply at buildbot.pypy.org  Wed Dec 31 16:34:10 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 31 Dec 2014 16:34:10 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: audioop: Use cffi instead of ctypes to
 create mutable buffers
Message-ID: <20141231153410.0F4AC1C34AB@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r75190:f9375d4e25c3
Date: 2014-12-31 16:32 +0100
http://bitbucket.org/pypy/pypy/changeset/f9375d4e25c3/

Log:	audioop: Use cffi instead of ctypes to create mutable buffers also
	precompile the module.

diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -2,7 +2,7 @@
 import math
 import struct
 from fractions import gcd
-from ctypes import create_string_buffer
+from cffi import FFI
 
 
 _buffer = memoryview
@@ -327,13 +327,14 @@
     _check_params(len(cp), size)
     clip = _get_clipfn(size)
 
-    result = create_string_buffer(len(cp))
+    rv = ffi.new("unsigned char[]", len(cp))
+    result = ffi.buffer(rv)
 
     for i, sample in enumerate(_get_samples(cp, size)):
         sample = clip(int(sample * factor))
         _put_sample(result, size, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def tomono(cp, size, fac1, fac2):
@@ -342,7 +343,8 @@
 
     sample_count = _sample_count(cp, size)
 
-    result = create_string_buffer(len(cp) // 2)
+    rv = ffi.new("unsigned char[]", len(cp) // 2)
+    result = ffi.buffer(rv)
 
     for i in range(0, sample_count, 2):
         l_sample = getsample(cp, size, i)
@@ -353,7 +355,7 @@
 
         _put_sample(result, size, i // 2, sample)
 
-    return result.raw
+    return result[:]
 
 
 def tostereo(cp, size, fac1, fac2):
@@ -361,7 +363,8 @@
 
     sample_count = _sample_count(cp, size)
 
-    result = create_string_buffer(len(cp) * 2)
+    rv = ffi.new("unsigned char[]", len(cp) * 2)
+    result = ffi.buffer(rv)
     clip = _get_clipfn(size)
 
     for i in range(sample_count):
@@ -373,7 +376,7 @@
         _put_sample(result, size, i * 2, l_sample)
         _put_sample(result, size, i * 2 + 1, r_sample)
 
-    return result.raw
+    return result[:]
 
 
 def add(cp1, cp2, size):
@@ -384,7 +387,8 @@
 
     clip = _get_clipfn(size)
     sample_count = _sample_count(cp1, size)
-    result = create_string_buffer(len(cp1))
+    rv = ffi.new("unsigned char[]", len(cp1))
+    result = ffi.buffer(rv)
 
     for i in range(sample_count):
         sample1 = getsample(cp1, size, i)
@@ -394,30 +398,32 @@
 
         _put_sample(result, size, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def bias(cp, size, bias):
     _check_params(len(cp), size)
 
-    result = create_string_buffer(len(cp))
+    rv = ffi.new("unsigned char[]", len(cp))
+    result = ffi.buffer(rv)
 
     for i, sample in enumerate(_get_samples(cp, size)):
         sample = _overflow(sample + bias, size)
         _put_sample(result, size, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def reverse(cp, size):
     _check_params(len(cp), size)
     sample_count = _sample_count(cp, size)
 
-    result = create_string_buffer(len(cp))
+    rv = ffi.new("unsigned char[]", len(cp))
+    result = ffi.buffer(rv)
     for i, sample in enumerate(_get_samples(cp, size)):
         _put_sample(result, size, sample_count - i - 1, sample)
 
-    return result.raw
+    return result[:]
 
 
 def lin2lin(cp, size, size2):
@@ -428,7 +434,8 @@
         return cp
 
     new_len = (len(cp) // size) * size2
-    result = create_string_buffer(new_len)
+    rv = ffi.new("unsigned char[]", new_len)
+    result = ffi.buffer(rv)
 
     for i in range(_sample_count(cp, size)):
         sample = _get_sample(cp, size, i)
@@ -443,7 +450,7 @@
         sample = _overflow(sample, size2)
         _put_sample(result, size2, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def ratecv(cp, size, nchannels, inrate, outrate, state, weightA=1, weightB=0):
@@ -488,7 +495,8 @@
     ceiling = (q + 1) * outrate
     nbytes = ceiling * bytes_per_frame
 
-    result = create_string_buffer(nbytes)
+    rv = ffi.new("unsigned char[]", nbytes)
+    result = ffi.buffer(rv)
 
     samples = _get_samples(cp, size)
     out_i = 0
@@ -496,7 +504,7 @@
         while d < 0:
             if frame_count == 0:
                 samps = zip(prev_i, cur_i)
-                retval = result.raw
+                retval = result[:]
 
                 # slice off extra bytes
                 trim_index = (out_i * bytes_per_frame) - len(retval)
@@ -527,7 +535,6 @@
                 d -= inrate
 
 
-from cffi import FFI
 ffi = FFI()
 ffi.cdef("""
 typedef short PyInt16;
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -111,7 +111,7 @@
 '''
 
 def create_cffi_import_libraries(pypy_c, options):
-    modules = ['_sqlite3', '_lzma']
+    modules = ['_sqlite3', '_lzma', 'audioop']
     subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
     if not sys.platform == 'win32':
         modules += ['_curses', 'syslog', '_gdbm', '_sqlite3']

From noreply at buildbot.pypy.org  Wed Dec 31 16:40:15 2014
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 31 Dec 2014 16:40:15 +0100 (CET)
Subject: [pypy-commit] pypy default: (larstiq) Tell virtualenv to copy
 libpypy-c.dylib as well.
Message-ID: <20141231154015.6F4901C00BB@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r75191:3fcbbd6f6e29
Date: 2014-12-31 16:39 +0100
http://bitbucket.org/pypy/pypy/changeset/3fcbbd6f6e29/

Log:	(larstiq) Tell virtualenv to copy libpypy-c.dylib as well.

diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -1589,7 +1589,7 @@
                   'copyfile' in caller.f_globals):
         dest_dir = sys.pypy_resolvedirof(target_executable)
         src_dir = sys.pypy_resolvedirof(sys.executable)
-        for libname in ['libpypy-c.so']:
+        for libname in ['libpypy-c.so', 'libpypy-c.dylib']:
             dest_library = os.path.join(dest_dir, libname)
             src_library = os.path.join(src_dir, libname)
             if os.path.exists(src_library):

From noreply at buildbot.pypy.org  Wed Dec 31 18:48:11 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 31 Dec 2014 18:48:11 +0100 (CET)
Subject: [pypy-commit] cffi default: Recognize more literal constants,
 notably octals and chars
Message-ID: <20141231174811.0E5FF1C00BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1615:051a8e70baab
Date: 2014-12-31 18:48 +0100
http://bitbucket.org/cffi/cffi/changeset/051a8e70baab/

Log:	Recognize more literal constants, notably octals and chars

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -537,7 +537,18 @@
         # for now, limited to expressions that are an immediate number
         # or positive/negative number
         if isinstance(exprnode, pycparser.c_ast.Constant):
-            return int(exprnode.value, 0)
+            s = exprnode.value
+            if s.startswith('0'):
+                if s.startswith('0x') or s.startswith('0X'):
+                    return int(s, 16)
+                return int(s, 8)
+            elif '1' <= s[0] <= '9':
+                return int(s, 10)
+            elif s[0] == "'" and s[-1] == "'" and (
+                    len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
+                return ord(s[-2])
+            else:
+                raise api.CDefError("invalid constant %r" % (s,))
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '+'):
diff --git a/testing/backend_tests.py b/testing/backend_tests.py
--- a/testing/backend_tests.py
+++ b/testing/backend_tests.py
@@ -916,6 +916,16 @@
         assert int(invalid_value) == 2
         assert ffi.string(invalid_value) == "2"
 
+    def test_enum_char_hex_oct(self):
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef(r"enum foo{A='!', B='\'', C=0x10, D=010, E=- 0x10, F=-010};")
+        assert ffi.string(ffi.cast("enum foo", ord('!'))) == "A"
+        assert ffi.string(ffi.cast("enum foo", ord("'"))) == "B"
+        assert ffi.string(ffi.cast("enum foo", 16)) == "C"
+        assert ffi.string(ffi.cast("enum foo", 8)) == "D"
+        assert ffi.string(ffi.cast("enum foo", -16)) == "E"
+        assert ffi.string(ffi.cast("enum foo", -8)) == "F"
+
     def test_array_of_struct(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("struct foo { int a, b; };")

From noreply at buildbot.pypy.org  Wed Dec 31 19:10:38 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 31 Dec 2014 19:10:38 +0100 (CET)
Subject: [pypy-commit] pypy default: Change the default C++ compiler from
 'cc' to 'c++'. On CPython this setting
Message-ID: <20141231181038.9CADB1C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r75192:c6e45dfbda90
Date: 2014-12-31 19:10 +0100
http://bitbucket.org/pypy/pypy/changeset/c6e45dfbda90/

Log:	Change the default C++ compiler from 'cc' to 'c++'. On CPython this
	setting is ignored anyway, but on PyPy the value 'cc' creates this
	issue for example:

	 https://bitbucket.org/cffi/cffi/issue/165/

diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -58,7 +58,7 @@
     executables = {'preprocessor' : None,
                    'compiler'     : ["cc"],
                    'compiler_so'  : ["cc"],
-                   'compiler_cxx' : ["cc"],
+                   'compiler_cxx' : ["c++"],  # pypy: changed, 'cc' is bogus
                    'linker_so'    : ["cc", "-shared"],
                    'linker_exe'   : ["cc"],
                    'archiver'     : ["ar", "-cr"],

From noreply at buildbot.pypy.org  Wed Dec 31 19:34:26 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 31 Dec 2014 19:34:26 +0100 (CET)
Subject: [pypy-commit] cffi default: Document 'relative_to'.
Message-ID: <20141231183426.86F841C31CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1616:f7fa2106ddf3
Date: 2014-12-31 19:34 +0100
http://bitbucket.org/cffi/cffi/changeset/f7fa2106ddf3/

Log:	Document 'relative_to'.

diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -673,7 +673,9 @@
    whenever you change your sources.
 
 .. versionadded:: 0.9
-   You can give C++ source code in ``ffi.verify()``::
+   You can give C++ source code in ``ffi.verify()``:
+
+::
 
      ext = ffi.verify(r'''
          extern "C" {
@@ -685,6 +687,27 @@
    The optional ``flags`` argument has been added, see ``man dlopen`` (ignored
    on Windows).  It defaults to ``ffi.RTLD_NOW``.
 
+.. versionadded:: 0.9
+   The optional ``relative_to`` argument is useful if you need to list
+   local files passed to the C compiler:
+
+::
+
+     ext = ffi.verify(..., sources=['foo.c'], relative_to=__file__)
+
+The line above is roughly the same as::
+
+     ext = ffi.verify(..., sources=['/path/to/this/file/foo.c'])
+
+except that the default name of the produced library is built from the
+CRC checkum of the argument ``sources``, as well as most other arguments
+you give to ``ffi.verify()`` -- but not ``relative_to``.  So if you used
+the second line, it would stop finding the already-compiled library
+after your project is installed, because the ``'/path/to/this/file'``
+suddenly changed.  The first line does not have this problem.
+
+---------------------
+
 This function returns a "library" object that gets closed when it goes
 out of scope.  Make sure you keep the library object around as long as
 needed.