From pypy.commits at gmail.com  Wed May  2 13:47:32 2018
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 02 May 2018 10:47:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix translation
Message-ID: <5ae9f9b4.1c69fb81.279bc.323c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.6
Changeset: r94460:d3671e3e862c
Date: 2018-05-02 18:46 +0100
http://bitbucket.org/pypy/pypy/changeset/d3671e3e862c/

Log:	fix translation

diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -119,8 +119,8 @@
 
     if first_escape_error_char != '':
         space.warn(
-            space.newtext("invalid escape sequence '\\%c'"
-                             % first_escape_error_char),
+            space.newtext("invalid escape sequence '\\%s'"
+                          % first_escape_error_char),
             space.w_DeprecationWarning)
 
     return space.newbytes(v)

From pypy.commits at gmail.com  Thu May  3 04:27:35 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 May 2018 01:27:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: getting started with __set_name__ support
 (PEP 487)
Message-ID: <5aeac7f7.424a1c0a.b296f.3a47@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94461:97a56d2df52b
Date: 2018-05-02 22:23 +0200
http://bitbucket.org/pypy/pypy/changeset/97a56d2df52b/

Log:	getting started with __set_name__ support (PEP 487)

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1544,3 +1544,15 @@
 
     def test_type_construct_unicode_surrogate_issue(self):
         raises(ValueError, type, 'A\udcdcb', (), {})
+
+    def test_set_name(self):
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                self.owner = owner
+                self.name = name
+
+        class X:
+            a = Descriptor()
+        assert X.a.owner is X
+        assert X.a.name == "a"
+
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -800,6 +800,8 @@
     W_TypeObject.__init__(w_type, space, name, bases_w or [space.w_object],
                           dict_w, is_heaptype=True)
     w_type.ready()
+
+    _set_names(space, w_type)
     return w_type
 
 def _calculate_metaclass(space, w_metaclass, bases_w):
@@ -822,6 +824,13 @@
         raise oefmt(space.w_TypeError, "X is not a type object (%T)", w_type)
     return w_type
 
+def _set_names(space, w_type):
+    for key, w_value in w_type.dict_w.iteritems():
+        w_meth = space.lookup(w_value, '__set_name__')
+        if w_meth is not None:
+            # XXX what happens when the call raises, gets turned into a RuntimeError?
+            space.get_and_call_function(w_meth, w_value, w_type, space.newtext(key))
+
 
 def descr__init__(space, w_type, __args__):
     if __args__.keywords:

From pypy.commits at gmail.com  Thu May  3 04:27:37 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 03 May 2018 01:27:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: introduces the __init_subclass__ hook
 (PEP 487)
Message-ID: <5aeac7f9.1c69fb81.79179.6640@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94462:9e10c6bdc41e
Date: 2018-05-03 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/9e10c6bdc41e/

Log:	introduces the __init_subclass__ hook (PEP 487)

diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -122,6 +122,8 @@
 def descr___subclasshook__(space, __args__):
     return space.w_NotImplemented
 
+def descr___init_subclass__(space, w_cls):
+    return space.w_None
 
 def descr__init__(space, w_obj, __args__):
     if _excess_args(__args__):
@@ -284,12 +286,14 @@
     __doc__ = "The most base type",
     __new__ = interp2app(descr__new__),
     __subclasshook__ = interp2app(descr___subclasshook__, as_classmethod=True),
+    __init_subclass__ = interp2app(descr___init_subclass__, as_classmethod=True),
 
     # these are actually implemented in pypy.objspace.descroperation
     __getattribute__ = interp2app(Object.descr__getattribute__.im_func),
     __setattr__ = interp2app(Object.descr__setattr__.im_func),
     __delattr__ = interp2app(Object.descr__delattr__.im_func),
 
+
     __init__ = interp2app(descr__init__),
     __class__ = GetSetProperty(descr_get___class__, descr_set___class__),
     __repr__ = interp2app(descr__repr__),
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -359,6 +359,11 @@
         assert o.__ge__(o2) is NotImplemented
         assert o.__gt__(o2) is NotImplemented
 
+    def test_init_subclass(self):
+        object().__init_subclass__() # does not crash
+        object.__init_subclass__() # does not crash
+        raises(TypeError, object.__init_subclass__, 1)
+
 def test_isinstance_shortcut():
     from pypy.objspace.std import objspace
     space = objspace.StdObjSpace()
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1556,3 +1556,43 @@
         assert X.a.owner is X
         assert X.a.name == "a"
 
+    def test_type_init_accepts_kwargs(self):
+        type.__init__(type, "a", (object, ), {}, a=1)
+
+    def test_init_subclass_classmethod(self):
+        assert isinstance(object.__dict__['__init_subclass__'], classmethod)
+        class A(object):
+            subclasses = []
+
+            def __init_subclass__(cls):
+                cls.subclass.append(cls)
+        assert isinstance(A.__dict__['__init_subclass__'], classmethod)
+
+    def test_init_subclass(self):
+        class PluginBase(object):
+            subclasses = []
+
+            def __init_subclass__(cls):
+                cls.subclasses.append(cls)
+
+        class B(PluginBase):
+            pass
+
+        class C(PluginBase):
+            pass
+
+        assert PluginBase.subclasses == [B, C]
+
+
+        class X(object):
+            subclasses = []
+
+            def __init_subclass__(cls, **kwargs):
+                cls.kwargs = kwargs
+
+        exec("""if 1:
+        class Y(X, a=1, b=2):
+            pass
+
+        assert Y.kwargs == dict(a=1, b=2)
+        """)
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -749,20 +749,26 @@
         return space.newbool(not space.is_w(self, w_other))
 
 
-def descr__new__(space, w_typetype, w_name, w_bases=None, w_dict=None):
+def descr__new__(space, w_typetype, __args__):
     """This is used to create user-defined classes only."""
+    if len(__args__.arguments_w) not in (1, 3):
+        raise oefmt(space.w_TypeError,
+                    "type.__new__() takes 1 or 3 arguments")
+
+    w_name = __args__.arguments_w[0]
+
     w_typetype = _precheck_for_new(space, w_typetype)
 
     # special case for type(x)
     if (space.is_w(space.type(w_typetype), space.w_type) and
-        w_bases is None and w_dict is None):
+            len(__args__.arguments_w) == 1):
         return space.type(w_name)
-    return _create_new_type(space, w_typetype, w_name, w_bases, w_dict)
+    w_bases = __args__.arguments_w[1]
+    w_dict = __args__.arguments_w[2]
+    return _create_new_type(space, w_typetype, w_name, w_bases, w_dict, __args__)
 
 
 def _check_new_args(space, w_name, w_bases, w_dict):
-    if w_bases is None or w_dict is None:
-        raise oefmt(space.w_TypeError, "type() takes 1 or 3 arguments")
     if not space.isinstance_w(w_name, space.w_text):
         raise oefmt(space.w_TypeError,
                     "type() argument 1 must be string, not %T", w_name)
@@ -774,7 +780,7 @@
                     "type() argument 3 must be dict, not %T", w_dict)
 
 
-def _create_new_type(space, w_typetype, w_name, w_bases, w_dict):
+def _create_new_type(space, w_typetype, w_name, w_bases, w_dict, __args__):
     # this is in its own function because we want the special case 'type(x)'
     # above to be seen by the jit.
     _check_new_args(space, w_name, w_bases, w_dict)
@@ -802,6 +808,7 @@
     w_type.ready()
 
     _set_names(space, w_type)
+    _init_subclass(space, w_type, __args__)
     return w_type
 
 def _calculate_metaclass(space, w_metaclass, bases_w):
@@ -831,11 +838,15 @@
             # XXX what happens when the call raises, gets turned into a RuntimeError?
             space.get_and_call_function(w_meth, w_value, w_type, space.newtext(key))
 
+def _init_subclass(space, w_type, __args__):
+    # bit of a mess, but I didn't feel like implementing the super logic
+    w_super = space.getattr(space.builtin, space.newtext("super"))
+    w_func = space.getattr(space.call_function(w_super, w_type, w_type),
+                           space.newtext("__init_subclass__"))
+    args = __args__.replace_arguments([])
+    space.call_args(w_func, args)
 
 def descr__init__(space, w_type, __args__):
-    if __args__.keywords:
-        raise oefmt(space.w_TypeError,
-                    "type.__init__() takes no keyword arguments")
     if len(__args__.arguments_w) not in (1, 3):
         raise oefmt(space.w_TypeError,
                     "type.__init__() takes 1 or 3 arguments")
@@ -1311,6 +1322,7 @@
     w_self.mro_w = []      # temporarily
     w_self.hasmro = False
     compute_mro(w_self)
+    ensure_classmethod_init_subclass(w_self)
 
 def ensure_static_new(w_self):
     # special-case __new__, as in CPython:
@@ -1320,6 +1332,12 @@
         if isinstance(w_new, Function):
             w_self.dict_w['__new__'] = StaticMethod(w_new)
 
+def ensure_classmethod_init_subclass(w_self):
+    if '__init_subclass__' in w_self.dict_w:
+        w_init_subclass = w_self.dict_w['__init_subclass__']
+        if isinstance(w_init_subclass, Function):
+            w_self.dict_w['__init_subclass__'] = ClassMethod(w_init_subclass)
+
 def ensure_module_attr(w_self):
     # initialize __module__ in the dict (user-defined types only)
     if '__module__' not in w_self.dict_w:

From pypy.commits at gmail.com  Thu May  3 05:51:38 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 03 May 2018 02:51:38 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8: minimize diff to py3.5
Message-ID: <5aeadbaa.81991c0a.a2e4b.a9a7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8
Changeset: r94463:b1f2a7018522
Date: 2018-05-01 23:24 +0300
http://bitbucket.org/pypy/pypy/changeset/b1f2a7018522/

Log:	minimize diff to py3.5

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -1115,25 +1115,48 @@
                                                          errorhandler, "native")
     return result, c, lgt
 
+def py3k_str_decode_utf_32(s, size, errors, final=True,
+                           errorhandler=None):
+    result, c, lgt, _ = str_decode_utf_32_helper(s, errors, final,
+                                 errorhandler, "native", 'utf-32-' + BYTEORDER2,
+        allow_surrogates=False)
+    return result, c, lgt
+
 def str_decode_utf_32_be(s, errors, final=True,
                          errorhandler=None):
     result, c, lgt, _ = str_decode_utf_32_helper(s, errors, final,
                                                          errorhandler, "big")
     return result, c, lgt
 
+def py3k_str_decode_utf_32_be(s, size, errors, final=True,
+                              errorhandler=None):
+    result, c, lgt, _ = str_decode_utf_32_helper(
+        s, errors, final, errorhandler, "big", 'utf-32-be',
+        allow_surrogates=False)
+    return result, c, lgt
+
 def str_decode_utf_32_le(s, errors, final=True,
                          errorhandler=None):
     result, c, lgt, _ = str_decode_utf_32_helper(s, errors, final,
                                                          errorhandler, "little")
     return result, c, lgt
 
+def py3k_str_decode_utf_32_le(s, size, errors, final=True,
+                              errorhandler=None):
+    result, c, lgt, _ = str_decode_utf_32_helper(
+        s, errors, final, errorhandler, "little", 'utf-32-le',
+        allow_surrogates=False)
+    return result, c, lgt
+
 BOM32_DIRECT  = intmask(0x0000FEFF)
 BOM32_REVERSE = intmask(0xFFFE0000)
 
-def str_decode_utf_32_helper(s, errors, final=True,
-                             errorhandler=None,
+def str_decode_utf_32_helper(s, errors, final,
+                             errorhandler,
                              byteorder="native",
-                             public_encoding_name='utf32'):
+                             public_encoding_name='utf32',
+                             allow_surrogates=True):
+    assert errorhandler is not None
     bo = 0
     size = len(s)
 
@@ -1196,14 +1219,21 @@
             continue
         ch = ((ord(s[pos + iorder[3]]) << 24) | (ord(s[pos + iorder[2]]) << 16) |
               (ord(s[pos + iorder[1]]) << 8)  | ord(s[pos + iorder[0]]))
-        if ch >= 0x110000:
+        if not allow_surrogates and 0xD800 <= ch <= 0xDFFF:
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "code point in surrogate code point "
+                                  "range(0xd800, 0xe000)",
+                                  s, pos, pos + 4)
+            result.append(r)
+            continue
+        elif ch >= 0x110000:
             r, pos = errorhandler(errors, public_encoding_name,
                                   "codepoint not in range(0x110000)",
                                   s, pos, len(s))
             result.append(r)
             continue
 
-        rutf8.unichr_as_utf8_append(result, ch, allow_surrogates=True)
+        rutf8.unichr_as_utf8_append(result, ch, allow_surrogates=allow_surrogates)
         pos += 4
     r = result.build()
     lgt = rutf8.check_utf8(r, True)
@@ -1283,24 +1313,6 @@
     return unicode_encode_utf_32_helper(s, errors, errorhandler,
                                         allow_surrogates, "little")
 
-def py3k_str_decode_utf_32(s, size, errors, final=True,
-                           errorhandler=None):
-    result, length, byteorder = str_decode_utf_32_helper(
-        s, size, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2)
-    return result, length
-
-def py3k_str_decode_utf_32_be(s, size, errors, final=True,
-                              errorhandler=None):
-    result, length, byteorder = str_decode_utf_32_helper(
-        s, size, errors, final, errorhandler, "big", 'utf-32-be')
-    return result, length
-
-def py3k_str_decode_utf_32_le(s, size, errors, final=True,
-                              errorhandler=None):
-    result, length, byteorder = str_decode_utf_32_helper(
-        s, size, errors, final, errorhandler, "little", 'utf-32-le')
-    return result, length
-
 def py3k_unicode_encode_utf_32(s, size, errors,
                                errorhandler=None, allow_surrogates=True):
     return unicode_encode_utf_32_helper(s, size, errors, errorhandler,

From pypy.commits at gmail.com  Thu May  3 05:51:41 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 03 May 2018 02:51:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: enable cppyy
Message-ID: <5aeadbad.1c69fb81.ea4d7.b5c7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94464:fff10263dfb0
Date: 2018-05-03 12:50 +0300
http://bitbucket.org/pypy/pypy/changeset/fff10263dfb0/

Log:	enable cppyy

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,7 +39,7 @@
     "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
-    "_csv", "_pypyjson", "_posixsubprocess", # "_cppyy", "micronumpy"
+    "_csv", "_pypyjson", "_posixsubprocess", "_cppyy", # "micronumpy",
     "_jitlog",
 ])
 

From pypy.commits at gmail.com  Thu May  3 11:57:54 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 03 May 2018 08:57:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: change macro for cpython compatibility,
 fixes #2819
Message-ID: <5aeb3182.1c69fb81.c7b9f.20a0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94465:0525720a751e
Date: 2018-05-03 18:57 +0300
http://bitbucket.org/pypy/pypy/changeset/0525720a751e/

Log:	change macro for cpython compatibility, fixes #2819

diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -333,7 +333,7 @@
         } while (0)
 
 #define PyObject_TypeCheck(ob, tp) \
-    ((ob)->ob_type == (tp) || PyType_IsSubtype((ob)->ob_type, (tp)))
+    (Py_TYPE(ob) == (tp) || PyType_IsSubtype(Py_TYPE(ob), (tp)))
 
 #define Py_TRASHCAN_SAFE_BEGIN(pyObj) do {
 #define Py_TRASHCAN_SAFE_END(pyObj)   ; } while(0);

From pypy.commits at gmail.com  Thu May  3 12:05:04 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 03 May 2018 09:05:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Backed out changeset: fff10263dfb0,
 _cppyy does not translate
Message-ID: <5aeb3330.831d1c0a.c1d03.648c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94466:f785459b8198
Date: 2018-05-03 19:04 +0300
http://bitbucket.org/pypy/pypy/changeset/f785459b8198/

Log:	Backed out changeset: fff10263dfb0, _cppyy does not translate

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,7 +39,7 @@
     "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
-    "_csv", "_pypyjson", "_posixsubprocess", "_cppyy", # "micronumpy",
+    "_csv", "_pypyjson", "_posixsubprocess", # "_cppyy", "micronumpy"
     "_jitlog",
 ])
 

From pypy.commits at gmail.com  Fri May  4 09:26:34 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 04 May 2018 06:26:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix a very rare segfault in the JIT,
 shown by Pycket
Message-ID: <5aec5f8a.81131c0a.d8be7.ec1a@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94470:b0fdbba6aeab
Date: 2018-05-04 15:25 +0200
http://bitbucket.org/pypy/pypy/changeset/b0fdbba6aeab/

Log:	fix a very rare segfault in the JIT, shown by Pycket

	(neither did I manage to write a test, nor to fix the broken
	invariant, which we've seen in other contexts before already.)

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -260,6 +260,12 @@
             # we don't know about this item
             return
         op = optimizer.get_box_replacement(self._fields[fielddescr.get_index()])
+        if op is None:
+            # XXX same bug as in serialize_opt:
+            # op should never be None, because that's an invariant violation in
+            # AbstractCachedEntry. But it still seems to happen when the info
+            # is attached to a Constant. At least we shouldn't crash.
+            return
         opnum = OpHelpers.getfield_for_descr(fielddescr)
         getfield_op = ResOperation(opnum, [structbox], descr=fielddescr)
         shortboxes.add_heap_op(op, getfield_op)
@@ -589,6 +595,7 @@
             return
         item = self._items[index]
         if item is not None:
+            # see comment in AbstractStructPtrInfo.produce_short_preamble_ops
             op = optimizer.get_box_replacement(item)
             opnum = OpHelpers.getarrayitem_for_descr(descr)
             getarrayitem_op = ResOperation(opnum, [structbox, ConstInt(index)],

From pypy.commits at gmail.com  Sun May  6 15:27:25 2018
From: pypy.commits at gmail.com (pv)
Date: Sun, 06 May 2018 12:27:25 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-tls-operror: cpyext: add a test for error
 state race condition
Message-ID: <5aef571d.1c69fb81.536cd.1ffd@mx.google.com>

Author: Pauli Virtanen <pav at iki.fi>
Branch: cpyext-tls-operror
Changeset: r94472:db2b75a607e4
Date: 2018-03-20 21:17 +0100
http://bitbucket.org/pypy/pypy/changeset/db2b75a607e4/

Log:	cpyext: add a test for error state race condition

diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -478,3 +478,59 @@
              '''),
             ])
         raises(SystemError, module.oops)
+
+    def test_error_thread_race(self):
+        # Check race condition: thread 0 returns from cpyext with error set,
+        # after thread 1 has set an error but before it returns.
+        module = self.import_extension('foo', [
+            ("emit_error", "METH_VARARGS",
+             '''
+             PyThreadState *save = NULL;
+             PyGILState_STATE gilsave;
+
+             /* NB. synchronization due to GIL */
+             static volatile int flag = 0;
+             int id;
+
+             if (!PyArg_ParseTuple(args, "i", &id))
+                 return NULL;
+
+             /* Proceed in thread 1 first */
+             save = PyEval_SaveThread();
+             while (id == 0 && flag == 0);
+             gilsave = PyGILState_Ensure();
+
+             PyErr_SetString(PyExc_ValueError, "failure");
+
+             /* Proceed in thread 0 first */
+             if (id == 1) flag = 1;
+             PyGILState_Release(gilsave);
+             while (id == 1 && flag == 1);
+             PyEval_RestoreThread(save);
+
+             if (id == 0) flag = 0;
+             return NULL;
+             '''
+             ),
+            ])
+
+        import threading
+
+        failures = []
+
+        def worker(arg):
+            try:
+                module.emit_error(arg)
+                failures.append(True)
+            except ValueError as exc:
+                if str(exc) != "failure":
+                    failures.append(exc)
+
+        threads = [threading.Thread(target=worker, args=(j,))
+                   for j in (0, 1)]
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+        assert not failures

From pypy.commits at gmail.com  Sun May  6 15:27:23 2018
From: pypy.commits at gmail.com (pv)
Date: Sun, 06 May 2018 12:27:23 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-tls-operror: cpyext: store error state
 thread-locally in executioncontext
Message-ID: <5aef571b.1c69fb81.49d1.8a39@mx.google.com>

Author: Pauli Virtanen <pav at iki.fi>
Branch: cpyext-tls-operror
Changeset: r94471:925ca6fe70e4
Date: 2018-03-11 17:56 +0100
http://bitbucket.org/pypy/pypy/changeset/925ca6fe70e4/

Log:	cpyext: store error state thread-locally in executioncontext

diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -82,10 +82,10 @@
 def PyTraceBack_Here(space, w_frame):
     from pypy.interpreter.pytraceback import record_application_traceback
     state = space.fromcache(State)
-    if state.operror is None:
+    if state.get_exception() is None:
         return -1
     frame = space.interp_w(PyFrame, w_frame)
-    record_application_traceback(space, state.operror, frame, 0)
+    record_application_traceback(space, state.get_exception(), frame, 0)
     return 0
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -67,9 +67,10 @@
 @cpython_api([], PyObject, result_borrowed=True)
 def PyErr_Occurred(space):
     state = space.fromcache(State)
-    if state.operror is None:
+    operror = state.get_exception()
+    if operror is None:
         return None
-    return state.operror.w_type     # borrowed ref
+    return operror.w_type     # borrowed ref
 
 @cpython_api([], lltype.Void)
 def PyErr_Clear(space):
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -2,11 +2,18 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter import executioncontext
+from pypy.interpreter.executioncontext import ExecutionContext
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
 from rpython.rlib import rawrefcount
 import sys
 
+
+# Keep track of exceptions raised in cpyext for a particular execution
+# context.
+ExecutionContext.cpyext_operror = None
+
+
 class State:
     def __init__(self, space):
         self.space = space
@@ -18,7 +25,8 @@
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
-        self.operror = None
+        ec = self.space.getexecutioncontext()
+        ec.cpyext_operror = None
         self.new_method_def = lltype.nullptr(PyMethodDef)
 
         # When importing a package, use this to keep track
@@ -37,17 +45,24 @@
 
     def set_exception(self, operror):
         self.clear_exception()
-        self.operror = operror
+        ec = self.space.getexecutioncontext()
+        ec.cpyext_operror = operror
 
     def clear_exception(self):
         """Clear the current exception state, and return the operror."""
-        operror = self.operror
-        self.operror = None
+        ec = self.space.getexecutioncontext()
+        operror = ec.cpyext_operror
+        ec.cpyext_operror = None
         return operror
 
+    def get_exception(self):
+        ec = self.space.getexecutioncontext()
+        return ec.cpyext_operror
+
     @specialize.arg(1)
     def check_and_raise_exception(self, always=False):
-        operror = self.operror
+        ec = self.space.getexecutioncontext()
+        operror = ec.cpyext_operror
         if operror:
             self.clear_exception()
             raise operror
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -364,7 +364,7 @@
             self.unimport_module(name)
         self.cleanup()
         state = self.space.fromcache(State)
-        assert not state.operror
+        assert state.get_exception() is None
 
 
 class AppTestCpythonExtension(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -43,7 +43,8 @@
         api.PyErr_SetObject(space.w_ValueError, space.wrap("a value"))
         assert api.PyErr_Occurred() is space.w_ValueError
         state = space.fromcache(State)
-        assert space.eq_w(state.operror.get_w_value(space),
+        operror = state.get_exception()
+        assert space.eq_w(operror.get_w_value(space),
                           space.wrap("a value"))
 
         api.PyErr_Clear()
@@ -51,12 +52,14 @@
     def test_SetNone(self, space, api):
         api.PyErr_SetNone(space.w_KeyError)
         state = space.fromcache(State)
-        assert space.eq_w(state.operror.w_type, space.w_KeyError)
-        assert space.eq_w(state.operror.get_w_value(space), space.w_None)
+        operror = state.get_exception()
+        assert space.eq_w(operror.w_type, space.w_KeyError)
+        assert space.eq_w(operror.get_w_value(space), space.w_None)
         api.PyErr_Clear()
 
         api.PyErr_NoMemory()
-        assert space.eq_w(state.operror.w_type, space.w_MemoryError)
+        operror = state.get_exception()
+        assert space.eq_w(operror.w_type, space.w_MemoryError)
         api.PyErr_Clear()
 
     def test_Warning(self, space, api, capfd):

From pypy.commits at gmail.com  Sun May  6 15:27:27 2018
From: pypy.commits at gmail.com (pv)
Date: Sun, 06 May 2018 12:27:27 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-tls-operror: Use more specific exception
 for the test
Message-ID: <5aef571f.1c69fb81.7471c.d9b0@mx.google.com>

Author: Pauli Virtanen <pav at iki.fi>
Branch: cpyext-tls-operror
Changeset: r94473:e33ecbb4d6bd
Date: 2018-03-20 21:37 +0100
http://bitbucket.org/pypy/pypy/changeset/e33ecbb4d6bd/

Log:	Use more specific exception for the test

diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -500,7 +500,7 @@
              while (id == 0 && flag == 0);
              gilsave = PyGILState_Ensure();
 
-             PyErr_SetString(PyExc_ValueError, "failure");
+             PyErr_Format(PyExc_ValueError, "%d", id);
 
              /* Proceed in thread 0 first */
              if (id == 1) flag = 1;
@@ -523,7 +523,7 @@
                 module.emit_error(arg)
                 failures.append(True)
             except ValueError as exc:
-                if str(exc) != "failure":
+                if str(exc) != str(arg):
                     failures.append(exc)
 
         threads = [threading.Thread(target=worker, args=(j,))

From pypy.commits at gmail.com  Sun May  6 15:27:35 2018
From: pypy.commits at gmail.com (filipsalomonsson)
Date: Sun, 06 May 2018 12:27:35 -0700 (PDT)
Subject: [pypy-commit] pypy socket_default_timeout_blockingness: Make sure
 'blocking-ness' of socket is set along with default timeout
Message-ID: <5aef5727.1c69fb81.aec57.89af@mx.google.com>

Author: Filip Salomonsson <filip.salomonsson at gmail.com>
Branch: socket_default_timeout_blockingness
Changeset: r94477:e1b858e8b416
Date: 2018-05-04 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/e1b858e8b416/

Log:	Make sure 'blocking-ness' of socket is set along with default
	timeout

diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -550,7 +550,7 @@
         self.family = family
         self.type = type
         self.proto = proto
-        self.timeout = defaults.timeout
+        self.settimeout(defaults.timeout)
 
     @staticmethod
     def empty_rsocket():
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -465,6 +465,15 @@
     s.connect(INETAddress('python.org', 80))
     s.close()
 
+def test_connect_with_default_timeout_fail():
+    rsocket.setdefaulttimeout(0.1)
+    s = RSocket()
+    rsocket.setdefaulttimeout(None)
+    assert s.gettimeout() == 0.1
+    with py.test.raises(SocketTimeout):
+        s.connect(INETAddress('172.30.172.30', 12345))
+    s.close()
+
 def test_getsetsockopt():
     import struct
     assert struct.calcsize("i") == rffi.sizeof(rffi.INT)

From pypy.commits at gmail.com  Sun May  6 15:27:31 2018
From: pypy.commits at gmail.com (pv)
Date: Sun, 06 May 2018 12:27:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-user-site-impl: Add test for user-site dir
Message-ID: <5aef5723.1c69fb81.311ef.dd5f@mx.google.com>

Author: Pauli Virtanen <pav at iki.fi>
Branch: py3.5-user-site-impl
Changeset: r94475:041d69a3aea1
Date: 2018-05-06 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/041d69a3aea1/

Log:	Add test for user-site dir

diff --git a/lib-python/3/test/test_sysconfig_pypy.py b/lib-python/3/test/test_sysconfig_pypy.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/test_sysconfig_pypy.py
@@ -0,0 +1,17 @@
+import os
+import sys
+import unittest
+import site
+
+
+class TestSysConfigPypy(unittest.TestCase):
+    def test_install_schemes(self):
+        # User-site etc. paths should have "pypy" and not "python"
+        # inside them.
+        if site.ENABLE_USER_SITE:
+            parts = site.USER_SITE.lower().split(os.path.sep)
+            assert any(x.startswith('pypy') for x in parts[-2:]), parts
+
+
+if __name__ == "__main__":
+    unittest.main()

From pypy.commits at gmail.com  Sun May  6 15:27:33 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:27:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge py3.5-user-site-impl into py3.5
Message-ID: <5aef5725.1c69fb81.d93f8.3854@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94476:00bffc96bc78
Date: 2018-05-06 22:04 +0300
http://bitbucket.org/pypy/pypy/changeset/00bffc96bc78/

Log:	merge py3.5-user-site-impl into py3.5

diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -20,30 +20,30 @@
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{installed_base}/lib/python{py_version_short}',
-        'platstdlib': '{platbase}/lib/python{py_version_short}',
-        'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
+        'stdlib': '{installed_base}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{platbase}/lib/{implementation_lower}{py_version_short}/site-packages',
         'include':
-            '{installed_base}/include/python{py_version_short}{abiflags}',
+            '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
         'platinclude':
-            '{installed_platbase}/include/python{py_version_short}{abiflags}',
+            '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'posix_home': {
-        'stdlib': '{installed_base}/lib/python',
-        'platstdlib': '{base}/lib/python',
-        'purelib': '{base}/lib/python',
-        'platlib': '{base}/lib/python',
-        'include': '{installed_base}/include/python',
-        'platinclude': '{installed_base}/include/python',
+        'stdlib': '{installed_base}/lib/{implementation_lower}',
+        'platstdlib': '{base}/lib/{implementation_lower}',
+        'purelib': '{base}/lib/{implementation_lower}',
+        'platlib': '{base}/lib/{implementation_lower}',
+        'include': '{installed_base}/include/{implementation_lower}',
+        'platinclude': '{installed_base}/include/{implementation_lower}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'pypy': {
-        'stdlib': '{installed_base}/lib-python',
-        'platstdlib': '{base}/lib-python',
+        'stdlib': '{installed_base}/lib-{implementation_lower}',
+        'platstdlib': '{base}/lib-{implementation_lower}',
         'purelib': '{base}/site-packages',
         'platlib': '{base}/site-packages',
         'include': '{installed_base}/include',
@@ -62,28 +62,28 @@
         'data': '{base}',
         },
     'nt_user': {
-        'stdlib': '{userbase}/Python{py_version_nodot}',
-        'platstdlib': '{userbase}/Python{py_version_nodot}',
-        'purelib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'include': '{userbase}/Python{py_version_nodot}/Include',
-        'scripts': '{userbase}/Python{py_version_nodot}/Scripts',
+        'stdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'platstdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'purelib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'platlib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'include': '{userbase}/{implementation}{py_version_nodot}/Include',
+        'scripts': '{userbase}/{implementation}{py_version_nodot}/Scripts',
         'data': '{userbase}',
         },
     'posix_user': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
+        'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{userbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
         },
     'osx_framework_user': {
-        'stdlib': '{userbase}/lib/python',
-        'platstdlib': '{userbase}/lib/python',
-        'purelib': '{userbase}/lib/python/site-packages',
-        'platlib': '{userbase}/lib/python/site-packages',
+        'stdlib': '{userbase}/lib/{implementation_lower}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}',
+        'purelib': '{userbase}/lib/{implementation_lower}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}/site-packages',
         'include': '{userbase}/include',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
@@ -106,6 +106,11 @@
 _USER_BASE = None
 
 
+def _get_implementation():
+    if '__pypy__' in sys.builtin_module_names:
+        return 'PyPy'
+    return 'Python'
+
 def _safe_realpath(path):
     try:
         return realpath(path)
@@ -540,6 +545,8 @@
         except AttributeError:
             # sys.abiflags may not be defined on all platforms.
             _CONFIG_VARS['abiflags'] = ''
+        _CONFIG_VARS['implementation'] = _get_implementation()
+        _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
 
         if os.name == 'nt':
             _init_non_posix(_CONFIG_VARS)
@@ -715,6 +722,8 @@
     _print_dict('Paths', get_paths())
     print()
     _print_dict('Variables', get_config_vars())
+    print
+    _print_dict('User', get_paths('%s_user' % os.name))
 
 
 if __name__ == '__main__':
diff --git a/lib-python/3/test/test_sysconfig_pypy.py b/lib-python/3/test/test_sysconfig_pypy.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/test_sysconfig_pypy.py
@@ -0,0 +1,17 @@
+import os
+import sys
+import unittest
+import site
+
+
+class TestSysConfigPypy(unittest.TestCase):
+    def test_install_schemes(self):
+        # User-site etc. paths should have "pypy" and not "python"
+        # inside them.
+        if site.ENABLE_USER_SITE:
+            parts = site.USER_SITE.lower().split(os.path.sep)
+            assert any(x.startswith('pypy') for x in parts[-2:]), parts
+
+
+if __name__ == "__main__":
+    unittest.main()

From pypy.commits at gmail.com  Sun May  6 15:27:37 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:27:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge
 socket_default_timeout_blockingness into default
Message-ID: <5aef5729.1c69fb81.dfe9c.4bd6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94478:7fbd70c05558
Date: 2018-05-06 22:24 +0300
http://bitbucket.org/pypy/pypy/changeset/7fbd70c05558/

Log:	merge socket_default_timeout_blockingness into default

diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -550,7 +550,7 @@
         self.family = family
         self.type = type
         self.proto = proto
-        self.timeout = defaults.timeout
+        self.settimeout(defaults.timeout)
 
     @staticmethod
     def empty_rsocket():
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -465,6 +465,15 @@
     s.connect(INETAddress('python.org', 80))
     s.close()
 
+def test_connect_with_default_timeout_fail():
+    rsocket.setdefaulttimeout(0.1)
+    s = RSocket()
+    rsocket.setdefaulttimeout(None)
+    assert s.gettimeout() == 0.1
+    with py.test.raises(SocketTimeout):
+        s.connect(INETAddress('172.30.172.30', 12345))
+    s.close()
+
 def test_getsetsockopt():
     import struct
     assert struct.calcsize("i") == rffi.sizeof(rffi.INT)

From pypy.commits at gmail.com  Sun May  6 15:27:39 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:27:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge default into py3.5
Message-ID: <5aef572b.1c69fb81.98d21.4b89@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94479:2885e6b1619d
Date: 2018-05-06 22:26 +0300
http://bitbucket.org/pypy/pypy/changeset/2885e6b1619d/

Log:	merge default into py3.5

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -260,6 +260,12 @@
             # we don't know about this item
             return
         op = optimizer.get_box_replacement(self._fields[fielddescr.get_index()])
+        if op is None:
+            # XXX same bug as in serialize_opt:
+            # op should never be None, because that's an invariant violation in
+            # AbstractCachedEntry. But it still seems to happen when the info
+            # is attached to a Constant. At least we shouldn't crash.
+            return
         opnum = OpHelpers.getfield_for_descr(fielddescr)
         getfield_op = ResOperation(opnum, [structbox], descr=fielddescr)
         shortboxes.add_heap_op(op, getfield_op)
@@ -589,6 +595,7 @@
             return
         item = self._items[index]
         if item is not None:
+            # see comment in AbstractStructPtrInfo.produce_short_preamble_ops
             op = optimizer.get_box_replacement(item)
             opnum = OpHelpers.getarrayitem_for_descr(descr)
             getarrayitem_op = ResOperation(opnum, [structbox, ConstInt(index)],
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -550,7 +550,7 @@
         self.family = family
         self.type = type
         self.proto = proto
-        self.timeout = defaults.timeout
+        self.settimeout(defaults.timeout)
 
     @staticmethod
     def empty_rsocket():
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -465,6 +465,15 @@
     s.connect(INETAddress('python.org', 80))
     s.close()
 
+def test_connect_with_default_timeout_fail():
+    rsocket.setdefaulttimeout(0.1)
+    s = RSocket()
+    rsocket.setdefaulttimeout(None)
+    assert s.gettimeout() == 0.1
+    with py.test.raises(SocketTimeout):
+        s.connect(INETAddress('172.30.172.30', 12345))
+    s.close()
+
 def test_getsetsockopt():
     import struct
     assert struct.calcsize("i") == rffi.sizeof(rffi.INT)

From pypy.commits at gmail.com  Sun May  6 15:27:29 2018
From: pypy.commits at gmail.com (pv)
Date: Sun, 06 May 2018 12:27:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-user-site-impl: Use
 implementation-specific site directories in sysconfig.
Message-ID: <5aef5721.92081c0a.e7a6e.5583@mx.google.com>

Author: Pauli Virtanen <pav at iki.fi>
Branch: py3.5-user-site-impl
Changeset: r94474:5ca68e43950b
Date: 2018-04-08 21:01 +0200
http://bitbucket.org/pypy/pypy/changeset/5ca68e43950b/

Log:	Use implementation-specific site directories in sysconfig.

	This is a forward port of fbb5b4cefb37ea859eccd0f715101a6c8c57e32f
	to py3.5

diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -20,30 +20,30 @@
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{installed_base}/lib/python{py_version_short}',
-        'platstdlib': '{platbase}/lib/python{py_version_short}',
-        'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
+        'stdlib': '{installed_base}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{platbase}/lib/{implementation_lower}{py_version_short}/site-packages',
         'include':
-            '{installed_base}/include/python{py_version_short}{abiflags}',
+            '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
         'platinclude':
-            '{installed_platbase}/include/python{py_version_short}{abiflags}',
+            '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'posix_home': {
-        'stdlib': '{installed_base}/lib/python',
-        'platstdlib': '{base}/lib/python',
-        'purelib': '{base}/lib/python',
-        'platlib': '{base}/lib/python',
-        'include': '{installed_base}/include/python',
-        'platinclude': '{installed_base}/include/python',
+        'stdlib': '{installed_base}/lib/{implementation_lower}',
+        'platstdlib': '{base}/lib/{implementation_lower}',
+        'purelib': '{base}/lib/{implementation_lower}',
+        'platlib': '{base}/lib/{implementation_lower}',
+        'include': '{installed_base}/include/{implementation_lower}',
+        'platinclude': '{installed_base}/include/{implementation_lower}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'pypy': {
-        'stdlib': '{installed_base}/lib-python',
-        'platstdlib': '{base}/lib-python',
+        'stdlib': '{installed_base}/lib-{implementation_lower}',
+        'platstdlib': '{base}/lib-{implementation_lower}',
         'purelib': '{base}/site-packages',
         'platlib': '{base}/site-packages',
         'include': '{installed_base}/include',
@@ -62,28 +62,28 @@
         'data': '{base}',
         },
     'nt_user': {
-        'stdlib': '{userbase}/Python{py_version_nodot}',
-        'platstdlib': '{userbase}/Python{py_version_nodot}',
-        'purelib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'include': '{userbase}/Python{py_version_nodot}/Include',
-        'scripts': '{userbase}/Python{py_version_nodot}/Scripts',
+        'stdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'platstdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'purelib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'platlib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'include': '{userbase}/{implementation}{py_version_nodot}/Include',
+        'scripts': '{userbase}/{implementation}{py_version_nodot}/Scripts',
         'data': '{userbase}',
         },
     'posix_user': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
+        'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{userbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
         },
     'osx_framework_user': {
-        'stdlib': '{userbase}/lib/python',
-        'platstdlib': '{userbase}/lib/python',
-        'purelib': '{userbase}/lib/python/site-packages',
-        'platlib': '{userbase}/lib/python/site-packages',
+        'stdlib': '{userbase}/lib/{implementation_lower}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}',
+        'purelib': '{userbase}/lib/{implementation_lower}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}/site-packages',
         'include': '{userbase}/include',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
@@ -106,6 +106,11 @@
 _USER_BASE = None
 
 
+def _get_implementation():
+    if '__pypy__' in sys.builtin_module_names:
+        return 'PyPy'
+    return 'Python'
+
 def _safe_realpath(path):
     try:
         return realpath(path)
@@ -540,6 +545,8 @@
         except AttributeError:
             # sys.abiflags may not be defined on all platforms.
             _CONFIG_VARS['abiflags'] = ''
+        _CONFIG_VARS['implementation'] = _get_implementation()
+        _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
 
         if os.name == 'nt':
             _init_non_posix(_CONFIG_VARS)
@@ -715,6 +722,8 @@
     _print_dict('Paths', get_paths())
     print()
     _print_dict('Variables', get_config_vars())
+    print
+    _print_dict('User', get_paths('%s_user' % os.name))
 
 
 if __name__ == '__main__':

From pypy.commits at gmail.com  Sun May  6 15:46:00 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-user-site-impl: close merged branch
Message-ID: <5aef5b78.92081c0a.e7a6e.58e0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5-user-site-impl
Changeset: r94480:2edbc073a984
Date: 2018-05-06 22:28 +0300
http://bitbucket.org/pypy/pypy/changeset/2edbc073a984/

Log:	close merged branch


From pypy.commits at gmail.com  Sun May  6 15:46:04 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:04 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-tls-operror: close merged branch
Message-ID: <5aef5b7c.1c69fb81.34d09.fac5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-tls-operror
Changeset: r94482:1275b1c9fddc
Date: 2018-05-06 22:36 +0300
http://bitbucket.org/pypy/pypy/changeset/1275b1c9fddc/

Log:	close merged branch


From pypy.commits at gmail.com  Sun May  6 15:46:05 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:05 -0700 (PDT)
Subject: [pypy-commit] pypy socket_default_timeout_blockingness: close
 merged branch
Message-ID: <5aef5b7d.1c69fb81.51e2e.aa08@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: socket_default_timeout_blockingness
Changeset: r94483:abf6010cf7ff
Date: 2018-05-06 22:36 +0300
http://bitbucket.org/pypy/pypy/changeset/abf6010cf7ff/

Log:	close merged branch


From pypy.commits at gmail.com  Sun May  6 15:46:07 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: update whatsnew for branches
Message-ID: <5aef5b7f.1c69fb81.9fc8.b63f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94484:b3a61c9cdf11
Date: 2018-05-06 22:40 +0300
http://bitbucket.org/pypy/pypy/changeset/b3a61c9cdf11/

Log:	update whatsnew for branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,5 +5,13 @@
 .. this is a revision shortly after release-pypy-6.0.0
 .. startrev: e50e11af23f1
 
+.. branch: cppyy-packaging
 
+Upgrade to backend 0.6.0, support exception handling from wrapped functions,
+update enum handling, const correctness for data members and associated tests,
+support anonymous enums, support for function pointer arguments
 
+.. branch: socket_default_timeout_blockingness
+
+Make sure 'blocking-ness' of socket is set along with default timeout
+

From pypy.commits at gmail.com  Sun May  6 15:46:13 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:13 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 17c6fa64f211
 on branch rawrefcount-free-early
Message-ID: <5aef5b85.424a1c0a.f447c.8188@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r94487:c523844f4586
Date: 2018-05-06 22:44 +0300
http://bitbucket.org/pypy/pypy/changeset/c523844f4586/

Log:	Merge closed head 17c6fa64f211 on branch rawrefcount-free-early


From pypy.commits at gmail.com  Sun May  6 15:46:09 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:09 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head d10a6cb00365
 on branch py3.5-msvc14
Message-ID: <5aef5b81.1c69fb81.415fb.0b58@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r94485:362902a9ea52
Date: 2018-05-06 22:44 +0300
http://bitbucket.org/pypy/pypy/changeset/362902a9ea52/

Log:	Merge closed head d10a6cb00365 on branch py3.5-msvc14


From pypy.commits at gmail.com  Sun May  6 15:46:15 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:15 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head d53e1ebe51e1
 on branch memory-accounting
Message-ID: <5aef5b87.1c69fb81.7250.0ed5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r94488:d4462be9cfce
Date: 2018-05-06 22:44 +0300
http://bitbucket.org/pypy/pypy/changeset/d4462be9cfce/

Log:	Merge closed head d53e1ebe51e1 on branch memory-accounting


From pypy.commits at gmail.com  Sun May  6 15:46:16 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:16 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head bc36eabcc716
 on branch jit-hooks-can-be-disabled
Message-ID: <5aef5b88.1c69fb81.3f68c.1520@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r94489:d54a200c903b
Date: 2018-05-06 22:44 +0300
http://bitbucket.org/pypy/pypy/changeset/d54a200c903b/

Log:	Merge closed head bc36eabcc716 on branch jit-hooks-can-be-disabled


From pypy.commits at gmail.com  Sun May  6 15:46:11 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:11 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head b579b2e8a71e
 on branch throw-away
Message-ID: <5aef5b83.87981c0a.8f77f.2f54@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r94486:51301b8a330b
Date: 2018-05-06 22:44 +0300
http://bitbucket.org/pypy/pypy/changeset/51301b8a330b/

Log:	Merge closed head b579b2e8a71e on branch throw-away


From pypy.commits at gmail.com  Sun May  6 15:46:18 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:18 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 1275b1c9fddc
 on branch cpyext-tls-operror
Message-ID: <5aef5b8a.1c69fb81.76527.3148@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r94490:d6edebd8039a
Date: 2018-05-06 22:44 +0300
http://bitbucket.org/pypy/pypy/changeset/d6edebd8039a/

Log:	Merge closed head 1275b1c9fddc on branch cpyext-tls-operror


From pypy.commits at gmail.com  Sun May  6 15:46:20 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:20 -0700 (PDT)
Subject: [pypy-commit] pypy closed-branches: re-close this branch
Message-ID: <5aef5b8c.45341c0a.22d06.872f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: closed-branches
Changeset: r94491:ce1948553091
Date: 2018-05-06 22:44 +0300
http://bitbucket.org/pypy/pypy/changeset/ce1948553091/

Log:	re-close this branch


From pypy.commits at gmail.com  Sun May  6 15:46:02 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 12:46:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: update whatsnew for branches
Message-ID: <5aef5b7a.1c69fb81.b28eb.fd5b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94481:d25f1e718fd7
Date: 2018-05-06 22:35 +0300
http://bitbucket.org/pypy/pypy/changeset/d25f1e718fd7/

Log:	update whatsnew for branches

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -5,3 +5,10 @@
 .. this is the revision after release-pypy3.5-v6.0
 .. startrev: 580e3e26cd32
 
+.. branch: hroncok/fix-multiprocessing-regression-on-newer--1524656522151
+
+Fix multiprocessing regression on newer glibcs
+
+.. branch: py3.5-user-site-impl
+
+Use implementation-specific site directories in sysconfig like in Python2

From pypy.commits at gmail.com  Sun May  6 19:09:41 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 May 2018 16:09:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in pv/pypy/crypt_h (pull request
 #611)
Message-ID: <5aef8b35.1c69fb81.e7fae.8799@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r94494:740ad838d0c5
Date: 2018-05-06 23:09 +0000
http://bitbucket.org/pypy/pypy/changeset/740ad838d0c5/

Log:	Merged in pv/pypy/crypt_h (pull request #611)

	Include crypt.h for crypt() on Linux

diff --git a/pypy/module/crypt/interp_crypt.py b/pypy/module/crypt/interp_crypt.py
--- a/pypy/module/crypt/interp_crypt.py
+++ b/pypy/module/crypt/interp_crypt.py
@@ -5,6 +5,9 @@
 
 if sys.platform.startswith('darwin'):
     eci = ExternalCompilationInfo()
+elif sys.platform.startswith('linux'):
+    # crypt() is defined only in crypt.h on some Linux variants (eg. Fedora 28)
+    eci = ExternalCompilationInfo(libraries=['crypt'], includes=["crypt.h"])
 else:
     eci = ExternalCompilationInfo(libraries=['crypt'])
 c_crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP,

From pypy.commits at gmail.com  Sun May  6 19:09:50 2018
From: pypy.commits at gmail.com (pv)
Date: Sun, 06 May 2018 16:09:50 -0700 (PDT)
Subject: [pypy-commit] pypy crypt_h: Include crypt.h for crypt() on Linux
Message-ID: <5aef8b3e.1c69fb81.58e1.b2d0@mx.google.com>

Author: Pauli Virtanen <pav at iki.fi>
Branch: crypt_h
Changeset: r94493:32f93d4eabfa
Date: 2018-05-06 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/32f93d4eabfa/

Log:	Include crypt.h for crypt() on Linux

diff --git a/pypy/module/crypt/interp_crypt.py b/pypy/module/crypt/interp_crypt.py
--- a/pypy/module/crypt/interp_crypt.py
+++ b/pypy/module/crypt/interp_crypt.py
@@ -5,6 +5,9 @@
 
 if sys.platform.startswith('darwin'):
     eci = ExternalCompilationInfo()
+elif sys.platform.startswith('linux'):
+    # crypt() is defined only in crypt.h on some Linux variants (eg. Fedora 28)
+    eci = ExternalCompilationInfo(libraries=['crypt'], includes=["crypt.h"])
 else:
     eci = ExternalCompilationInfo(libraries=['crypt'])
 c_crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP,

From pypy.commits at gmail.com  Sun May  6 19:41:28 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 06 May 2018 16:41:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2817
Message-ID: <5aef92a8.1c69fb81.34d09.3ef6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94495:5d37e5adb23c
Date: 2018-05-07 01:40 +0200
http://bitbucket.org/pypy/pypy/changeset/5d37e5adb23c/

Log:	Issue #2817

	Copy a few more definitions from CPython's pyport.h

diff --git a/pypy/module/cpyext/include/pyport.h b/pypy/module/cpyext/include/pyport.h
--- a/pypy/module/cpyext/include/pyport.h
+++ b/pypy/module/cpyext/include/pyport.h
@@ -110,4 +110,36 @@
 #else
 #endif
 
+/*
+ * Hide GCC attributes from compilers that don't support them.
+ */
+#if (!defined(__GNUC__) || __GNUC__ < 2 || \
+     (__GNUC__ == 2 && __GNUC_MINOR__ < 7) ) && \
+    !defined(RISCOS)
+#define Py_GCC_ATTRIBUTE(x)
+#else
+#define Py_GCC_ATTRIBUTE(x) __attribute__(x)
+#endif
+
+/*
+ * Specify alignment on compilers that support it.
+ */
+#if defined(__GNUC__) && __GNUC__ >= 3
+#define Py_ALIGNED(x) __attribute__((aligned(x)))
+#else
+#define Py_ALIGNED(x)
+#endif
+
+/*
+ * Older Microsoft compilers don't support the C99 long long literal suffixes,
+ * so these will be defined in PC/pyconfig.h for those compilers.
+ */
+#ifndef Py_LL
+#define Py_LL(x) x##LL
+#endif
+
+#ifndef Py_ULL
+#define Py_ULL(x) Py_LL(x##U)
+#endif
+
 #endif /* Py_PYPORT_H */

From pypy.commits at gmail.com  Mon May  7 01:28:16 2018
From: pypy.commits at gmail.com (mattip)
Date: Sun, 06 May 2018 22:28:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: resync with vmprof upstream,
 the typo should be fixed there first
Message-ID: <5aefe3f0.1c69fb81.536cd.724f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94496:cb8878f4cca1
Date: 2018-05-07 08:25 +0300
http://bitbucket.org/pypy/pypy/changeset/cb8878f4cca1/

Log:	resync with vmprof upstream, the typo should be fixed there first

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -15,3 +15,6 @@
 
 Make sure 'blocking-ness' of socket is set along with default timeout
 
+.. branch: crypt_h
+
+Include crypt.h for crypt() on Linux
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -214,7 +214,7 @@
     ret = unw_getcontext(&uc);
     if (ret < 0) {
         // could not initialize lib unwind cursor and context
-        fprintf(stderr, "WARNING: unw_getcontext did not retrieve context, switching to python profiling mode \n");
+        fprintf(stderr, "WARNING: unw_getcontext did not retreive context, switching to python profiling mode \n");
         vmp_native_disable();
         return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
     }

From pypy.commits at gmail.com  Mon May  7 22:42:52 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 07 May 2018 19:42:52 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: merge default into branch
Message-ID: <5af10eac.1c69fb81.8be09.fbaa@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94497:0eee6ab380c7
Date: 2018-05-07 13:34 -0700
http://bitbucket.org/pypy/pypy/changeset/0eee6ab380c7/

Log:	merge default into branch

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -51,3 +51,5 @@
 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0
 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0
 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1
+ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0
+fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -247,6 +247,7 @@
   Lukas Vacek
   Omer Katz
   Jacek Generowicz
+  Tomasz Dziopa
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -307,6 +308,7 @@
   Yury V. Zaytsev
   florinpapa
   Anders Sigfridsson
+  Matt Jackson
   Nikolay Zinov
   rafalgalczynski at gmail.com
   Joshua Gilbert
diff --git a/dotviewer/font/NOTICE b/dotviewer/font/COPYING.txt
rename from dotviewer/font/NOTICE
rename to dotviewer/font/COPYING.txt
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -82,8 +82,11 @@
     def _CData_output(self, resarray, base=None, index=-1):
         from _rawffi.alt import types
         # If a char_p or unichar_p is received, skip the string interpretation
-        if base._ffiargtype != types.Pointer(types.char_p) and \
-           base._ffiargtype != types.Pointer(types.unichar_p):
+        try:
+            deref = type(base)._deref_ffiargtype()
+        except AttributeError:
+            deref = None
+        if deref != types.char_p and deref != types.unichar_p:
             # this seems to be a string if we're array of char, surprise!
             from ctypes import c_char, c_wchar
             if self._type_ is c_char:
@@ -120,6 +123,12 @@
                 value = self(*value)
         return _CDataMeta.from_param(self, value)
 
+    def _build_ffiargtype(self):
+        return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+    def _deref_ffiargtype(self):
+        return self._type_.get_ffi_argtype()
+
 def array_get_slice_params(self, index):
     if hasattr(self, '_length_'):
         start, stop, step = index.indices(self._length_)
@@ -248,6 +257,5 @@
             _type_ = base
         )
         cls = ArrayMeta(name, (Array,), tpdict)
-        cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype())
         ARRAY_CACHE[key] = cls
         return cls
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -49,10 +49,13 @@
         else:
             return self.from_param(as_parameter)
 
+    def _build_ffiargtype(self):
+        return _shape_to_ffi_type(self._ffiargshape_)
+
     def get_ffi_argtype(self):
         if self._ffiargtype:
             return self._ffiargtype
-        self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_)
+        self._ffiargtype = self._build_ffiargtype()
         return self._ffiargtype
 
     def _CData_output(self, resbuffer, base=None, index=-1):
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -70,7 +70,12 @@
         self._ffiarray = ffiarray
         self.__init__ = __init__
         self._type_ = TP
-        self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype())
+
+    def _build_ffiargtype(self):
+        return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+    def _deref_ffiargtype(self):
+        return self._type_.get_ffi_argtype()
 
     from_address = cdata_from_address
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -160,6 +160,10 @@
             raise AttributeError("_fields_ is final")
         if self in [f[1] for f in value]:
             raise AttributeError("Structure or union cannot contain itself")
+        if self._ffiargtype is not None:
+            raise NotImplementedError("Too late to set _fields_: we already "
+                        "said to libffi that the structure type %s is opaque"
+                        % (self,))
         names_and_fields(
             self,
             value, self.__bases__[0],
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -214,6 +214,7 @@
   Lukas Vacek
   Omer Katz
   Jacek Generowicz
+  Tomasz Dziopa
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -274,6 +275,7 @@
   Yury V. Zaytsev
   florinpapa
   Anders Sigfridsson
+  Matt Jackson
   Nikolay Zinov
   rafalgalczynski at gmail.com
   Joshua Gilbert
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -152,7 +152,7 @@
 to wait until it reaches a point in which the interpreter is in a known state
 and calling user-defined code is harmless.  It might happen that multiple
 events occur before the hook is invoked: in this case, you can inspect the
-value ``stats.count`` to know how many times the event occured since the last
+value ``stats.count`` to know how many times the event occurred since the last
 time the hook was called.  Similarly, ``stats.duration`` contains the
 **total** time spent by the GC for this specific event since the last time the
 hook was called.
@@ -163,7 +163,7 @@
 The attributes for ``GcMinorStats`` are:
 
 ``count``
-    The number of minor collections occured since the last hook call.
+    The number of minor collections occurred since the last hook call.
 
 ``duration``
     The total time spent inside minor collections since the last hook
diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst
--- a/pypy/doc/release-v6.0.0.rst
+++ b/pypy/doc/release-v6.0.0.rst
@@ -8,28 +8,35 @@
 the dual release.
 
 This release is a feature release following our previous 5.10 incremental
-release in late December 2017. Our C-API compatability layer ``cpyext`` is
+release in late December 2017. Our C-API compatibility layer ``cpyext`` is
 now much faster (see the `blog post`_) as well as more complete. We have made
 many other improvements in speed and CPython compatibility. Since the changes
 affect the included python development header files, all c-extension modules must
 be recompiled for this version.
 
-First-time python users are often stumped by silly typos and emissions when
+Until we can work with downstream providers to distribute builds with PyPy, we
+have made packages for some common packages `available as wheels`_. You may
+compile yourself using ``pip install --no-build-isolation <package>``, the
+``no-build-isolation`` is currently needed for pip v10.
+
+First-time python users are often stumped by silly typos and omissions when
 getting started writing code. We have improved our parser to emit more friendly
 `syntax errors`_,  making PyPy not only faster but more friendly.
 
 The GC now has `hooks`_ to gain more insights into its performance
 
-The Windows PyPy3.5 release is still considered beta-quality. There are open
-issues with unicode handling especially around system calls and c-extensions.
+The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_.
 
-The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_.
+We updated the `cffi`_ module included in PyPy to version 1.11.5, and the
+`cppyy`_ backend to 0.6.0. Please use these to wrap your C and C++ code,
+respectively, for a JIT friendly experience.
 
 As always, this release is 100% compatible with the previous one and fixed
 several issues and bugs raised by the growing community of PyPy users.
 We strongly recommend updating.
 
-We updated the cffi module included in PyPy to version 1.11.5
+The Windows PyPy3.5 release is still considered beta-quality. There are open
+issues with unicode handling especially around system calls and c-extensions.
 
 The utf8 branch that changes internal representation of unicode to utf8 did not
 make it into the release, so there is still more goodness coming. We also
@@ -56,6 +63,9 @@
 .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html
 .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html
 .. _`hooks`: gc_info.html#gc-hooks
+.. _`cffi`: http://cffi.readthedocs.io
+.. _`cppyy`: https://cppyy.readthedocs.io
+.. _`available as wheels`: https://github.com/antocuni/pypy-wheels
 
 What is PyPy?
 =============
@@ -106,7 +116,7 @@
 * Fix JIT bugs exposed in the sre module
 * Improve speed of Python parser, improve ParseError messages and SyntaxError
 * Handle JIT hooks more efficiently
-* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s
+* Fix a rare GC bug exposed by intensive use of cpyext ``Buffer`` s
 
 We also refactored many parts of the JIT bridge optimizations, as well as cpyext
 internals, and together with new contributors fixed issues, added new
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,7 +3,18 @@
 ==========================
 
 .. this is a revision shortly after release-pypy-6.0.0
-.. startrev: ad79cc0ce9a8
+.. startrev: e50e11af23f1
 
+.. branch: cppyy-packaging
 
+Upgrade to backend 0.6.0, support exception handling from wrapped functions,
+update enum handling, const correctness for data members and associated tests,
+support anonymous enums, support for function pointer arguments
 
+.. branch: socket_default_timeout_blockingness
+
+Make sure 'blocking-ness' of socket is set along with default timeout
+
+.. branch: crypt_h
+
+Include crypt.h for crypt() on Linux
diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst
--- a/pypy/doc/whatsnew-pypy2-6.0.0.rst
+++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst
@@ -121,4 +121,8 @@
 
 .. branch: gc-hook-better-timestamp
 
-Improve GC hooksd
+Improve GC hooks
+
+.. branch: cppyy-packaging
+
+Update backend to 0.6.0 and support exceptions through wrappers
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -182,7 +182,7 @@
     """iter(collection) -> iterator over the elements of the collection.
 
 iter(callable, sentinel) -> iterator calling callable() until it returns
-                            the sentinal.
+                            the sentinel.
 """
     if w_sentinel is None:
         return space.iter(w_collection_or_callable)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -333,7 +333,7 @@
     to the default encoding. errors may be given to set a different error
     handling scheme. Default is 'strict' meaning that encoding errors raise
     a ValueError. Other possible values are 'ignore' and 'replace'
-    as well as any other name registerd with codecs.register_error that is
+    as well as any other name registered with codecs.register_error that is
     able to handle ValueErrors.
     """
     if w_encoding is None:
diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -83,8 +83,8 @@
             if len(value) != 1:
                 raise oefmt(space.w_ValueError,
                         "char expected, got string of size %d", len(value))
-            value = rffi.cast(rffi.CHAR, value[0])
 
+        value = rffi.cast(rffi.CHAR, value[0])
         return value     # turn it into a "char" to the annotator
 
     def cffi_type(self, space):
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -1226,9 +1226,11 @@
                     "'%s' has no length", self.clsdecl.name)
 
     def instance__cmp__(self, w_other):
-        w_as_builtin = self._get_as_builtin()
-        if w_as_builtin is not None:
-            return self.space.cmp(w_as_builtin, w_other)
+        from pypy.module.sys.version import CPYTHON_VERSION
+        if CPYTHON_VERSION[0] != 3:
+            w_as_builtin = self._get_as_builtin()
+            if w_as_builtin is not None:
+                return self.space.cmp(w_as_builtin, w_other)
         raise oefmt(self.space.w_AttributeError,
                     "'%s' has no attribute __cmp__", self.clsdecl.name)
 
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -482,7 +482,7 @@
         assert c.get_valid_string('aap') == 'aap'
         #assert c.get_invalid_string() == ''
 
-    def test12_copy_contructor(self):
+    def test12_copy_constructor(self):
         """Test copy constructor"""
 
         import _cppyy as cppyy
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -606,7 +606,7 @@
         This will return the certificate even if it wasn't validated.
         """
         if not self.handshake_done:
-            raise oefmt(space.w_ValueError, "hanshake not done yet")
+            raise oefmt(space.w_ValueError, "handshake not done yet")
         if not self.peer_cert:
             return space.w_None
 
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -333,7 +333,7 @@
         } while (0)
 
 #define PyObject_TypeCheck(ob, tp) \
-    ((ob)->ob_type == (tp) || PyType_IsSubtype((ob)->ob_type, (tp)))
+    (Py_TYPE(ob) == (tp) || PyType_IsSubtype(Py_TYPE(ob), (tp)))
 
 #define Py_TRASHCAN_SAFE_BEGIN(pyObj) do {
 #define Py_TRASHCAN_SAFE_END(pyObj)   ; } while(0);
diff --git a/pypy/module/cpyext/include/pyport.h b/pypy/module/cpyext/include/pyport.h
--- a/pypy/module/cpyext/include/pyport.h
+++ b/pypy/module/cpyext/include/pyport.h
@@ -110,4 +110,36 @@
 #else
 #endif
 
+/*
+ * Hide GCC attributes from compilers that don't support them.
+ */
+#if (!defined(__GNUC__) || __GNUC__ < 2 || \
+     (__GNUC__ == 2 && __GNUC_MINOR__ < 7) ) && \
+    !defined(RISCOS)
+#define Py_GCC_ATTRIBUTE(x)
+#else
+#define Py_GCC_ATTRIBUTE(x) __attribute__(x)
+#endif
+
+/*
+ * Specify alignment on compilers that support it.
+ */
+#if defined(__GNUC__) && __GNUC__ >= 3
+#define Py_ALIGNED(x) __attribute__((aligned(x)))
+#else
+#define Py_ALIGNED(x)
+#endif
+
+/*
+ * Older Microsoft compilers don't support the C99 long long literal suffixes,
+ * so these will be defined in PC/pyconfig.h for those compilers.
+ */
+#ifndef Py_LL
+#define Py_LL(x) x##LL
+#endif
+
+#ifndef Py_ULL
+#define Py_ULL(x) Py_LL(x##U)
+#endif
+
 #endif /* Py_PYPORT_H */
diff --git a/pypy/module/crypt/interp_crypt.py b/pypy/module/crypt/interp_crypt.py
--- a/pypy/module/crypt/interp_crypt.py
+++ b/pypy/module/crypt/interp_crypt.py
@@ -5,6 +5,9 @@
 
 if sys.platform.startswith('darwin'):
     eci = ExternalCompilationInfo()
+elif sys.platform.startswith('linux'):
+    # crypt() is defined only in crypt.h on some Linux variants (eg. Fedora 28)
+    eci = ExternalCompilationInfo(libraries=['crypt'], includes=["crypt.h"])
 else:
     eci = ExternalCompilationInfo(libraries=['crypt'])
 c_crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP,
diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -497,7 +497,7 @@
 def chain_from_iterable(space, w_cls, w_arg):
     """chain.from_iterable(iterable) --> chain object
 
-    Alternate chain() contructor taking a single iterable argument
+    Alternate chain() constructor taking a single iterable argument
     that evaluates lazily."""
     r = space.allocate_instance(W_Chain, w_cls)
     r.__init__(space, space.iter(w_arg))
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -828,7 +828,7 @@
         if (axis1 < 0 or axis2 < 0 or axis1 >= self.ndims() or
                 axis2 >= self.ndims()):
             raise oefmt(space.w_ValueError,
-                        "axis1(=%d) and axis2(=%d) must be withing range "
+                        "axis1(=%d) and axis2(=%d) must be within range "
                         "(ndim=%d)", axis1, axis2, self.ndims())
         if axis1 == axis2:
             raise oefmt(space.w_ValueError,
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
@@ -62,7 +62,7 @@
             assert t(h).value == h
 
     def test_typeerror(self):
-        # Only numbers are allowed in the contructor,
+        # Only numbers are allowed in the constructor,
         # otherwise TypeError is raised
         for t in signed_types + unsigned_types + float_types:
             with pytest.raises(TypeError):
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py
@@ -272,3 +272,18 @@
         base = cast(d, c_void_p).value
         for i in [0, 1, 4, 1444, -10293]:
             assert cast(byref(c, i), c_void_p).value == base + i
+
+    def test_issue2813_fix(self):
+        class C(Structure):
+            pass
+        POINTER(C)
+        C._fields_ = [('x', c_int)]
+        ffitype = C.get_ffi_argtype()
+        assert C.get_ffi_argtype() is ffitype
+        assert ffitype.sizeof() == sizeof(c_int)
+
+    def test_issue2813_cant_change_fields_after_get_ffi_argtype(self):
+        class C(Structure):
+            pass
+        ffitype = C.get_ffi_argtype()
+        raises(NotImplementedError, "C._fields_ = [('x', c_int)]")
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -63,7 +63,7 @@
     def remember_known_length(self, op, val):
         self._known_lengths[op] = val
 
-    def remember_setarrayitem_occured(self, op, index):
+    def remember_setarrayitem_occurred(self, op, index):
         op = self.get_box_replacement(op)
         try:
             subs = self._setarrayitems_occurred[op]
@@ -456,7 +456,7 @@
         array_box = op.getarg(0)
         index_box = op.getarg(1)
         if not isinstance(array_box, ConstPtr) and index_box.is_constant():
-            self.remember_setarrayitem_occured(array_box, index_box.getint())
+            self.remember_setarrayitem_occurred(array_box, index_box.getint())
 
     def clear_varsize_gc_fields(self, kind, descr, result, v_length, opnum):
         if self.gc_ll_descr.malloc_zero_filled:
diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py
--- a/rpython/jit/backend/zarch/opassembler.py
+++ b/rpython/jit/backend/zarch/opassembler.py
@@ -74,7 +74,7 @@
         mc.MLGR(lr, l1)
         mc.LGHI(r.SCRATCH, l.imm(-1))
         mc.RISBG(r.SCRATCH, r.SCRATCH, l.imm(0), l.imm(0x80 | 0), l.imm(0))
-        # is the value greater than 2**63 ? then an overflow occured
+        # is the value greater than 2**63 ? then an overflow occurred
         jmp_xor_lq_overflow = mc.get_relative_pos()
         mc.reserve_cond_jump() # CLGRJ lq > 0x8000 ... 00 -> (label_overflow)
         jmp_xor_lr_overflow = mc.get_relative_pos()
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -260,6 +260,12 @@
             # we don't know about this item
             return
         op = optimizer.get_box_replacement(self._fields[fielddescr.get_index()])
+        if op is None:
+            # XXX same bug as in serialize_opt:
+            # op should never be None, because that's an invariant violation in
+            # AbstractCachedEntry. But it still seems to happen when the info
+            # is attached to a Constant. At least we shouldn't crash.
+            return
         opnum = OpHelpers.getfield_for_descr(fielddescr)
         getfield_op = ResOperation(opnum, [structbox], descr=fielddescr)
         shortboxes.add_heap_op(op, getfield_op)
@@ -589,6 +595,7 @@
             return
         item = self._items[index]
         if item is not None:
+            # see comment in AbstractStructPtrInfo.produce_short_preamble_ops
             op = optimizer.get_box_replacement(item)
             opnum = OpHelpers.getarrayitem_for_descr(descr)
             getarrayitem_op = ResOperation(opnum, [structbox, ConstInt(index)],
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -550,7 +550,7 @@
         self.family = family
         self.type = type
         self.proto = proto
-        self.timeout = defaults.timeout
+        self.settimeout(defaults.timeout)
 
     @staticmethod
     def empty_rsocket():
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -465,6 +465,15 @@
     s.connect(INETAddress('python.org', 80))
     s.close()
 
+def test_connect_with_default_timeout_fail():
+    rsocket.setdefaulttimeout(0.1)
+    s = RSocket()
+    rsocket.setdefaulttimeout(None)
+    assert s.gettimeout() == 0.1
+    with py.test.raises(SocketTimeout):
+        s.connect(INETAddress('172.30.172.30', 12345))
+    s.close()
+
 def test_getsetsockopt():
     import struct
     assert struct.calcsize("i") == rffi.sizeof(rffi.INT)
diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -999,7 +999,7 @@
                 classdef.has_no_attrs()):
                 # special case for instanciating simple built-in
                 # exceptions: always return the same prebuilt instance,
-                # and ignore any arguments passed to the contructor.
+                # and ignore any arguments passed to the constructor.
                 r_instance = rclass.getinstancerepr(hop.rtyper, classdef)
                 example = r_instance.get_reusable_prebuilt_instance()
                 hop.exception_cannot_occur()
diff --git a/rpython/translator/c/extfunc.py b/rpython/translator/c/extfunc.py
--- a/rpython/translator/c/extfunc.py
+++ b/rpython/translator/c/extfunc.py
@@ -17,7 +17,7 @@
     yield ('RPYTHON_EXCEPTION_MATCH',  exceptiondata.fn_exception_match)
     yield ('RPYTHON_TYPE_OF_EXC_INST', exceptiondata.fn_type_of_exc_inst)
 
-    yield ('RPyExceptionOccurred1',    exctransformer.rpyexc_occured_ptr.value)
+    yield ('RPyExceptionOccurred1',    exctransformer.rpyexc_occurred_ptr.value)
     yield ('RPyFetchExceptionType',    exctransformer.rpyexc_fetch_type_ptr.value)
     yield ('RPyFetchExceptionValue',   exctransformer.rpyexc_fetch_value_ptr.value)
     yield ('RPyClearException',        exctransformer.rpyexc_clear_ptr.value)
diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py
--- a/rpython/translator/exceptiontransform.py
+++ b/rpython/translator/exceptiontransform.py
@@ -66,7 +66,7 @@
             assertion_error_ll_exc_type)
         self.c_n_i_error_ll_exc_type = constant_value(n_i_error_ll_exc_type)
 
-        def rpyexc_occured():
+        def rpyexc_occurred():
             exc_type = exc_data.exc_type
             return bool(exc_type)
 
@@ -109,9 +109,9 @@
                 exc_data.exc_type = ll_inst_type(evalue)
                 exc_data.exc_value = evalue
 
-        self.rpyexc_occured_ptr = self.build_func(
+        self.rpyexc_occurred_ptr = self.build_func(
             "RPyExceptionOccurred",
-            rpyexc_occured,
+            rpyexc_occurred,
             [], lltype.Bool)
 
         self.rpyexc_fetch_type_ptr = self.build_func(

From pypy.commits at gmail.com  Mon May  7 22:42:55 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 07 May 2018 19:42:55 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: do not pull in the backend
 until the very last moment to prevent errors when importing _cppyy for doc
 purposes
Message-ID: <5af10eaf.43581c0a.b63f8.2dc8@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94498:4600e4a5a904
Date: 2018-05-07 19:25 -0700
http://bitbucket.org/pypy/pypy/changeset/4600e4a5a904/

Log:	do not pull in the backend until the very last moment to prevent
	errors when importing _cppyy for doc purposes

diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py
--- a/pypy/module/_cppyy/__init__.py
+++ b/pypy/module/_cppyy/__init__.py
@@ -22,7 +22,7 @@
     }
 
     appleveldefs = {
-        '_init_pythonify'        : 'pythonify._init_pythonify',
+        '_post_import_startup'   : 'pythonify._post_import_startup',
         'add_pythonization'      : 'pythonify.add_pythonization',
         'Template'               : 'pythonify.CPPTemplate',
     }
@@ -35,9 +35,3 @@
         # code generation is not, so give it a chance to run now
         from pypy.module._cppyy import capi
         capi.register_pythonizations(space)
-
-    def startup(self, space):
-        from pypy.module._cppyy import capi
-        capi.verify_backend(space)      # may raise ImportError
-
-        space.call_method(self, '_init_pythonify')
diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -308,7 +308,7 @@
         c_call = state.capi_calls[name]
     except KeyError:
         if state.backend is None:
-            load_backend(space)
+            verify_backend(space)
         iface = state.capi_call_ifaces[name]
         cfunc = W_RCTypeFunc(space, iface[0], iface[1], False)
         c_call = state.backend.load_function(cfunc, 'cppyy_'+name)
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -1,5 +1,5 @@
 # NOT_RPYTHON
-# do not load _cppyy here, see _init_pythonify()
+# do not load _cppyy here, see _post_import_startup()
 import types
 import sys
 
@@ -22,7 +22,7 @@
 class CPPClass(CPPScope):
     pass
 
-# namespace base class (class base class defined in _init_pythonify)
+# namespace base class (class base class defined in _post_import_startup()
 class CPPNamespace(object):
     __metatype__ = CPPMetaNamespace
 
@@ -407,7 +407,7 @@
         pyclass.__len__     = return2
 
 
-def _init_pythonify():
+def _post_import_startup():
     # _cppyy should not be loaded at the module level, as that will trigger a
     # call to space.getbuiltinmodule(), which will cause _cppyy to be loaded
     # at pypy-c startup, rather than on the "import _cppyy" statement
diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py
--- a/pypy/module/_cppyy/test/test_advancedcpp.py
+++ b/pypy/module/_cppyy/test/test_advancedcpp.py
@@ -22,7 +22,8 @@
     def setup_class(cls):
         cls.w_test_dct = cls.space.newtext(test_dct)
         cls.w_advanced = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_default_arguments(self):
diff --git a/pypy/module/_cppyy/test/test_cpp11features.py b/pypy/module/_cppyy/test/test_cpp11features.py
--- a/pypy/module/_cppyy/test/test_cpp11features.py
+++ b/pypy/module/_cppyy/test/test_cpp11features.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_shared_ptr(self):
diff --git a/pypy/module/_cppyy/test/test_cppyy.py b/pypy/module/_cppyy/test/test_cppyy.py
--- a/pypy/module/_cppyy/test/test_cppyy.py
+++ b/pypy/module/_cppyy/test/test_cppyy.py
@@ -33,6 +33,7 @@
         cls.w_lib, cls.w_instantiate, cls.w_example01, cls.w_payload = \
                    cls.space.unpackiterable(cls.space.appexec([], """():
             import _cppyy, ctypes
+            _cppyy._post_import_startup()
             lib = ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)
             def cpp_instantiate(tt, *args):
                 inst = _cppyy._bind_object(0, tt, True)
diff --git a/pypy/module/_cppyy/test/test_crossing.py b/pypy/module/_cppyy/test/test_crossing.py
--- a/pypy/module/_cppyy/test/test_crossing.py
+++ b/pypy/module/_cppyy/test/test_crossing.py
@@ -72,7 +72,9 @@
         # to allow the generated extension module be loaded first)
         cls.w_test_dct    = cls.space.newtext(test_dct)
         cls.w_pre_imports = cls.space.appexec([], """():
-            import ctypes, _cppyy""")   # prevents leak-checking complaints on ctypes' statics
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()""")   # early import of ctypes
+                  # prevents leak-checking complaints on ctypes' statics
 
     def setup_method(self, func):
         @unwrap_spec(name='text', init='text', body='text')
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_datatypes = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
         cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
 
diff --git a/pypy/module/_cppyy/test/test_fragile.py b/pypy/module/_cppyy/test/test_fragile.py
--- a/pypy/module/_cppyy/test/test_fragile.py
+++ b/pypy/module/_cppyy/test/test_fragile.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_fragile = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_missing_classes(self):
diff --git a/pypy/module/_cppyy/test/test_operators.py b/pypy/module/_cppyy/test/test_operators.py
--- a/pypy/module/_cppyy/test/test_operators.py
+++ b/pypy/module/_cppyy/test/test_operators.py
@@ -15,7 +15,8 @@
         cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_operators = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def teardown_method(self, meth):
diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py
--- a/pypy/module/_cppyy/test/test_overloads.py
+++ b/pypy/module/_cppyy/test/test_overloads.py
@@ -17,7 +17,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_overloads = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_class_based_overloads(self):
diff --git a/pypy/module/_cppyy/test/test_pythonify.py b/pypy/module/_cppyy/test/test_pythonify.py
--- a/pypy/module/_cppyy/test/test_pythonify.py
+++ b/pypy/module/_cppyy/test/test_pythonify.py
@@ -16,7 +16,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_finding_classes(self):
diff --git a/pypy/module/_cppyy/test/test_stltypes.py b/pypy/module/_cppyy/test/test_stltypes.py
--- a/pypy/module/_cppyy/test/test_stltypes.py
+++ b/pypy/module/_cppyy/test/test_stltypes.py
@@ -15,7 +15,8 @@
         cls.w_N = cls.space.newint(13)
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlvector = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_builtin_type_vector_types(self):
diff --git a/pypy/module/_cppyy/test/test_templates.py b/pypy/module/_cppyy/test/test_templates.py
--- a/pypy/module/_cppyy/test/test_templates.py
+++ b/pypy/module/_cppyy/test/test_templates.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_datatypes = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_template_member_functions(self):

From pypy.commits at gmail.com  Tue May  8 00:20:05 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 07 May 2018 21:20:05 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merge of cppyy-packaging: pulls in
 fully deferred loading for the backend (needed for pydoc tests in p3.5
 branch), support for function pointer arguments, proper types for
Message-ID: <5af12575.1c69fb81.5e601.0ce6@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r94499:a4c2916c877f
Date: 2018-05-07 21:02 -0700
http://bitbucket.org/pypy/pypy/changeset/a4c2916c877f/

Log:	Merge of cppyy-packaging: pulls in fully deferred loading for the
	backend (needed for pydoc tests in p3.5 branch), support for
	function pointer arguments, proper types for anonymous enums, and
	correct handling of const data.

diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py
--- a/pypy/module/_cppyy/__init__.py
+++ b/pypy/module/_cppyy/__init__.py
@@ -7,6 +7,7 @@
     interpleveldefs = {
         '_resolve_name'          : 'interp_cppyy.resolve_name',
         '_scope_byname'          : 'interp_cppyy.scope_byname',
+        '_is_static_data'        : 'interp_cppyy.is_static_data',
         '_is_template'           : 'interp_cppyy.is_template',
         '_std_string_name'       : 'interp_cppyy.std_string_name',
         '_set_class_generator'   : 'interp_cppyy.set_class_generator',
@@ -21,7 +22,7 @@
     }
 
     appleveldefs = {
-        '_init_pythonify'        : 'pythonify._init_pythonify',
+        '_post_import_startup'   : 'pythonify._post_import_startup',
         'add_pythonization'      : 'pythonify.add_pythonization',
         'Template'               : 'pythonify.CPPTemplate',
     }
@@ -34,9 +35,3 @@
         # code generation is not, so give it a chance to run now
         from pypy.module._cppyy import capi
         capi.register_pythonizations(space)
-
-    def startup(self, space):
-        from pypy.module._cppyy import capi
-        capi.verify_backend(space)      # may raise ImportError
-
-        space.call_method(self, '_init_pythonify')
diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -308,7 +308,7 @@
         c_call = state.capi_calls[name]
     except KeyError:
         if state.backend is None:
-            load_backend(space)
+            verify_backend(space)
         iface = state.capi_call_ifaces[name]
         cfunc = W_RCTypeFunc(space, iface[0], iface[1], False)
         c_call = state.backend.load_function(cfunc, 'cppyy_'+name)
@@ -421,7 +421,7 @@
         _cdata_to_ptr(space, call_capi(space, 'function_address_from_index', args)))
 def c_function_address_from_method(space, cppmethod):
     return rffi.cast(C_FUNC_PTR,
-        _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', _ArgH(cppmethod))))
+        _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', [_ArgH(cppmethod)])))
 
 # handling of function argument buffer ---------------------------------------
 def c_allocate_function_args(space, size):
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -686,6 +686,34 @@
         decref(space, rffi.cast(PyObject, rffi.cast(rffi.VOIDPP, arg)[0]))
 
 
+class FunctionPointerConverter(TypeConverter):
+    _immutable_fields_ = ['signature']
+
+    def __init__(self, space, signature):
+        self.signature = signature
+
+    def convert_argument(self, space, w_obj, address, call_local):
+        # TODO: atm, does not actually get an overload, but a staticmethod
+        from pypy.module._cppyy.interp_cppyy import W_CPPOverload
+        cppol = space.interp_w(W_CPPOverload, w_obj)
+
+        # find the function with matching signature
+        for i in range(len(cppol.functions)):
+            m = cppol.functions[i]
+            if m.signature(False) == self.signature:
+                x = rffi.cast(rffi.VOIDPP, address)
+                x[0] = rffi.cast(rffi.VOIDP,
+                    capi.c_function_address_from_method(space, m.cppmethod))
+                address = rffi.cast(capi.C_OBJECT, address)
+                ba = rffi.cast(rffi.CCHARP, address)
+                ba[capi.c_function_arg_typeoffset(space)] = 'p'
+                return
+
+        # lookup failed
+        raise oefmt(space.w_TypeError,
+                    "no overload found matching %s", self.signature)
+
+
 class MacroConverter(TypeConverter):
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         # TODO: get the actual type info from somewhere ...
@@ -749,6 +777,14 @@
             return InstancePtrPtrConverter(space, clsdecl)
         elif compound == "":
             return InstanceConverter(space, clsdecl)
+    elif "(anonymous)" in name:
+        # special case: enum w/o a type name
+        return _converters["internal_enum_type_t"](space, default)
+    elif "(*)" in name or "::*)" in name:
+        # function pointer
+        pos = name.find("*)")
+        if pos > 0:
+            return FunctionPointerConverter(space, name[pos+2:])
 
     #   5) void* or void converter (which fails on use)
     if 0 <= compound.find('*'):
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -289,6 +289,9 @@
             return InstancePtrExecutor(space, cppclass)
         elif compound == '**' or compound == '*&':
             return InstancePtrPtrExecutor(space, cppclass)
+    elif "(anonymous)" in name:
+        # special case: enum w/o a type name
+        return _executors["internal_enum_type_t"](space, None)
 
     # 4) additional special cases
     if compound == '*':
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -128,7 +128,7 @@
 
 def register_class(space, w_pycppclass):
     w_cppclass = space.findattr(w_pycppclass, space.newtext("__cppdecl__"))
-    cppclass = space.interp_w(W_CPPClassDecl, w_cppclass, can_be_None=False)
+    cppclass = space.interp_w(W_CPPClassDecl, w_cppclass)
     # add back-end specific method pythonizations (doing this on the wrapped
     # class allows simple aliasing of methods)
     capi.pythonize(space, cppclass.name, w_pycppclass)
@@ -149,6 +149,24 @@
 W_CPPLibrary.typedef.acceptable_as_base_class = True
 
 
+#-----
+# Classes involved with methods and functions:
+#
+#  CPPMethod:         base class wrapping a single function or method
+#  CPPConstructor:    specialization for allocating a new object
+#  CPPFunction:       specialization for free and static functions
+#  CPPSetItem:        specialization for Python's __setitem__
+#  CPPTemplatedCall:  trampoline to instantiate and bind templated functions
+#  W_CPPOverload, W_CPPConstructorOverload, W_CPPTemplateOverload:
+#     user-facing, app-level, collection of overloads, with specializations
+#     for constructors and templates
+#  W_CPPBoundMethod:  instantiated template method
+#
+# All methods/functions derive from CPPMethod and are collected as overload
+# candidates in user-facing overload classes. Templated methods are a two-step
+# process, where first the template is instantiated (or selected if already
+# available), which returns a callable object that is the actual bound method.
+
 class CPPMethod(object):
     """Dispatcher of methods. Checks the arguments, find the corresponding FFI
     function if available, makes the call, and returns the wrapped result. It
@@ -177,7 +195,7 @@
 
     @staticmethod
     def unpack_cppthis(space, w_cppinstance, declaring_scope):
-        cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False)
+        cppinstance = space.interp_w(W_CPPInstance, w_cppinstance)
         cppinstance._nullcheck()
         return cppinstance.get_cppthis(declaring_scope)
 
@@ -424,7 +442,7 @@
 
 
 class CPPFunction(CPPMethod):
-    """Global (namespaced) function dispatcher."""
+    """Global (namespaced) / static function dispatcher."""
 
     _immutable_ = True
 
@@ -688,6 +706,18 @@
 )
 
 
+#-----
+# Classes for data members:
+#
+#  W_CPPDataMember:        instance data members
+#  W_CPPConstDataMember:   specialization for const data members
+#  W_CPPStaticData:        class-level and global/static data
+#  W_CPPConstStaticData:   specialization for const global/static data
+#
+# Data is represented by an offset which is either a global pointer (static data)
+# or an offset from the start of an instance (data members). The "const"
+# specializations raise when attempting to set their value.
+
 class W_CPPDataMember(W_Root):
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
@@ -698,9 +728,6 @@
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
 
-    def is_static(self):
-        return self.space.w_False
-
     def _get_offset(self, cppinstance):
         if cppinstance:
             assert lltype.typeOf(cppinstance.clsdecl.handle) == lltype.typeOf(self.scope.handle)
@@ -728,16 +755,25 @@
 
 W_CPPDataMember.typedef = TypeDef(
     'CPPDataMember',
-    is_static = interp2app(W_CPPDataMember.is_static),
     __get__ = interp2app(W_CPPDataMember.get),
     __set__ = interp2app(W_CPPDataMember.set),
 )
 W_CPPDataMember.typedef.acceptable_as_base_class = False
 
+
+class W_CPPConstDataMember(W_CPPDataMember):
+    def set(self, w_cppinstance, w_value):
+        raise oefmt(self.space.w_TypeError, "assignment to const data not allowed")
+
+W_CPPConstDataMember.typedef = TypeDef(
+    'CPPConstDataMember',
+    __get__ = interp2app(W_CPPDataMember.get),
+    __set__ = interp2app(W_CPPConstDataMember.set),
+)
+W_CPPConstDataMember.typedef.acceptable_as_base_class = False
+
+
 class W_CPPStaticData(W_CPPDataMember):
-    def is_static(self):
-        return self.space.w_True
-
     @jit.elidable_promote()
     def _get_offset(self, cppinstance):
         return self.offset
@@ -751,19 +787,34 @@
 
 W_CPPStaticData.typedef = TypeDef(
     'CPPStaticData',
-    is_static = interp2app(W_CPPStaticData.is_static),
     __get__ = interp2app(W_CPPStaticData.get),
     __set__ = interp2app(W_CPPStaticData.set),
 )
 W_CPPStaticData.typedef.acceptable_as_base_class = False
 
-def is_static(space, w_obj):
+
+class W_CPPConstStaticData(W_CPPStaticData):
+    def set(self, w_cppinstance, w_value):
+        raise oefmt(self.space.w_TypeError, "assignment to const data not allowed")
+
+W_CPPConstStaticData.typedef = TypeDef(
+    'CPPConstStaticData',
+    __get__ = interp2app(W_CPPConstStaticData.get),
+    __set__ = interp2app(W_CPPConstStaticData.set),
+)
+W_CPPConstStaticData.typedef.acceptable_as_base_class = False
+
+
+def is_static_data(space, w_obj):
     try:
-        space.interp_w(W_CPPStaticData, w_obj, can_be_None=False)
+        space.interp_w(W_CPPStaticData, w_obj)
         return space.w_True
     except Exception:
         return space.w_False
 
+#-----
+
+
 class W_CPPScopeDecl(W_Root):
     _attrs_ = ['space', 'handle', 'name', 'methods', 'datamembers']
     _immutable_fields_ = ['handle', 'name']
@@ -847,7 +898,10 @@
         offset = capi.c_datamember_offset(self.space, self, dm_idx)
         if offset == -1:
             raise self.missing_attribute_error(dm_name)
-        datamember = W_CPPStaticData(self.space, self, type_name, offset)
+        if capi.c_is_const_data(self.space, self, dm_idx):
+            datamember = W_CPPConstStaticData(self.space, self, type_name, offset)
+        else:
+            datamember = W_CPPStaticData(self.space, self, type_name, offset)
         self.datamembers[dm_name] = datamember
         return datamember
 
@@ -967,8 +1021,13 @@
             if offset == -1:
                 continue      # dictionary problem; raises AttributeError on use
             is_static = bool(capi.c_is_staticdata(self.space, self, i))
-            if is_static:
+            is_const  = bool(capi.c_is_const_data(self.space, self, i))
+            if is_static and is_const:
+                datamember = W_CPPConstStaticData(self.space, self, type_name, offset)
+            elif is_static:
                 datamember = W_CPPStaticData(self.space, self, type_name, offset)
+            elif is_const:
+                datamember = W_CPPConstDataMember(self.space, self, type_name, offset)
             else:
                 datamember = W_CPPDataMember(self.space, self, type_name, offset)
             self.datamembers[datamember_name] = datamember
@@ -1124,7 +1183,7 @@
         # scopes of the argument classes (TODO: implement that last option)
         try:
             # TODO: expecting w_other to be an W_CPPInstance is too limiting
-            other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False)
+            other = self.space.interp_w(W_CPPInstance, w_other)
             for name in ["", "__gnu_cxx", "__1"]:
                 nss = scope_byname(self.space, name)
                 meth_idx = capi.c_get_global_operator(
@@ -1146,7 +1205,7 @@
 
         # fallback 2: direct pointer comparison (the class comparison is needed since
         # the first data member in a struct and the struct have the same address)
-        other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False)  # TODO: factor out
+        other = self.space.interp_w(W_CPPInstance, w_other)  # TODO: factor out
         iseq = (self._rawobject == other._rawobject) and (self.clsdecl == other.clsdecl)
         return self.space.newbool(iseq)
 
@@ -1265,7 +1324,7 @@
                 offset = capi.c_base_offset1(space, actual, clsdecl, rawobject, -1)
                 rawobject = capi.direct_ptradd(rawobject, offset)
                 w_cppdecl = space.findattr(w_pycppclass, space.newtext("__cppdecl__"))
-                clsdecl = space.interp_w(W_CPPClassDecl, w_cppdecl, can_be_None=False)
+                clsdecl = space.interp_w(W_CPPClassDecl, w_cppdecl)
             except Exception:
                 # failed to locate/build the derived class, so stick to the base (note
                 # that only get_pythonized_cppclass is expected to raise, so none of
@@ -1283,7 +1342,7 @@
 
     # fresh creation
     w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass)
-    cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False)
+    cppinstance = space.interp_w(W_CPPInstance, w_cppinstance)
     cppinstance.__init__(space, clsdecl, rawobject, is_ref, python_owns)
     memory_regulator.register(cppinstance)
     return w_cppinstance
@@ -1311,7 +1370,7 @@
     except Exception:
         # accept integer value as address
         rawobject = rffi.cast(capi.C_OBJECT, space.uint_w(w_obj))
-    decl = space.interp_w(W_CPPClassDecl, w_clsdecl, can_be_None=False)
+    decl = space.interp_w(W_CPPClassDecl, w_clsdecl)
     return wrap_cppinstance(space, rawobject, decl, python_owns=owns, do_cast=cast)
 
 @unwrap_spec(owns=bool, cast=bool)
@@ -1327,7 +1386,7 @@
 
 def move(space, w_obj):
     """Casts the given instance into an C++-style rvalue."""
-    obj = space.interp_w(W_CPPInstance, w_obj, can_be_None=True)
+    obj = space.interp_w(W_CPPInstance, w_obj)
     if obj:
         obj.flags |= INSTANCE_FLAGS_IS_R_VALUE
     return w_obj
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -1,5 +1,5 @@
 # NOT_RPYTHON
-# do not load _cppyy here, see _init_pythonify()
+# do not load _cppyy here, see _post_import_startup()
 import types
 import sys
 
@@ -22,7 +22,7 @@
 class CPPClass(CPPScope):
     pass
 
-# namespace base class (class base class defined in _init_pythonify)
+# namespace base class (class base class defined in _post_import_startup()
 class CPPNamespace(object):
     __metatype__ = CPPMetaNamespace
 
@@ -169,6 +169,7 @@
     return method
 
 def make_cppclass(scope, cl_name, decl):
+    import _cppyy
 
     # get a list of base classes for class creation
     bases = [get_pycppclass(base) for base in decl.get_base_names()]
@@ -209,7 +210,7 @@
     for d_name in decl.get_datamember_names():
         cppdm = decl.get_datamember(d_name)
         d_class[d_name] = cppdm
-        if cppdm.is_static():
+        if _cppyy._is_static_data(cppdm):
             d_meta[d_name] = cppdm
 
     # create a metaclass to allow properties (for static data write access)
@@ -278,7 +279,7 @@
         try:
             cppdm = scope.__cppdecl__.get_datamember(name)
             setattr(scope, name, cppdm)
-            if cppdm.is_static():
+            if _cppyy._is_static_data(cppdm):
                 setattr(scope.__class__, name, cppdm)
             pycppitem = getattr(scope, name)      # gets actual property value
         except AttributeError:
@@ -406,7 +407,7 @@
         pyclass.__len__     = return2
 
 
-def _init_pythonify():
+def _post_import_startup():
     # _cppyy should not be loaded at the module level, as that will trigger a
     # call to space.getbuiltinmodule(), which will cause _cppyy to be loaded
     # at pypy-c startup, rather than on the "import _cppyy" statement
diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx
--- a/pypy/module/_cppyy/src/dummy_backend.cxx
+++ b/pypy/module/_cppyy/src/dummy_backend.cxx
@@ -348,6 +348,7 @@
         PUBLIC_CPPYY_DATA3(short,   short,              h);
         PUBLIC_CPPYY_DATA3(ushort,  unsigned short,     H);
         PUBLIC_CPPYY_DATA3(int,     int,                i);
+        PUBLIC_CPPYY_DATA (const_int, const int);
         PUBLIC_CPPYY_DATA3(uint,    unsigned int,       I);
         PUBLIC_CPPYY_DATA3(long,    long,               l);
         PUBLIC_CPPYY_DATA3(ulong,   unsigned long,      L);
@@ -1032,7 +1033,9 @@
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
 
-int cppyy_is_const_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) {
+int cppyy_is_const_data(cppyy_scope_t handle, cppyy_index_t idatambr) {
+    if (s_scopes[handle].m_datambrs[idatambr].m_name == "m_const_int")
+        return 1;
     return 0;
 }
 
diff --git a/pypy/module/_cppyy/test/datatypes.cxx b/pypy/module/_cppyy/test/datatypes.cxx
--- a/pypy/module/_cppyy/test/datatypes.cxx
+++ b/pypy/module/_cppyy/test/datatypes.cxx
@@ -6,7 +6,7 @@
 
 
 //===========================================================================
-CppyyTestData::CppyyTestData() : m_owns_arrays(false)
+CppyyTestData::CppyyTestData() : m_const_int(17), m_owns_arrays(false)
 {
     m_bool    = false;
     m_char    = 'a';
@@ -333,3 +333,17 @@
 CppyyTestPod* get_null_pod() {
     return (CppyyTestPod*)0;
 }
+
+
+//= function pointer passing ================================================
+int sum_of_int(int i1, int i2) {
+    return i1+i2;
+}
+
+double sum_of_double(double d1, double d2) {
+    return d1+d2;
+}
+
+double call_double_double(double (*d)(double, double), double d1, double d2) {
+    return d(d1, d2);
+}
diff --git a/pypy/module/_cppyy/test/datatypes.h b/pypy/module/_cppyy/test/datatypes.h
--- a/pypy/module/_cppyy/test/datatypes.h
+++ b/pypy/module/_cppyy/test/datatypes.h
@@ -1,5 +1,5 @@
 // copied from RtypesCore.h ...
-#if defined(R__WIN32)
+#if defined(R__WIN32) && !defined(__CINT__)
 typedef __int64          Long64_t;  //Portable signed long integer 8 bytes
 typedef unsigned __int64 ULong64_t; //Portable unsigned long integer 8 bytes
 #else
@@ -26,8 +26,13 @@
 
 //===========================================================================
 namespace EnumSpace {
-   enum E {E1 = 1, E2};
-};
+    enum E {E1 = 1, E2};
+    class EnumClass {
+    public:
+        enum    {E1 = -1};
+        enum EE {E2 = -1};
+    };
+}
 
 
 //===========================================================================
@@ -243,6 +248,7 @@
     short                m_short;
     unsigned short       m_ushort;
     int                  m_int;
+    const int            m_const_int;   // special case: const testing
     unsigned int         m_uint;
     long                 m_long;
     unsigned long        m_ulong;
@@ -364,3 +370,9 @@
 void set_global_pod(CppyyTestPod* t);
 CppyyTestPod* get_global_pod();
 CppyyTestPod* get_null_pod();
+
+
+//= function pointer passing ================================================
+int sum_of_int(int i1, int i2);
+double sum_of_double(double d1, double d2);
+double call_double_double(double (*d)(double, double), double d1, double d2);
diff --git a/pypy/module/_cppyy/test/datatypes.xml b/pypy/module/_cppyy/test/datatypes.xml
--- a/pypy/module/_cppyy/test/datatypes.xml
+++ b/pypy/module/_cppyy/test/datatypes.xml
@@ -4,6 +4,8 @@
   <class name="FourVector" />
 
   <enum name="EFruit" />
+  <enum name="EnumSpace::E" />
+  <class name="EnumSpace::EnumClass" />
 
   <function pattern="get_*" />
   <function pattern="set_*" />
@@ -14,4 +16,8 @@
   <variable name="g_int" />
   <variable name="g_pod" />
 
+  <function name="sum_of_int" />
+  <function name="sum_of_double" />
+  <function name="call_double_double" />
+
 </lcgdict>
diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py
--- a/pypy/module/_cppyy/test/test_advancedcpp.py
+++ b/pypy/module/_cppyy/test/test_advancedcpp.py
@@ -22,7 +22,8 @@
     def setup_class(cls):
         cls.w_test_dct = cls.space.newtext(test_dct)
         cls.w_advanced = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_default_arguments(self):
diff --git a/pypy/module/_cppyy/test/test_cpp11features.py b/pypy/module/_cppyy/test/test_cpp11features.py
--- a/pypy/module/_cppyy/test/test_cpp11features.py
+++ b/pypy/module/_cppyy/test/test_cpp11features.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_shared_ptr(self):
diff --git a/pypy/module/_cppyy/test/test_cppyy.py b/pypy/module/_cppyy/test/test_cppyy.py
--- a/pypy/module/_cppyy/test/test_cppyy.py
+++ b/pypy/module/_cppyy/test/test_cppyy.py
@@ -33,6 +33,7 @@
         cls.w_lib, cls.w_instantiate, cls.w_example01, cls.w_payload = \
                    cls.space.unpackiterable(cls.space.appexec([], """():
             import _cppyy, ctypes
+            _cppyy._post_import_startup()
             lib = ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)
             def cpp_instantiate(tt, *args):
                 inst = _cppyy._bind_object(0, tt, True)
diff --git a/pypy/module/_cppyy/test/test_crossing.py b/pypy/module/_cppyy/test/test_crossing.py
--- a/pypy/module/_cppyy/test/test_crossing.py
+++ b/pypy/module/_cppyy/test/test_crossing.py
@@ -72,7 +72,9 @@
         # to allow the generated extension module be loaded first)
         cls.w_test_dct    = cls.space.newtext(test_dct)
         cls.w_pre_imports = cls.space.appexec([], """():
-            import ctypes, _cppyy""")   # prevents leak-checking complaints on ctypes' statics
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()""")   # early import of ctypes
+                  # prevents leak-checking complaints on ctypes' statics
 
     def setup_method(self, func):
         @unwrap_spec(name='text', init='text', body='text')
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_datatypes = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
         cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
 
@@ -191,6 +192,10 @@
             for i in range(self.N):
                 assert eval('c.m_%s_array2[i]' % names[j]) == b[i]
 
+        # can not write to constant data
+        assert c.m_const_int == 17
+        raises(TypeError, setattr, c, 'm_const_int', 71)
+
         c.__destruct__()
 
     def test03_array_passing(self):
@@ -464,6 +469,10 @@
         assert gbl.kBanana == 29
         assert gbl.kCitrus == 34
 
+        assert gbl.EnumSpace.E
+        assert gbl.EnumSpace.EnumClass.E1 == -1   # anonymous
+        assert gbl.EnumSpace.EnumClass.E2 == -1   # named type
+
     def test11_string_passing(self):
         """Test passing/returning of a const char*"""
 
@@ -741,3 +750,22 @@
 
         c.s_voidp = c2
         address_equality_test(c.s_voidp, c2)
+
+    def test21_function_pointers(self):
+        """Function pointer passing"""
+
+        import _cppyy as cppyy
+
+        f1 = cppyy.gbl.sum_of_int
+        f2 = cppyy.gbl.sum_of_double
+        f3 = cppyy.gbl.call_double_double
+
+        assert 5 == f1(2, 3)
+        assert 5. == f2(5., 0.)
+
+        raises(TypeError, f3, f1, 2, 3)
+
+        # TODO: get straightforward access to the overload type
+        f2 = cppyy.gbl.__cppdecl__.get_overload('sum_of_double')
+
+        assert 5. == f3(f2, 5., 0.)
diff --git a/pypy/module/_cppyy/test/test_fragile.py b/pypy/module/_cppyy/test/test_fragile.py
--- a/pypy/module/_cppyy/test/test_fragile.py
+++ b/pypy/module/_cppyy/test/test_fragile.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_fragile = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_missing_classes(self):
diff --git a/pypy/module/_cppyy/test/test_operators.py b/pypy/module/_cppyy/test/test_operators.py
--- a/pypy/module/_cppyy/test/test_operators.py
+++ b/pypy/module/_cppyy/test/test_operators.py
@@ -15,7 +15,8 @@
         cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_operators = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def teardown_method(self, meth):
diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py
--- a/pypy/module/_cppyy/test/test_overloads.py
+++ b/pypy/module/_cppyy/test/test_overloads.py
@@ -17,7 +17,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_overloads = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_class_based_overloads(self):
diff --git a/pypy/module/_cppyy/test/test_pythonify.py b/pypy/module/_cppyy/test/test_pythonify.py
--- a/pypy/module/_cppyy/test/test_pythonify.py
+++ b/pypy/module/_cppyy/test/test_pythonify.py
@@ -16,7 +16,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_finding_classes(self):
diff --git a/pypy/module/_cppyy/test/test_stltypes.py b/pypy/module/_cppyy/test/test_stltypes.py
--- a/pypy/module/_cppyy/test/test_stltypes.py
+++ b/pypy/module/_cppyy/test/test_stltypes.py
@@ -15,7 +15,8 @@
         cls.w_N = cls.space.newint(13)
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlvector = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_builtin_type_vector_types(self):
diff --git a/pypy/module/_cppyy/test/test_templates.py b/pypy/module/_cppyy/test/test_templates.py
--- a/pypy/module/_cppyy/test/test_templates.py
+++ b/pypy/module/_cppyy/test/test_templates.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_datatypes = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_template_member_functions(self):

From pypy.commits at gmail.com  Tue May  8 00:25:56 2018
From: pypy.commits at gmail.com (wlav)
Date: Mon, 07 May 2018 21:25:56 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: fix typo in comment
Message-ID: <5af126d4.1c69fb81.66ab7.a88f@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94500:49f83bfa0c9e
Date: 2018-05-07 21:08 -0700
http://bitbucket.org/pypy/pypy/changeset/49f83bfa0c9e/

Log:	fix typo in comment

diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py
--- a/pypy/module/_cppyy/__init__.py
+++ b/pypy/module/_cppyy/__init__.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
-    "This module brigdes the cppyy frontend with its backend, through PyPy.\n\
+    "This module bridges the cppyy frontend with its backend, through PyPy.\n\
     See http://cppyy.readthedocs.io/en/latest for full details."
 
     interpleveldefs = {

From pypy.commits at gmail.com  Tue May  8 11:13:48 2018
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 08 May 2018 08:13:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3tests: Fix translation
Message-ID: <5af1beac.1c69fb81.50294.0cca@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3tests
Changeset: r94501:a79923d36cf1
Date: 2018-05-08 16:12 +0100
http://bitbucket.org/pypy/pypy/changeset/a79923d36cf1/

Log:	Fix translation

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -341,10 +341,10 @@
         translate.log_config(config.objspace, "PyPy config object")
 
         # obscure hack to stuff the translation options into the translated PyPy
-        import pypy.module.sys
+        from pypy.module.sys.moduledef import Module as SysModule
         options = make_dict(config)
-        wrapstr = 'space.wrap(%r)' % (options) # import time
-        pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
+        wrapstr = 'space.wrap(%r)' % (options)  # import time
+        SysModule.interpleveldefs['pypy_translation_info'] = wrapstr
         if config.objspace.usemodules._cffi_backend:
             self.hack_for_cffi_modules(driver)
 
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -189,7 +189,7 @@
 
     def extra_interpdef(self, name, spec):
         cls = self.__class__
-        pkgroot = cls.__module__
+        pkgroot = cls.__module__.rsplit('.', 1)[0]
         loader = getinterpevalloader(pkgroot, spec)
         space = self.space
         w_obj = loader(space)
diff --git a/pypy/module/_rawffi/alt/__init__.py b/pypy/module/_rawffi/alt/__init__.py
--- a/pypy/module/_rawffi/alt/__init__.py
+++ b/pypy/module/_rawffi/alt/__init__.py
@@ -1,18 +0,0 @@
-from pypy.interpreter.mixedmodule import MixedModule
-import os
-
-class Module(MixedModule):
-
-    interpleveldefs = {
-        'types':   'interp_ffitype.W_types',
-        'CDLL':    'interp_funcptr.W_CDLL',
-        'FuncPtr': 'interp_funcptr.W_FuncPtr',
-        'get_libc':'interp_funcptr.get_libc',
-        '_StructDescr': 'interp_struct.W__StructDescr',
-        'Field':     'interp_struct.W_Field',
-    }
-    if os.name == 'nt':
-        interpleveldefs['WinDLL'] = 'interp_funcptr.W_WinDLL'
-    appleveldefs = {
-        'Structure': 'app_struct.Structure',
-        }
diff --git a/pypy/module/_rawffi/alt/__init__.py b/pypy/module/_rawffi/alt/moduledef.py
copy from pypy/module/_rawffi/alt/__init__.py
copy to pypy/module/_rawffi/alt/moduledef.py
diff --git a/pypy/module/_rawffi/moduledef.py b/pypy/module/_rawffi/moduledef.py
--- a/pypy/module/_rawffi/moduledef.py
+++ b/pypy/module/_rawffi/moduledef.py
@@ -2,7 +2,7 @@
 """
 
 from pypy.interpreter.mixedmodule import MixedModule
-from pypy.module._rawffi import alt
+from pypy.module._rawffi.alt.moduledef import Module as AltModule
 
 class Module(MixedModule):
     interpleveldefs = {
@@ -36,7 +36,7 @@
     }
 
     submodules = {
-        'alt': alt.Module,
+        'alt': AltModule,
     }
 
     def buildloaders(cls):
diff --git a/pypy/module/cpyext/pythonrun.py b/pypy/module/cpyext/pythonrun.py
--- a/pypy/module/cpyext/pythonrun.py
+++ b/pypy/module/cpyext/pythonrun.py
@@ -40,9 +40,9 @@
     called at most once.  Since Python's internal finalization will have
     completed before the cleanup function, no Python APIs should be called by
     func."""
-    from pypy.module import cpyext
+    from pypy.module.cpyext.moduledef import Module
     w_module = space.getbuiltinmodule('cpyext')
-    module = space.interp_w(cpyext.Module, w_module)
+    module = space.interp_w(Module, w_module)
     try:
         module.register_atexit(func_ptr)
     except ValueError:

From pypy.commits at gmail.com  Wed May  9 12:26:32 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 May 2018 09:26:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add a failing test from issue #2826. Add
 also a hypothesis test
Message-ID: <5af32138.1c69fb81.ab820.3b62@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94502:bf0a8b696526
Date: 2018-05-09 18:13 +0200
http://bitbucket.org/pypy/pypy/changeset/bf0a8b696526/

Log:	Add a failing test from issue #2826. Add also a hypothesis test that
	finds the problem if run a few times

diff --git a/pypy/objspace/std/test/test_specialisedtupleobject.py b/pypy/objspace/std/test/test_specialisedtupleobject.py
--- a/pypy/objspace/std/test/test_specialisedtupleobject.py
+++ b/pypy/objspace/std/test/test_specialisedtupleobject.py
@@ -23,20 +23,22 @@
         w_tuple = self.space.newtuple([self.space.wrap(1), self.space.wrap(2)])
         assert w_tuple.__class__.__name__ == 'W_SpecialisedTupleObject_ii'
 
+    def hash_test(self, values, must_be_specialized):
+        N_values_w = [self.space.wrap(value) for value in values]
+        S_values_w = [self.space.wrap(value) for value in values]
+        N_w_tuple = W_TupleObject(N_values_w)
+        S_w_tuple = self.space.newtuple(S_values_w)
+
+        if must_be_specialized:
+            assert 'W_SpecialisedTupleObject' in type(S_w_tuple).__name__
+        assert self.space.is_true(self.space.eq(N_w_tuple, S_w_tuple))
+        assert self.space.is_true(
+                self.space.eq(self.space.hash(N_w_tuple),
+                              self.space.hash(S_w_tuple)))
+
     def test_hash_against_normal_tuple(self):
         def hash_test(values, must_be_specialized=True):
-            N_values_w = [self.space.wrap(value) for value in values]
-            S_values_w = [self.space.wrap(value) for value in values]
-            N_w_tuple = W_TupleObject(N_values_w)
-            S_w_tuple = self.space.newtuple(S_values_w)
-
-            if must_be_specialized:
-                assert 'W_SpecialisedTupleObject' in type(S_w_tuple).__name__
-            assert self.space.is_true(self.space.eq(N_w_tuple, S_w_tuple))
-            assert self.space.is_true(
-                    self.space.eq(self.space.hash(N_w_tuple),
-                                  self.space.hash(S_w_tuple)))
-
+            self.hash_test(values, must_be_specialized=must_be_specialized)
         hash_test([-1, -1])
         hash_test([-1.0, -1.0])
         hash_test([1, 2])
@@ -48,6 +50,20 @@
         hash_test([1, ('a', 2)])
         hash_test([1, ()])
         hash_test([1, 2, 3], must_be_specialized=False)
+        hash_test([1 << 62, 0])
+
+    try:
+        from hypothesis import given, strategies
+    except ImportError:
+        pass
+    else:
+        _int_float_text = strategies.one_of(
+                            strategies.integers(),
+                            strategies.floats(),
+                            strategies.text())
+        @given(_int_float_text, _int_float_text)
+        def test_hash_with_hypothesis(self, x, y):
+            self.hash_test([x, y], must_be_specialized=False)
 
 
 class AppTestW_SpecialisedTupleObject:

From pypy.commits at gmail.com  Wed May  9 12:26:35 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 May 2018 09:26:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix issue #2826
Message-ID: <5af3213b.1c69fb81.a9bdf.38c4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94503:b0d47899b958
Date: 2018-05-09 18:20 +0200
http://bitbucket.org/pypy/pypy/changeset/b0d47899b958/

Log:	Fix issue #2826

diff --git a/pypy/objspace/std/specialisedtupleobject.py b/pypy/objspace/std/specialisedtupleobject.py
--- a/pypy/objspace/std/specialisedtupleobject.py
+++ b/pypy/objspace/std/specialisedtupleobject.py
@@ -71,10 +71,6 @@
                 value = getattr(self, 'value%s' % i)
                 if typetuple[i] == object:
                     y = space.int_w(space.hash(value))
-                elif typetuple[i] == int:
-                    # mimic cpythons behavior of a hash value of -2 for -1
-                    y = value
-                    y -= (y == -1)  # No explicit condition, to avoid JIT bridges
                 elif typetuple[i] == float:
                     # get the correct hash for float which is an
                     # integer & other less frequent cases

From pypy.commits at gmail.com  Wed May  9 12:39:14 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 May 2018 09:39:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backport the new test
Message-ID: <5af32432.1c69fb81.bc712.685f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94504:021cff0c0abb
Date: 2018-05-09 18:38 +0200
http://bitbucket.org/pypy/pypy/changeset/021cff0c0abb/

Log:	Backport the new test

diff --git a/pypy/objspace/std/test/test_specialisedtupleobject.py b/pypy/objspace/std/test/test_specialisedtupleobject.py
--- a/pypy/objspace/std/test/test_specialisedtupleobject.py
+++ b/pypy/objspace/std/test/test_specialisedtupleobject.py
@@ -23,20 +23,22 @@
         w_tuple = self.space.newtuple([self.space.wrap(1), self.space.wrap(2)])
         assert w_tuple.__class__.__name__ == 'W_SpecialisedTupleObject_ii'
 
+    def hash_test(self, values, must_be_specialized=True):
+        N_values_w = [self.space.wrap(value) for value in values]
+        S_values_w = [self.space.wrap(value) for value in values]
+        N_w_tuple = W_TupleObject(N_values_w)
+        S_w_tuple = self.space.newtuple(S_values_w)
+
+        if must_be_specialized:
+            assert 'W_SpecialisedTupleObject' in type(S_w_tuple).__name__
+        assert self.space.is_true(self.space.eq(N_w_tuple, S_w_tuple))
+        assert self.space.is_true(
+                self.space.eq(self.space.hash(N_w_tuple),
+                              self.space.hash(S_w_tuple)))
+
     def test_hash_against_normal_tuple(self):
         def hash_test(values, must_be_specialized=True):
-            N_values_w = [self.space.wrap(value) for value in values]
-            S_values_w = [self.space.wrap(value) for value in values]
-            N_w_tuple = W_TupleObject(N_values_w)
-            S_w_tuple = self.space.newtuple(S_values_w)
-
-            if must_be_specialized:
-                assert 'W_SpecialisedTupleObject' in type(S_w_tuple).__name__
-            assert self.space.is_true(self.space.eq(N_w_tuple, S_w_tuple))
-            assert self.space.is_true(
-                    self.space.eq(self.space.hash(N_w_tuple),
-                                  self.space.hash(S_w_tuple)))
-
+            self.hash_test(values, must_be_specialized=must_be_specialized)
         hash_test([-1, -1])
         hash_test([-1.0, -1.0])
         hash_test([1, 2])
@@ -48,6 +50,20 @@
         hash_test([1, ('a', 2)])
         hash_test([1, ()])
         hash_test([1, 2, 3], must_be_specialized=False)
+        hash_test([1 << 62, 0])
+
+    try:
+        from hypothesis import given, strategies
+    except ImportError:
+        pass
+    else:
+        _int_float_text = strategies.one_of(
+                            strategies.integers(),
+                            strategies.floats(),
+                            strategies.text())
+        @given(_int_float_text, _int_float_text)
+        def test_hash_with_hypothesis(self, x, y):
+            self.hash_test([x, y], must_be_specialized=False)
 
 
 class AppTestW_SpecialisedTupleObject:

From pypy.commits at gmail.com  Wed May  9 12:57:56 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 May 2018 09:57:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5af32894.1c69fb81.8a506.c5fe@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94506:e4a332215983
Date: 2018-05-09 18:55 +0200
http://bitbucket.org/pypy/pypy/changeset/e4a332215983/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,5 +5,16 @@
 .. this is a revision shortly after release-pypy-6.0.0
 .. startrev: e50e11af23f1
 
+.. branch: cppyy-packaging
 
+Upgrade to backend 0.6.0, support exception handling from wrapped functions,
+update enum handling, const correctness for data members and associated tests,
+support anonymous enums, support for function pointer arguments
 
+.. branch: socket_default_timeout_blockingness
+
+Make sure 'blocking-ness' of socket is set along with default timeout
+
+.. branch: crypt_h
+
+Include crypt.h for crypt() on Linux
diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py
--- a/pypy/module/_cppyy/__init__.py
+++ b/pypy/module/_cppyy/__init__.py
@@ -7,6 +7,7 @@
     interpleveldefs = {
         '_resolve_name'          : 'interp_cppyy.resolve_name',
         '_scope_byname'          : 'interp_cppyy.scope_byname',
+        '_is_static_data'        : 'interp_cppyy.is_static_data',
         '_is_template'           : 'interp_cppyy.is_template',
         '_std_string_name'       : 'interp_cppyy.std_string_name',
         '_set_class_generator'   : 'interp_cppyy.set_class_generator',
@@ -21,7 +22,7 @@
     }
 
     appleveldefs = {
-        '_init_pythonify'        : 'pythonify._init_pythonify',
+        '_post_import_startup'   : 'pythonify._post_import_startup',
         'add_pythonization'      : 'pythonify.add_pythonization',
         'Template'               : 'pythonify.CPPTemplate',
     }
@@ -34,9 +35,3 @@
         # code generation is not, so give it a chance to run now
         from pypy.module._cppyy import capi
         capi.register_pythonizations(space)
-
-    def startup(self, space):
-        from pypy.module._cppyy import capi
-        capi.verify_backend(space)      # may raise ImportError
-
-        space.call_method(self, '_init_pythonify')
diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -308,7 +308,7 @@
         c_call = state.capi_calls[name]
     except KeyError:
         if state.backend is None:
-            load_backend(space)
+            verify_backend(space)
         iface = state.capi_call_ifaces[name]
         cfunc = W_RCTypeFunc(space, iface[0], iface[1], False)
         c_call = state.backend.load_function(cfunc, 'cppyy_'+name)
@@ -421,7 +421,7 @@
         _cdata_to_ptr(space, call_capi(space, 'function_address_from_index', args)))
 def c_function_address_from_method(space, cppmethod):
     return rffi.cast(C_FUNC_PTR,
-        _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', _ArgH(cppmethod))))
+        _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', [_ArgH(cppmethod)])))
 
 # handling of function argument buffer ---------------------------------------
 def c_allocate_function_args(space, size):
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -686,6 +686,34 @@
         decref(space, rffi.cast(PyObject, rffi.cast(rffi.VOIDPP, arg)[0]))
 
 
+class FunctionPointerConverter(TypeConverter):
+    _immutable_fields_ = ['signature']
+
+    def __init__(self, space, signature):
+        self.signature = signature
+
+    def convert_argument(self, space, w_obj, address, call_local):
+        # TODO: atm, does not actually get an overload, but a staticmethod
+        from pypy.module._cppyy.interp_cppyy import W_CPPOverload
+        cppol = space.interp_w(W_CPPOverload, w_obj)
+
+        # find the function with matching signature
+        for i in range(len(cppol.functions)):
+            m = cppol.functions[i]
+            if m.signature(False) == self.signature:
+                x = rffi.cast(rffi.VOIDPP, address)
+                x[0] = rffi.cast(rffi.VOIDP,
+                    capi.c_function_address_from_method(space, m.cppmethod))
+                address = rffi.cast(capi.C_OBJECT, address)
+                ba = rffi.cast(rffi.CCHARP, address)
+                ba[capi.c_function_arg_typeoffset(space)] = 'p'
+                return
+
+        # lookup failed
+        raise oefmt(space.w_TypeError,
+                    "no overload found matching %s", self.signature)
+
+
 class MacroConverter(TypeConverter):
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         # TODO: get the actual type info from somewhere ...
@@ -749,6 +777,14 @@
             return InstancePtrPtrConverter(space, clsdecl)
         elif compound == "":
             return InstanceConverter(space, clsdecl)
+    elif "(anonymous)" in name:
+        # special case: enum w/o a type name
+        return _converters["internal_enum_type_t"](space, default)
+    elif "(*)" in name or "::*)" in name:
+        # function pointer
+        pos = name.find("*)")
+        if pos > 0:
+            return FunctionPointerConverter(space, name[pos+2:])
 
     #   5) void* or void converter (which fails on use)
     if 0 <= compound.find('*'):
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -293,6 +293,9 @@
             return InstancePtrExecutor(space, cppclass)
         elif compound == '**' or compound == '*&':
             return InstancePtrPtrExecutor(space, cppclass)
+    elif "(anonymous)" in name:
+        # special case: enum w/o a type name
+        return _executors["internal_enum_type_t"](space, None)
 
     # 4) additional special cases
     if compound == '*':
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -128,7 +128,7 @@
 
 def register_class(space, w_pycppclass):
     w_cppclass = space.findattr(w_pycppclass, space.newtext("__cppdecl__"))
-    cppclass = space.interp_w(W_CPPClassDecl, w_cppclass, can_be_None=False)
+    cppclass = space.interp_w(W_CPPClassDecl, w_cppclass)
     # add back-end specific method pythonizations (doing this on the wrapped
     # class allows simple aliasing of methods)
     capi.pythonize(space, cppclass.name, w_pycppclass)
@@ -149,6 +149,24 @@
 W_CPPLibrary.typedef.acceptable_as_base_class = True
 
 
+#-----
+# Classes involved with methods and functions:
+#
+#  CPPMethod:         base class wrapping a single function or method
+#  CPPConstructor:    specialization for allocating a new object
+#  CPPFunction:       specialization for free and static functions
+#  CPPSetItem:        specialization for Python's __setitem__
+#  CPPTemplatedCall:  trampoline to instantiate and bind templated functions
+#  W_CPPOverload, W_CPPConstructorOverload, W_CPPTemplateOverload:
+#     user-facing, app-level, collection of overloads, with specializations
+#     for constructors and templates
+#  W_CPPBoundMethod:  instantiated template method
+#
+# All methods/functions derive from CPPMethod and are collected as overload
+# candidates in user-facing overload classes. Templated methods are a two-step
+# process, where first the template is instantiated (or selected if already
+# available), which returns a callable object that is the actual bound method.
+
 class CPPMethod(object):
     """Dispatcher of methods. Checks the arguments, find the corresponding FFI
     function if available, makes the call, and returns the wrapped result. It
@@ -177,7 +195,7 @@
 
     @staticmethod
     def unpack_cppthis(space, w_cppinstance, declaring_scope):
-        cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False)
+        cppinstance = space.interp_w(W_CPPInstance, w_cppinstance)
         cppinstance._nullcheck()
         return cppinstance.get_cppthis(declaring_scope)
 
@@ -424,7 +442,7 @@
 
 
 class CPPFunction(CPPMethod):
-    """Global (namespaced) function dispatcher."""
+    """Global (namespaced) / static function dispatcher."""
 
     _immutable_ = True
 
@@ -688,6 +706,18 @@
 )
 
 
+#-----
+# Classes for data members:
+#
+#  W_CPPDataMember:        instance data members
+#  W_CPPConstDataMember:   specialization for const data members
+#  W_CPPStaticData:        class-level and global/static data
+#  W_CPPConstStaticData:   specialization for const global/static data
+#
+# Data is represented by an offset which is either a global pointer (static data)
+# or an offset from the start of an instance (data members). The "const"
+# specializations raise when attempting to set their value.
+
 class W_CPPDataMember(W_Root):
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
@@ -698,9 +728,6 @@
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
 
-    def is_static(self):
-        return self.space.w_False
-
     def _get_offset(self, cppinstance):
         if cppinstance:
             assert lltype.typeOf(cppinstance.clsdecl.handle) == lltype.typeOf(self.scope.handle)
@@ -728,16 +755,25 @@
 
 W_CPPDataMember.typedef = TypeDef(
     'CPPDataMember',
-    is_static = interp2app(W_CPPDataMember.is_static),
     __get__ = interp2app(W_CPPDataMember.get),
     __set__ = interp2app(W_CPPDataMember.set),
 )
 W_CPPDataMember.typedef.acceptable_as_base_class = False
 
+
+class W_CPPConstDataMember(W_CPPDataMember):
+    def set(self, w_cppinstance, w_value):
+        raise oefmt(self.space.w_TypeError, "assignment to const data not allowed")
+
+W_CPPConstDataMember.typedef = TypeDef(
+    'CPPConstDataMember',
+    __get__ = interp2app(W_CPPDataMember.get),
+    __set__ = interp2app(W_CPPConstDataMember.set),
+)
+W_CPPConstDataMember.typedef.acceptable_as_base_class = False
+
+
 class W_CPPStaticData(W_CPPDataMember):
-    def is_static(self):
-        return self.space.w_True
-
     @jit.elidable_promote()
     def _get_offset(self, cppinstance):
         return self.offset
@@ -751,19 +787,34 @@
 
 W_CPPStaticData.typedef = TypeDef(
     'CPPStaticData',
-    is_static = interp2app(W_CPPStaticData.is_static),
     __get__ = interp2app(W_CPPStaticData.get),
     __set__ = interp2app(W_CPPStaticData.set),
 )
 W_CPPStaticData.typedef.acceptable_as_base_class = False
 
-def is_static(space, w_obj):
+
+class W_CPPConstStaticData(W_CPPStaticData):
+    def set(self, w_cppinstance, w_value):
+        raise oefmt(self.space.w_TypeError, "assignment to const data not allowed")
+
+W_CPPConstStaticData.typedef = TypeDef(
+    'CPPConstStaticData',
+    __get__ = interp2app(W_CPPConstStaticData.get),
+    __set__ = interp2app(W_CPPConstStaticData.set),
+)
+W_CPPConstStaticData.typedef.acceptable_as_base_class = False
+
+
+def is_static_data(space, w_obj):
     try:
-        space.interp_w(W_CPPStaticData, w_obj, can_be_None=False)
+        space.interp_w(W_CPPStaticData, w_obj)
         return space.w_True
     except Exception:
         return space.w_False
 
+#-----
+
+
 class W_CPPScopeDecl(W_Root):
     _attrs_ = ['space', 'handle', 'name', 'methods', 'datamembers']
     _immutable_fields_ = ['handle', 'name']
@@ -847,7 +898,10 @@
         offset = capi.c_datamember_offset(self.space, self, dm_idx)
         if offset == -1:
             raise self.missing_attribute_error(dm_name)
-        datamember = W_CPPStaticData(self.space, self, type_name, offset)
+        if capi.c_is_const_data(self.space, self, dm_idx):
+            datamember = W_CPPConstStaticData(self.space, self, type_name, offset)
+        else:
+            datamember = W_CPPStaticData(self.space, self, type_name, offset)
         self.datamembers[dm_name] = datamember
         return datamember
 
@@ -967,8 +1021,13 @@
             if offset == -1:
                 continue      # dictionary problem; raises AttributeError on use
             is_static = bool(capi.c_is_staticdata(self.space, self, i))
-            if is_static:
+            is_const  = bool(capi.c_is_const_data(self.space, self, i))
+            if is_static and is_const:
+                datamember = W_CPPConstStaticData(self.space, self, type_name, offset)
+            elif is_static:
                 datamember = W_CPPStaticData(self.space, self, type_name, offset)
+            elif is_const:
+                datamember = W_CPPConstDataMember(self.space, self, type_name, offset)
             else:
                 datamember = W_CPPDataMember(self.space, self, type_name, offset)
             self.datamembers[datamember_name] = datamember
@@ -1124,7 +1183,7 @@
         # scopes of the argument classes (TODO: implement that last option)
         try:
             # TODO: expecting w_other to be an W_CPPInstance is too limiting
-            other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False)
+            other = self.space.interp_w(W_CPPInstance, w_other)
             for name in ["", "__gnu_cxx", "__1"]:
                 nss = scope_byname(self.space, name)
                 meth_idx = capi.c_get_global_operator(
@@ -1146,7 +1205,7 @@
 
         # fallback 2: direct pointer comparison (the class comparison is needed since
         # the first data member in a struct and the struct have the same address)
-        other = self.space.interp_w(W_CPPInstance, w_other, can_be_None=False)  # TODO: factor out
+        other = self.space.interp_w(W_CPPInstance, w_other)  # TODO: factor out
         iseq = (self._rawobject == other._rawobject) and (self.clsdecl == other.clsdecl)
         return self.space.newbool(iseq)
 
@@ -1265,7 +1324,7 @@
                 offset = capi.c_base_offset1(space, actual, clsdecl, rawobject, -1)
                 rawobject = capi.direct_ptradd(rawobject, offset)
                 w_cppdecl = space.findattr(w_pycppclass, space.newtext("__cppdecl__"))
-                clsdecl = space.interp_w(W_CPPClassDecl, w_cppdecl, can_be_None=False)
+                clsdecl = space.interp_w(W_CPPClassDecl, w_cppdecl)
             except Exception:
                 # failed to locate/build the derived class, so stick to the base (note
                 # that only get_pythonized_cppclass is expected to raise, so none of
@@ -1283,7 +1342,7 @@
 
     # fresh creation
     w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass)
-    cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False)
+    cppinstance = space.interp_w(W_CPPInstance, w_cppinstance)
     cppinstance.__init__(space, clsdecl, rawobject, is_ref, python_owns)
     memory_regulator.register(cppinstance)
     return w_cppinstance
@@ -1311,7 +1370,7 @@
     except Exception:
         # accept integer value as address
         rawobject = rffi.cast(capi.C_OBJECT, space.uint_w(w_obj))
-    decl = space.interp_w(W_CPPClassDecl, w_clsdecl, can_be_None=False)
+    decl = space.interp_w(W_CPPClassDecl, w_clsdecl)
     return wrap_cppinstance(space, rawobject, decl, python_owns=owns, do_cast=cast)
 
 @unwrap_spec(owns=bool, cast=bool)
@@ -1327,7 +1386,7 @@
 
 def move(space, w_obj):
     """Casts the given instance into an C++-style rvalue."""
-    obj = space.interp_w(W_CPPInstance, w_obj, can_be_None=True)
+    obj = space.interp_w(W_CPPInstance, w_obj)
     if obj:
         obj.flags |= INSTANCE_FLAGS_IS_R_VALUE
     return w_obj
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -1,5 +1,5 @@
 # NOT_RPYTHON
-# do not load _cppyy here, see _init_pythonify()
+# do not load _cppyy here, see _post_import_startup()
 import types
 import sys
 
@@ -22,7 +22,7 @@
 class CPPClass(CPPScope):
     pass
 
-# namespace base class (class base class defined in _init_pythonify)
+# namespace base class (class base class defined in _post_import_startup()
 class CPPNamespace(object):
     __metatype__ = CPPMetaNamespace
 
@@ -169,6 +169,7 @@
     return method
 
 def make_cppclass(scope, cl_name, decl):
+    import _cppyy
 
     # get a list of base classes for class creation
     bases = [get_pycppclass(base) for base in decl.get_base_names()]
@@ -209,7 +210,7 @@
     for d_name in decl.get_datamember_names():
         cppdm = decl.get_datamember(d_name)
         d_class[d_name] = cppdm
-        if cppdm.is_static():
+        if _cppyy._is_static_data(cppdm):
             d_meta[d_name] = cppdm
 
     # create a metaclass to allow properties (for static data write access)
@@ -278,7 +279,7 @@
         try:
             cppdm = scope.__cppdecl__.get_datamember(name)
             setattr(scope, name, cppdm)
-            if cppdm.is_static():
+            if _cppyy._is_static_data(cppdm):
                 setattr(scope.__class__, name, cppdm)
             pycppitem = getattr(scope, name)      # gets actual property value
         except AttributeError:
@@ -406,7 +407,7 @@
         pyclass.__len__     = return2
 
 
-def _init_pythonify():
+def _post_import_startup():
     # _cppyy should not be loaded at the module level, as that will trigger a
     # call to space.getbuiltinmodule(), which will cause _cppyy to be loaded
     # at pypy-c startup, rather than on the "import _cppyy" statement
diff --git a/pypy/module/_cppyy/src/dummy_backend.cxx b/pypy/module/_cppyy/src/dummy_backend.cxx
--- a/pypy/module/_cppyy/src/dummy_backend.cxx
+++ b/pypy/module/_cppyy/src/dummy_backend.cxx
@@ -348,6 +348,7 @@
         PUBLIC_CPPYY_DATA3(short,   short,              h);
         PUBLIC_CPPYY_DATA3(ushort,  unsigned short,     H);
         PUBLIC_CPPYY_DATA3(int,     int,                i);
+        PUBLIC_CPPYY_DATA (const_int, const int);
         PUBLIC_CPPYY_DATA3(uint,    unsigned int,       I);
         PUBLIC_CPPYY_DATA3(long,    long,               l);
         PUBLIC_CPPYY_DATA3(ulong,   unsigned long,      L);
@@ -1032,7 +1033,9 @@
     return s_scopes[handle].m_datambrs[idatambr].m_isstatic;
 }
 
-int cppyy_is_const_data(cppyy_scope_t /* handle */, cppyy_index_t /* idatambr */) {
+int cppyy_is_const_data(cppyy_scope_t handle, cppyy_index_t idatambr) {
+    if (s_scopes[handle].m_datambrs[idatambr].m_name == "m_const_int")
+        return 1;
     return 0;
 }
 
diff --git a/pypy/module/_cppyy/test/datatypes.cxx b/pypy/module/_cppyy/test/datatypes.cxx
--- a/pypy/module/_cppyy/test/datatypes.cxx
+++ b/pypy/module/_cppyy/test/datatypes.cxx
@@ -6,7 +6,7 @@
 
 
 //===========================================================================
-CppyyTestData::CppyyTestData() : m_owns_arrays(false)
+CppyyTestData::CppyyTestData() : m_const_int(17), m_owns_arrays(false)
 {
     m_bool    = false;
     m_char    = 'a';
@@ -333,3 +333,17 @@
 CppyyTestPod* get_null_pod() {
     return (CppyyTestPod*)0;
 }
+
+
+//= function pointer passing ================================================
+int sum_of_int(int i1, int i2) {
+    return i1+i2;
+}
+
+double sum_of_double(double d1, double d2) {
+    return d1+d2;
+}
+
+double call_double_double(double (*d)(double, double), double d1, double d2) {
+    return d(d1, d2);
+}
diff --git a/pypy/module/_cppyy/test/datatypes.h b/pypy/module/_cppyy/test/datatypes.h
--- a/pypy/module/_cppyy/test/datatypes.h
+++ b/pypy/module/_cppyy/test/datatypes.h
@@ -1,5 +1,5 @@
 // copied from RtypesCore.h ...
-#if defined(R__WIN32)
+#if defined(R__WIN32) && !defined(__CINT__)
 typedef __int64          Long64_t;  //Portable signed long integer 8 bytes
 typedef unsigned __int64 ULong64_t; //Portable unsigned long integer 8 bytes
 #else
@@ -26,8 +26,13 @@
 
 //===========================================================================
 namespace EnumSpace {
-   enum E {E1 = 1, E2};
-};
+    enum E {E1 = 1, E2};
+    class EnumClass {
+    public:
+        enum    {E1 = -1};
+        enum EE {E2 = -1};
+    };
+}
 
 
 //===========================================================================
@@ -243,6 +248,7 @@
     short                m_short;
     unsigned short       m_ushort;
     int                  m_int;
+    const int            m_const_int;   // special case: const testing
     unsigned int         m_uint;
     long                 m_long;
     unsigned long        m_ulong;
@@ -364,3 +370,9 @@
 void set_global_pod(CppyyTestPod* t);
 CppyyTestPod* get_global_pod();
 CppyyTestPod* get_null_pod();
+
+
+//= function pointer passing ================================================
+int sum_of_int(int i1, int i2);
+double sum_of_double(double d1, double d2);
+double call_double_double(double (*d)(double, double), double d1, double d2);
diff --git a/pypy/module/_cppyy/test/datatypes.xml b/pypy/module/_cppyy/test/datatypes.xml
--- a/pypy/module/_cppyy/test/datatypes.xml
+++ b/pypy/module/_cppyy/test/datatypes.xml
@@ -4,6 +4,8 @@
   <class name="FourVector" />
 
   <enum name="EFruit" />
+  <enum name="EnumSpace::E" />
+  <class name="EnumSpace::EnumClass" />
 
   <function pattern="get_*" />
   <function pattern="set_*" />
@@ -14,4 +16,8 @@
   <variable name="g_int" />
   <variable name="g_pod" />
 
+  <function name="sum_of_int" />
+  <function name="sum_of_double" />
+  <function name="call_double_double" />
+
 </lcgdict>
diff --git a/pypy/module/_cppyy/test/test_advancedcpp.py b/pypy/module/_cppyy/test/test_advancedcpp.py
--- a/pypy/module/_cppyy/test/test_advancedcpp.py
+++ b/pypy/module/_cppyy/test/test_advancedcpp.py
@@ -22,7 +22,8 @@
     def setup_class(cls):
         cls.w_test_dct = cls.space.newtext(test_dct)
         cls.w_advanced = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_default_arguments(self):
diff --git a/pypy/module/_cppyy/test/test_cpp11features.py b/pypy/module/_cppyy/test/test_cpp11features.py
--- a/pypy/module/_cppyy/test/test_cpp11features.py
+++ b/pypy/module/_cppyy/test/test_cpp11features.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_shared_ptr(self):
diff --git a/pypy/module/_cppyy/test/test_cppyy.py b/pypy/module/_cppyy/test/test_cppyy.py
--- a/pypy/module/_cppyy/test/test_cppyy.py
+++ b/pypy/module/_cppyy/test/test_cppyy.py
@@ -33,6 +33,7 @@
         cls.w_lib, cls.w_instantiate, cls.w_example01, cls.w_payload = \
                    cls.space.unpackiterable(cls.space.appexec([], """():
             import _cppyy, ctypes
+            _cppyy._post_import_startup()
             lib = ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)
             def cpp_instantiate(tt, *args):
                 inst = _cppyy._bind_object(0, tt, True)
diff --git a/pypy/module/_cppyy/test/test_crossing.py b/pypy/module/_cppyy/test/test_crossing.py
--- a/pypy/module/_cppyy/test/test_crossing.py
+++ b/pypy/module/_cppyy/test/test_crossing.py
@@ -72,7 +72,9 @@
         # to allow the generated extension module be loaded first)
         cls.w_test_dct    = cls.space.newtext(test_dct)
         cls.w_pre_imports = cls.space.appexec([], """():
-            import ctypes, _cppyy""")   # prevents leak-checking complaints on ctypes' statics
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()""")   # early import of ctypes
+                  # prevents leak-checking complaints on ctypes' statics
 
     def setup_method(self, func):
         @unwrap_spec(name='text', init='text', body='text')
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_datatypes = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
         cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
 
@@ -193,6 +194,10 @@
             for i in range(self.N):
                 assert eval('c.m_%s_array2[i]' % names[j]) == b[i]
 
+        # can not write to constant data
+        assert c.m_const_int == 17
+        raises(TypeError, setattr, c, 'm_const_int', 71)
+
         c.__destruct__()
 
     def test03_array_passing(self):
@@ -466,6 +471,10 @@
         assert gbl.kBanana == 29
         assert gbl.kCitrus == 34
 
+        assert gbl.EnumSpace.E
+        assert gbl.EnumSpace.EnumClass.E1 == -1   # anonymous
+        assert gbl.EnumSpace.EnumClass.E2 == -1   # named type
+
     def test11_string_passing(self):
         """Test passing/returning of a const char*"""
 
@@ -743,3 +752,22 @@
 
         c.s_voidp = c2
         address_equality_test(c.s_voidp, c2)
+
+    def test21_function_pointers(self):
+        """Function pointer passing"""
+
+        import _cppyy as cppyy
+
+        f1 = cppyy.gbl.sum_of_int
+        f2 = cppyy.gbl.sum_of_double
+        f3 = cppyy.gbl.call_double_double
+
+        assert 5 == f1(2, 3)
+        assert 5. == f2(5., 0.)
+
+        raises(TypeError, f3, f1, 2, 3)
+
+        # TODO: get straightforward access to the overload type
+        f2 = cppyy.gbl.__cppdecl__.get_overload('sum_of_double')
+
+        assert 5. == f3(f2, 5., 0.)
diff --git a/pypy/module/_cppyy/test/test_fragile.py b/pypy/module/_cppyy/test/test_fragile.py
--- a/pypy/module/_cppyy/test/test_fragile.py
+++ b/pypy/module/_cppyy/test/test_fragile.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_fragile = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_missing_classes(self):
diff --git a/pypy/module/_cppyy/test/test_operators.py b/pypy/module/_cppyy/test/test_operators.py
--- a/pypy/module/_cppyy/test/test_operators.py
+++ b/pypy/module/_cppyy/test/test_operators.py
@@ -15,7 +15,8 @@
         cls.w_N = cls.space.newint(5)  # should be imported from the dictionary
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_operators = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def teardown_method(self, meth):
diff --git a/pypy/module/_cppyy/test/test_overloads.py b/pypy/module/_cppyy/test/test_overloads.py
--- a/pypy/module/_cppyy/test/test_overloads.py
+++ b/pypy/module/_cppyy/test/test_overloads.py
@@ -17,7 +17,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_overloads = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_class_based_overloads(self):
diff --git a/pypy/module/_cppyy/test/test_pythonify.py b/pypy/module/_cppyy/test/test_pythonify.py
--- a/pypy/module/_cppyy/test/test_pythonify.py
+++ b/pypy/module/_cppyy/test/test_pythonify.py
@@ -16,7 +16,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_example01 = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_finding_classes(self):
diff --git a/pypy/module/_cppyy/test/test_stltypes.py b/pypy/module/_cppyy/test/test_stltypes.py
--- a/pypy/module/_cppyy/test/test_stltypes.py
+++ b/pypy/module/_cppyy/test/test_stltypes.py
@@ -15,7 +15,8 @@
         cls.w_N = cls.space.newint(13)
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_stlvector = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_builtin_type_vector_types(self):
diff --git a/pypy/module/_cppyy/test/test_templates.py b/pypy/module/_cppyy/test/test_templates.py
--- a/pypy/module/_cppyy/test/test_templates.py
+++ b/pypy/module/_cppyy/test/test_templates.py
@@ -14,7 +14,8 @@
     def setup_class(cls):
         cls.w_test_dct  = cls.space.newtext(test_dct)
         cls.w_datatypes = cls.space.appexec([], """():
-            import ctypes
+            import ctypes, _cppyy
+            _cppyy._post_import_startup()
             return ctypes.CDLL(%r, ctypes.RTLD_GLOBAL)""" % (test_dct, ))
 
     def test01_template_member_functions(self):
diff --git a/pypy/module/cpyext/include/pyport.h b/pypy/module/cpyext/include/pyport.h
--- a/pypy/module/cpyext/include/pyport.h
+++ b/pypy/module/cpyext/include/pyport.h
@@ -128,4 +128,36 @@
 #else
 #endif
 
+/*
+ * Hide GCC attributes from compilers that don't support them.
+ */
+#if (!defined(__GNUC__) || __GNUC__ < 2 || \
+     (__GNUC__ == 2 && __GNUC_MINOR__ < 7) ) && \
+    !defined(RISCOS)
+#define Py_GCC_ATTRIBUTE(x)
+#else
+#define Py_GCC_ATTRIBUTE(x) __attribute__(x)
+#endif
+
+/*
+ * Specify alignment on compilers that support it.
+ */
+#if defined(__GNUC__) && __GNUC__ >= 3
+#define Py_ALIGNED(x) __attribute__((aligned(x)))
+#else
+#define Py_ALIGNED(x)
+#endif
+
+/*
+ * Older Microsoft compilers don't support the C99 long long literal suffixes,
+ * so these will be defined in PC/pyconfig.h for those compilers.
+ */
+#ifndef Py_LL
+#define Py_LL(x) x##LL
+#endif
+
+#ifndef Py_ULL
+#define Py_ULL(x) Py_LL(x##U)
+#endif
+
 #endif /* Py_PYPORT_H */
diff --git a/pypy/module/crypt/interp_crypt.py b/pypy/module/crypt/interp_crypt.py
--- a/pypy/module/crypt/interp_crypt.py
+++ b/pypy/module/crypt/interp_crypt.py
@@ -5,6 +5,9 @@
 
 if sys.platform.startswith('darwin'):
     eci = ExternalCompilationInfo()
+elif sys.platform.startswith('linux'):
+    # crypt() is defined only in crypt.h on some Linux variants (eg. Fedora 28)
+    eci = ExternalCompilationInfo(libraries=['crypt'], includes=["crypt.h"])
 else:
     eci = ExternalCompilationInfo(libraries=['crypt'])
 c_crypt = rffi.llexternal('crypt', [rffi.CCHARP, rffi.CCHARP], rffi.CCHARP,
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -410,7 +410,6 @@
 
     def descr_hash(self, space):
         h = _hash_float(space, self.floatval)
-        h -= (h == -1)
         return space.newint(h)
 
     def descr_format(self, space, w_spec):
@@ -761,7 +760,8 @@
     x = ((x << e) & HASH_MODULUS) | x >> (HASH_BITS - e)
 
     x = intmask(intmask(x) * sign)
-    return -2 if x == -1 else x
+    x -= (x == -1)
+    return x
 
 
 def _divmod_w(space, w_float1, w_float2):
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -214,7 +214,7 @@
     ret = unw_getcontext(&uc);
     if (ret < 0) {
         // could not initialize lib unwind cursor and context
-        fprintf(stderr, "WARNING: unw_getcontext did not retrieve context, switching to python profiling mode \n");
+        fprintf(stderr, "WARNING: unw_getcontext did not retreive context, switching to python profiling mode \n");
         vmp_native_disable();
         return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
     }

From pypy.commits at gmail.com  Wed May  9 12:57:53 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 May 2018 09:57:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: unify the hash computation more with
 py3.5
Message-ID: <5af32891.1c69fb81.dfe2.169a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94505:52d2cf0086d1
Date: 2018-05-09 18:53 +0200
http://bitbucket.org/pypy/pypy/changeset/52d2cf0086d1/

Log:	unify the hash computation more with py3.5

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -391,7 +391,6 @@
 
     def descr_hash(self, space):
         h = _hash_float(space, self.floatval)
-        h -= (h == -1)
         return space.newint(h)
 
     def descr_format(self, space, w_spec):
@@ -716,8 +715,6 @@
         # This must return the same hash as an equal int or long.
         try:
             x = ovfcheck_float_to_int(intpart)
-            # Fits in a C long == a Python int, so is its own hash.
-            return x
         except OverflowError:
             # Convert to long and use its hash.
             try:
@@ -729,6 +726,10 @@
                 else:
                     return 314159
             return space.int_w(space.hash(w_lval))
+        else:
+            # Fits in a C long == a Python int.
+            from pypy.objspace.std.intobject import _hash_int
+            return _hash_int(x)
 
     # The fractional part is non-zero, so we don't have to worry about
     # making this match the hash of some other type.
@@ -747,6 +748,7 @@
     hipart = int(v)    # take the top 32 bits
     v = (v - hipart) * 2147483648.0 # get the next 32 bits
     x = intmask(hipart + int(v) + (expo << 15))
+    x -= (x == -1)
     return x
 
 
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -361,13 +361,7 @@
         return _new_int(space, w_inttype, w_x, w_base)
 
     def descr_hash(self, space):
-        # For compatibility with CPython, we special-case -1
-        # Make sure this is consistent with the hash of floats and longs.
-        # The complete list of built-in types whose hash should be
-        # consistent is: int, long, bool, float, complex.
-        h = self.intval
-        h -= (h == -1)  # No explicit condition, to avoid JIT bridges
-        return wrapint(space, h)
+        return space.newint(_hash_int(self.intval))
 
     def _int(self, space):
         return self.int(space)
@@ -893,3 +887,12 @@
     __rpow__ = interp2app(W_IntObject.descr_rpow,
                           doc=W_AbstractIntObject.descr_rpow.__doc__),
 )
+
+
+def _hash_int(a):
+    # For compatibility with CPython, we special-case -1
+    # Make sure this is consistent with the hash of floats and longs.
+    # The complete list of built-in types whose hash should be
+    # consistent is: int, long, bool, float, complex.
+    #
+    return a - (a == -1)  # No explicit condition, to avoid JIT bridges
diff --git a/pypy/objspace/std/specialisedtupleobject.py b/pypy/objspace/std/specialisedtupleobject.py
--- a/pypy/objspace/std/specialisedtupleobject.py
+++ b/pypy/objspace/std/specialisedtupleobject.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import oefmt
 from pypy.objspace.std.tupleobject import W_AbstractTupleObject
 from pypy.objspace.std.util import negate
-from rpython.rlib.objectmodel import compute_hash, specialize
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
@@ -71,18 +71,19 @@
                 value = getattr(self, 'value%s' % i)
                 if typetuple[i] == object:
                     y = space.int_w(space.hash(value))
-                elif typetuple[i] == int:
-                    # mimic cpythons behavior of a hash value of -2 for -1
-                    y = value
-                    y -= (y == -1)  # No explicit condition, to avoid JIT bridges
                 elif typetuple[i] == float:
                     # get the correct hash for float which is an
                     # integer & other less frequent cases
                     from pypy.objspace.std.floatobject import _hash_float
                     y = _hash_float(space, value)
-                    y -= (y == -1)
+                elif typetuple[i] == int:
+                    # hash for int which is different from the hash
+                    # given by rpython
+                    from pypy.objspace.std.intobject import _hash_int
+                    y = _hash_int(value)
                 else:
-                    assert 0, "unreachable"
+                    raise NotImplementedError
+
                 x = (x ^ y) * mult
                 z -= 1
                 mult += 82520 + z + z
diff --git a/pypy/objspace/std/test/test_specialisedtupleobject.py b/pypy/objspace/std/test/test_specialisedtupleobject.py
--- a/pypy/objspace/std/test/test_specialisedtupleobject.py
+++ b/pypy/objspace/std/test/test_specialisedtupleobject.py
@@ -23,7 +23,7 @@
         w_tuple = self.space.newtuple([self.space.wrap(1), self.space.wrap(2)])
         assert w_tuple.__class__.__name__ == 'W_SpecialisedTupleObject_ii'
 
-    def hash_test(self, values, must_be_specialized=True):
+    def hash_test(self, values, must_be_specialized):
         N_values_w = [self.space.wrap(value) for value in values]
         S_values_w = [self.space.wrap(value) for value in values]
         N_w_tuple = W_TupleObject(N_values_w)

From pypy.commits at gmail.com  Wed May  9 13:29:15 2018
From: pypy.commits at gmail.com (mwjackson)
Date: Wed, 09 May 2018 10:29:15 -0700 (PDT)
Subject: [pypy-commit] pypy default: fixes for broken parts of docs
Message-ID: <5af32feb.1c69fb81.58e1.012b@mx.google.com>

Author: Matt Jackson <email at mwjackson.net>
Branch: 
Changeset: r94507:da9870c526f7
Date: 2018-04-21 19:29 +0100
http://bitbucket.org/pypy/pypy/changeset/da9870c526f7/

Log:	fixes for broken parts of docs

diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -8,3 +8,4 @@
    :maxdepth: 1
 
    man/pypy.1.rst
+   man/pypy3.1.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -34,6 +34,7 @@
    whatsnew-2.0.0-beta1.rst
    whatsnew-1.9.rst
 
+
 CPython 3.5 compatible versions
 -------------------------------
 
diff --git a/pypy/doc/whatsnew-pypy2-5.10.0.rst b/pypy/doc/whatsnew-pypy2-5.10.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.10.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst
@@ -32,6 +32,7 @@
 
 .. branch: fix-vmprof-stacklet-switch
 .. branch: fix-vmprof-stacklet-switch-2
+
 Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...)
 
 .. branch: win32-vcvars
@@ -39,4 +40,3 @@
 .. branch: rdict-fast-hash
 
 Make it possible to declare that the hash function of an r_dict is fast in RPython.
-
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -150,7 +150,7 @@
 build dependencies for windows. As part of the `rpython` setup stage, environment
 variables will be set to use these dependencies. The repository has a README
 file on how to replicate, and a branch for each supported platform. You may run
- the `get_externals.py` utility to checkout the proper branch for your platform
+the `get_externals.py` utility to checkout the proper branch for your platform
 and PyPy version.
 
 .. _repository:  https://bitbucket.org/pypy/external

From pypy.commits at gmail.com  Wed May  9 13:29:18 2018
From: pypy.commits at gmail.com (mwjackson)
Date: Wed, 09 May 2018 10:29:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: coalesced the 3 existing contributing
 docs into a single doc
Message-ID: <5af32fee.1c69fb81.415fb.1674@mx.google.com>

Author: Matt Jackson <email at mwjackson.net>
Branch: 
Changeset: r94508:b84ccf3ff0d6
Date: 2018-04-21 19:30 +0100
http://bitbucket.org/pypy/pypy/changeset/b84ccf3ff0d6/

Log:	coalesced the 3 existing contributing docs into a single doc

diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/contributing.rst
@@ -0,0 +1,490 @@
+Contributing Guidelines
+===========================
+
+.. contents::
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy's
+  approach is great if you do TDD, and not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below.
+
+The first thing to remember is that PyPy project is very different than most projects out there.
+It's also different from a classic compiler project, so academic courses
+about compilers often don't apply or lead in the wrong direction.
+
+Getting involved
+----------------
+
+PyPy employs an open development process.  You are invited to join our
+`pypy-dev mailing list`_, our IRC channel (#pypy on freenode) or look at the
+other :ref:`contact possibilities <contact>`.  Usually we give out commit
+rights fairly liberally, so if you want to do something with PyPy, you can
+become a committer. We also run frequent coding sprints which are separately
+announced and often happen around Python conferences such as EuroPython or
+PyCon. Upcoming events are usually announced on `the blog`_.
+
+.. _the blog: http://morepypy.blogspot.com
+.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
+
+
+Hacking
+--------
+
+The first and most important rule how _not_ to contribute to PyPy is
+"just hacking". This won't work. There are two major reasons why not
+-- build times are large and PyPy has very thick layer separation which
+make it harder to "just hack a feature". Instead, reach out on the dev mailing
+list or the IRC channel, and we're more than happy to help! :)
+
+Source Control
+==============
+
+Using Mercurial
+---------------
+
+PyPy development is based on Mercurial (hg).  If you are not used to
+version control, the cycle for a new PyPy contributor goes typically
+like this:
+
+* Make an account on bitbucket_.
+
+* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
+  icons).  You get a fork of the repository, e.g. in
+  https://bitbucket.org/yourname/pypy/.
+
+* Clone this new repo (i.e. the fork) to your local machine with the command
+  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
+  operation but only ever needs to be done once.  See also
+  http://pypy.org/download.html#building-from-source .
+  If you already cloned
+  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
+  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
+  your clone to contain the line ``default =
+  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
+  up``.  If you already have such a clone but don't want to change it,
+  you can clone that copy with ``hg clone /path/to/other/copy``, and
+  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
+
+* Now you have a complete copy of the PyPy repo.  Make a branch
+  with a command like ``hg branch name_of_your_branch``.
+
+* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
+  to make Mercurial aware of new files you added, e.g. new test files.
+  Use ``hg status`` to see if there are such files.  Write and run tests!
+  (See the rest of this page.)
+
+* Commit regularly with ``hg commit``.  A one-line commit message is
+  fine.  We love to have tons of commits; make one as soon as you have
+  some progress, even if it is only some new test that doesn't pass yet,
+  or fixing things even if not all tests pass.  Step by step, you are
+  building the history of your changes, which is the point of a version
+  control system.  (There are commands like ``hg log`` and ``hg up``
+  that you should read about later, to learn how to navigate this
+  history.)
+
+* The commits stay on your machine until you do ``hg push`` to "push"
+  them back to the repo named in the file ``.hg/hgrc``.  Repos are
+  basically just collections of commits (a commit is also called a
+  changeset): there is one repo per url, plus one for each local copy on
+  each local machine.  The commands ``hg push`` and ``hg pull`` copy
+  commits around, with the goal that all repos in question end up with
+  the exact same set of commits.  By opposition, ``hg up`` only updates
+  the "working copy" by reading the local repository, i.e. it makes the
+  files that you see correspond to the latest (or any other) commit
+  locally present.
+
+* You should push often; there is no real reason not to.  Remember that
+  even if they are pushed, with the setup above, the commits are (1)
+  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
+  named.  Yes, they are publicly visible, but don't worry about someone
+  walking around the thousands of repos on bitbucket saying "hah, look
+  at the bad coding style of that guy".  Try to get into the mindset
+  that your work is not secret and it's fine that way.  We might not
+  accept it as is for PyPy, asking you instead to improve some things,
+  but we are not going to judge you.
+
+* The final step is to open a pull request, so that we know that you'd
+  like to merge that branch back to the original ``pypy/pypy`` repo.
+  This can also be done several times if you have interesting
+  intermediate states, but if you get there, then we're likely to
+  proceed to the next stage, which is...
+
+* Get a regular account for pushing directly to
+  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
+  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
+  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
+  then be pushed directly to the official repo, but (if you follow these
+  rules) they are still on a branch, and we can still review the
+  branches you want to merge.
+
+* If you get closer to the regular day-to-day development, you'll notice
+  that we generally push small changes as one or a few commits directly
+  to the branch ``default``.  Also, we often collaborate even if we are
+  on other branches, which do not really "belong" to anyone.  At this
+  point you'll need ``hg merge`` and learn how to resolve conflicts that
+  sometimes occur when two people try to push different commits in
+  parallel on the same branch.  But it is likely an issue for later ``:-)``
+
+.. _bitbucket: https://bitbucket.org/
+
+
+Architecture
+============
+
+PyPy has layers. The 100 miles view:
+
+* :ref:`RPython <rpython:language>` is the language in which we write interpreters. Not the entire
+  PyPy project is written in RPython, only the parts that are compiled in
+  the translation process. The interesting point is that RPython has no parser,
+  it's compiled from the live python objects, which makes it possible to do
+  all kinds of metaprogramming during import time. In short, Python is a meta
+  programming language for RPython.
+
+  The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+* The translation toolchain - this is the part that takes care of translating
+  RPython to flow graphs and then to C. There is more in the :doc:`architecture <architecture>`
+  document written about it.
+
+  It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+  and ``rtyper``.
+
+* Python Interpreter and modules
+
+  This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
+  interpreter for Python written in RPython.  The fact that it is
+  RPython is not apparent at first.  Built-in modules are written in
+  ``pypy/module/*``.  Some modules that CPython implements in C are
+  simply written in pure Python; they are in the top-level ``lib_pypy``
+  directory.  The standard library of Python (with a few changes to
+  accomodate PyPy) is in ``lib-python``.
+
+* :ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
+  interpreter written in RPython, rather than the user program that it
+  interprets.  As a result it applies to any interpreter, i.e. any
+  language.  But getting it to work correctly is not trivial: it
+  requires a small number of precise "hints" and possibly some small
+  refactorings of the interpreter.  The JIT itself also has several
+  almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+  optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+  residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+  that turns it into machine code.  Writing a new backend is a
+  traditional way to get into the project.
+
+* Garbage Collectors (GC): as you may notice if you are used to CPython's
+  C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+  :ref:`rpython:garbage-collection` is inserted
+  during translation.  Moreover, this is not reference counting; it is a real
+  GC written as more RPython code.  The best one we have so far is in
+  ``rpython/memory/gc/incminimark.py``.
+
+Layers
+------
+
+PyPy has layers. Just like Ogres or onions.
+Those layers help us keep the respective parts separated enough
+to be worked on independently and make the complexity manageable. This is,
+again, just a sanity requirement for such a complex project. For example writing
+a new optimization for the JIT usually does **not** involve touching a Python
+interpreter at all or the JIT assembler backend or the garbage collector.
+Instead it requires writing small tests in
+``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
+After that, you can just compile PyPy and things should just work.
+
+The short list of layers for further reading. For each of those layers, a good
+entry point is a test subdirectory in respective directories. It usually
+describes (better or worse) the interfaces between the submodules. For the
+``pypy`` subdirectory, most tests are small snippets of python programs that
+check for correctness (calls ``AppTestXxx``) that will call the appropriate
+part of the interpreter. For the ``rpython`` directory, most tests are small
+RPython interpreters that perform certain tasks. To see how they translate
+to low-level graphs, run them with ``--view``. To see small interpreters
+with a JIT compiler, use ``--viewloops`` option.
+
+* **python interpreter** - it's the part implemented in the ``pypy/`` directory.
+  It's implemented in RPython, which is a high level static language with
+  classes, garbage collection, just-in-time compiler generation and the ability
+  to call C. A cool part about it is that it can be run untranslated, so all
+  the tests are runnable without translating PyPy.
+
+  **interpreter** contains the interpreter core
+
+  **objspace** contains implementations of various objects exported to
+  the Python layer
+
+  **module** directory contains extension modules written in RPython
+
+* **rpython compiler** that resides in ``rpython/annotator`` and
+  ``rpython/rtyper`` directories. Consult `Getting Started with RPython`_
+  for further reading
+
+* **JIT generator** lives in ``rpython/jit`` directory. optimizations live
+  in ``rpython/jit/metainterp/optimizeopt``, the main JIT in
+  ``rpython/jit/metainterp`` (runtime part) and
+  ``rpython/jit/codewriter`` (translation-time part). Backends live in
+  ``rpython/jit/backend``.
+
+* **garbage collection** lives in ``rpython/memory``
+
+The rest of directories serve specific niche goal and are unlikely a good
+entry point.
+
+.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
+
+Where to start reading the sources
+----------------------------------
+
+PyPy is made from parts that are relatively independent of each other.
+You should start looking at the part that attracts you most (all paths are
+relative to the PyPy top level directory).  You may look at our :doc:`directory reference <dir-reference>`
+or start off at one of the following points:
+
+*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
+   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
+   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
+   function objects and argument passing in :source:`pypy/interpreter/function.py`
+   and :source:`pypy/interpreter/argument.py`, the object space interface
+   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
+   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
+   Core types supporting the bytecode interpreter are defined in :source:`pypy/interpreter/typedef.py`.
+
+*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
+   and grammar files that allow it to parse the syntax of various Python
+   versions. Once the grammar has been processed, the parser can be
+   translated by the above machinery into efficient code.
+
+*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
+   contains a modified version of the compiler package from CPython
+   that fixes some bugs and is translatable.
+
+*  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)
+
+Testing
+=======
+
+Test driven development
+-----------------------
+
+Instead, we practice a lot of test driven development. This is partly because
+of very high quality requirements for compilers and partly because there is
+simply no other way to get around such complex project, that will keep you sane.
+There are probably people out there who are smart enough not to need it, we're
+not one of those. You may consider familiarizing yourself with `pytest`_,
+since this is a tool we use for tests.
+This leads to the next issue:
+
+.. _pytest: http://pytest.org/
+
+py.test and the py lib
+----------------------
+
+The `py.test testing tool`_ drives all our testing needs.
+
+We use the `py library`_ for filesystem path manipulations, terminal
+writing, logging and some other support  functionality.
+
+You don't necessarily need to install these two libraries because
+we also ship them inlined in the PyPy source tree.
+
+.. _py library: http://pylib.readthedocs.org/
+
+Running PyPy's unit tests
+-------------------------
+
+PyPy development always was and is still thoroughly test-driven.
+We use the flexible `py.test testing tool`_ which you can `install independently
+<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
+
+The PyPy source tree comes with an inlined version of ``py.test``
+which you can invoke by typing::
+
+    python pytest.py -h
+
+This is usually equivalent to using an installed version::
+
+    py.test -h
+
+If you encounter problems with the installed version
+make sure you have the correct version installed which
+you can find out with the ``--version`` switch.
+
+You will need the `build requirements`_ to run tests successfully, since many of
+them compile little pieces of PyPy and then run the tests inside that minimal
+interpreter
+
+Now on to running some tests.  PyPy has many different test directories
+and you can use shell completion to point at directories or files::
+
+    py.test pypy/interpreter/test/test_pyframe.py
+
+    # or for running tests of a whole subdirectory
+    py.test pypy/interpreter/
+
+See `py.test usage and invocations`_ for some more generic info
+on how you can run tests.
+
+Beware trying to run "all" pypy tests by pointing to the root
+directory or even the top level subdirectory ``pypy``.  It takes
+hours and uses huge amounts of RAM and is not recommended.
+
+To run CPython regression tests you can point to the ``lib-python``
+directory::
+
+    py.test lib-python/2.7/test/test_datetime.py
+
+This will usually take a long time because this will run
+the PyPy Python interpreter on top of CPython.  On the plus
+side, it's usually still faster than doing a full translation
+and running the regression test with the translated PyPy Python
+interpreter.
+
+.. _py.test testing tool: http://pytest.org
+.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
+.. _`build requirements`: build.html#install-build-time-dependencies
+
+Tooling & Utilities
+===================
+
+If you are interested in the inner workings of the PyPy Python interpreter,
+there are some features of the untranslated Python interpreter that allow you
+to introspect its internals.
+
+
+Interpreter-level console
+-------------------------
+
+To start interpreting Python with PyPy, install a C compiler that is
+supported by distutils and use Python 2.7 or greater to run PyPy::
+
+    cd pypy
+    python bin/pyinteractive.py
+
+After a few seconds (remember: this is running on top of CPython), you should
+be at the PyPy prompt, which is the same as the Python prompt, but with an
+extra ">".
+
+If you press
+<Ctrl-C> on the console you enter the interpreter-level console, a
+usual CPython console.  You can then access internal objects of PyPy
+(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
+prompt with the prefix ``w_``::
+
+    >>>> a = 123
+    >>>> <Ctrl-C>
+    *** Entering interpreter-level console ***
+    >>> w_a
+    W_IntObject(123)
+
+The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
+
+    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
+    >>> <Ctrl-D>
+    *** Leaving interpreter-level console ***
+
+    KeyboardInterrupt
+    >>>> l
+    [1, 'abc']
+
+Note that the prompt of the interpreter-level console is only '>>>' since
+it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
+Linux) or <Ctrl-Z>, <Enter> (under Windows).
+
+Also note that not all modules are available by default in this mode (for
+example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
+``--withmod-...`` command line options.
+
+You may be interested in reading more about the distinction between
+:ref:`interpreter-level and app-level <interpreter-level>`.
+
+pyinteractive.py options
+------------------------
+
+To list the PyPy interpreter command line options, type::
+
+    cd pypy
+    python bin/pyinteractive.py --help
+
+pyinteractive.py supports most of the options that CPython supports too (in addition to a
+large amount of options that can be used to customize pyinteractive.py).
+As an example of using PyPy from the command line, you could type::
+
+    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
+
+Alternatively, as with regular Python, you can simply give a
+script name on the command line::
+
+    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
+
+See our :doc:`configuration sections <config/index>` for details about what all the commandline
+options do.
+
+
+.. _trace example:
+
+Tracing bytecode and operations on objects
+------------------------------------------
+
+You can use a simple tracing mode to monitor the interpretation of
+bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
+PyPy console::
+
+    >>>> __pytrace__ = 1
+    Tracing enabled
+    >>>> x = 5
+            <module>:           LOAD_CONST    0 (5)
+            <module>:           STORE_NAME    0 (x)
+            <module>:           LOAD_CONST    1 (None)
+            <module>:           RETURN_VALUE    0
+    >>>> x
+            <module>:           LOAD_NAME    0 (x)
+            <module>:           PRINT_EXPR    0
+    5
+            <module>:           LOAD_CONST    0 (None)
+            <module>:           RETURN_VALUE    0
+    >>>>
+
+
+Demos
+-----
+
+The `example-interpreter`_ repository contains an example interpreter
+written using the RPython translation toolchain.
+
+.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
+
+
+graphviz & pygame for flow graph viewing (highly recommended)
+-------------------------------------------------------------
+
+graphviz and pygame are both necessary if you want to look at generated flow
+graphs:
+
+	graphviz: http://www.graphviz.org/Download.php
+
+	pygame: http://www.pygame.org/download.shtml
+
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
deleted file mode 100644
--- a/pypy/doc/getting-started-dev.rst
+++ /dev/null
@@ -1,345 +0,0 @@
-Getting Started Developing With PyPy
-====================================
-
-.. contents::
-
-
-Using Mercurial
----------------
-
-PyPy development is based on Mercurial (hg).  If you are not used to
-version control, the cycle for a new PyPy contributor goes typically
-like this:
-
-* Make an account on bitbucket_.
-
-* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
-  icons).  You get a fork of the repository, e.g. in
-  https://bitbucket.org/yourname/pypy/.
-
-* Clone this new repo (i.e. the fork) to your local machine with the command 
-  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
-  operation but only ever needs to be done once.  See also 
-  http://pypy.org/download.html#building-from-source .
-  If you already cloned
-  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
-  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
-  your clone to contain the line ``default =
-  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
-  up``.  If you already have such a clone but don't want to change it,
-  you can clone that copy with ``hg clone /path/to/other/copy``, and
-  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
-
-* Now you have a complete copy of the PyPy repo.  Make a branch
-  with a command like ``hg branch name_of_your_branch``.
-
-* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
-  to make Mercurial aware of new files you added, e.g. new test files.
-  Use ``hg status`` to see if there are such files.  Write and run tests!
-  (See the rest of this page.)
-
-* Commit regularly with ``hg commit``.  A one-line commit message is
-  fine.  We love to have tons of commits; make one as soon as you have
-  some progress, even if it is only some new test that doesn't pass yet,
-  or fixing things even if not all tests pass.  Step by step, you are
-  building the history of your changes, which is the point of a version
-  control system.  (There are commands like ``hg log`` and ``hg up``
-  that you should read about later, to learn how to navigate this
-  history.)
-
-* The commits stay on your machine until you do ``hg push`` to "push"
-  them back to the repo named in the file ``.hg/hgrc``.  Repos are
-  basically just collections of commits (a commit is also called a
-  changeset): there is one repo per url, plus one for each local copy on
-  each local machine.  The commands ``hg push`` and ``hg pull`` copy
-  commits around, with the goal that all repos in question end up with
-  the exact same set of commits.  By opposition, ``hg up`` only updates
-  the "working copy" by reading the local repository, i.e. it makes the
-  files that you see correspond to the latest (or any other) commit
-  locally present.
-
-* You should push often; there is no real reason not to.  Remember that
-  even if they are pushed, with the setup above, the commits are (1)
-  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
-  named.  Yes, they are publicly visible, but don't worry about someone
-  walking around the thousands of repos on bitbucket saying "hah, look
-  at the bad coding style of that guy".  Try to get into the mindset
-  that your work is not secret and it's fine that way.  We might not
-  accept it as is for PyPy, asking you instead to improve some things,
-  but we are not going to judge you.
-
-* The final step is to open a pull request, so that we know that you'd
-  like to merge that branch back to the original ``pypy/pypy`` repo.
-  This can also be done several times if you have interesting
-  intermediate states, but if you get there, then we're likely to
-  proceed to the next stage, which is...
-
-* Get a regular account for pushing directly to
-  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
-  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
-  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
-  then be pushed directly to the official repo, but (if you follow these
-  rules) they are still on a branch, and we can still review the
-  branches you want to merge.
-
-* If you get closer to the regular day-to-day development, you'll notice
-  that we generally push small changes as one or a few commits directly
-  to the branch ``default``.  Also, we often collaborate even if we are
-  on other branches, which do not really "belong" to anyone.  At this
-  point you'll need ``hg merge`` and learn how to resolve conflicts that
-  sometimes occur when two people try to push different commits in
-  parallel on the same branch.  But it is likely an issue for later ``:-)``
-
-.. _bitbucket: https://bitbucket.org/
-
-
-Running PyPy's unit tests
--------------------------
-
-PyPy development always was and is still thoroughly test-driven.
-We use the flexible `py.test testing tool`_ which you can `install independently
-<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
-
-The PyPy source tree comes with an inlined version of ``py.test``
-which you can invoke by typing::
-
-    python pytest.py -h
-
-This is usually equivalent to using an installed version::
-
-    py.test -h
-
-If you encounter problems with the installed version
-make sure you have the correct version installed which
-you can find out with the ``--version`` switch.
-
-You will need the `build requirements`_ to run tests successfully, since many of
-them compile little pieces of PyPy and then run the tests inside that minimal
-interpreter
-
-Now on to running some tests.  PyPy has many different test directories
-and you can use shell completion to point at directories or files::
-
-    py.test pypy/interpreter/test/test_pyframe.py
-
-    # or for running tests of a whole subdirectory
-    py.test pypy/interpreter/
-
-See `py.test usage and invocations`_ for some more generic info
-on how you can run tests.
-
-Beware trying to run "all" pypy tests by pointing to the root
-directory or even the top level subdirectory ``pypy``.  It takes
-hours and uses huge amounts of RAM and is not recommended.
-
-To run CPython regression tests you can point to the ``lib-python``
-directory::
-
-    py.test lib-python/2.7/test/test_datetime.py
-
-This will usually take a long time because this will run
-the PyPy Python interpreter on top of CPython.  On the plus
-side, it's usually still faster than doing a full translation
-and running the regression test with the translated PyPy Python
-interpreter.
-
-.. _py.test testing tool: http://pytest.org
-.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
-.. _`build requirements`: build.html#install-build-time-dependencies
-
-Special Introspection Features of the Untranslated Python Interpreter
----------------------------------------------------------------------
-
-If you are interested in the inner workings of the PyPy Python interpreter,
-there are some features of the untranslated Python interpreter that allow you
-to introspect its internals.
-
-
-Interpreter-level console
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To start interpreting Python with PyPy, install a C compiler that is
-supported by distutils and use Python 2.7 or greater to run PyPy::
-
-    cd pypy
-    python bin/pyinteractive.py
-
-After a few seconds (remember: this is running on top of CPython), you should
-be at the PyPy prompt, which is the same as the Python prompt, but with an
-extra ">".
-
-If you press
-<Ctrl-C> on the console you enter the interpreter-level console, a
-usual CPython console.  You can then access internal objects of PyPy
-(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
-prompt with the prefix ``w_``::
-
-    >>>> a = 123
-    >>>> <Ctrl-C>
-    *** Entering interpreter-level console ***
-    >>> w_a
-    W_IntObject(123)
-
-The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
-
-    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
-    >>> <Ctrl-D>
-    *** Leaving interpreter-level console ***
-
-    KeyboardInterrupt
-    >>>> l
-    [1, 'abc']
-
-Note that the prompt of the interpreter-level console is only '>>>' since
-it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
-Linux) or <Ctrl-Z>, <Enter> (under Windows).
-
-Also note that not all modules are available by default in this mode (for
-example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
-``--withmod-...`` command line options.
-
-You may be interested in reading more about the distinction between
-:ref:`interpreter-level and app-level <interpreter-level>`.
-
-pyinteractive.py options
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-To list the PyPy interpreter command line options, type::
-
-    cd pypy
-    python bin/pyinteractive.py --help
-
-pyinteractive.py supports most of the options that CPython supports too (in addition to a
-large amount of options that can be used to customize pyinteractive.py).
-As an example of using PyPy from the command line, you could type::
-
-    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
-
-Alternatively, as with regular Python, you can simply give a
-script name on the command line::
-
-    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
-
-The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
-default almost none of them are, because initializing them takes time.
-If you want anyway to enable all built-in modules, you can use
-``--allworkingmodules``.
-
-See our :doc:`configuration sections <config/index>` for details about what all the commandline
-options do.
-
-
-.. _trace example:
-
-Tracing bytecode and operations on objects
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can use a simple tracing mode to monitor the interpretation of
-bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
-PyPy console::
-
-    >>>> __pytrace__ = 1
-    Tracing enabled
-    >>>> x = 5
-            <module>:           LOAD_CONST    0 (5)
-            <module>:           STORE_NAME    0 (x)
-            <module>:           LOAD_CONST    1 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>> x
-            <module>:           LOAD_NAME    0 (x)
-            <module>:           PRINT_EXPR    0 
-    5
-            <module>:           LOAD_CONST    0 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>>
-
-
-Demos
------
-
-The `example-interpreter`_ repository contains an example interpreter
-written using the RPython translation toolchain.
-
-.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
-
-
-Additional Tools for running (and hacking) PyPy
------------------------------------------------
-
-We use some optional tools for developing PyPy. They are not required to run
-the basic tests or to get an interactive PyPy prompt but they help to
-understand  and debug PyPy especially for the translation process.
-
-
-graphviz & pygame for flow graph viewing (highly recommended)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-graphviz and pygame are both necessary if you
-want to look at generated flow graphs:
-
-	graphviz: http://www.graphviz.org/Download.php
-
-	pygame: http://www.pygame.org/download.shtml
-
-
-py.test and the py lib
-~~~~~~~~~~~~~~~~~~~~~~
-
-The `py.test testing tool`_ drives all our testing needs.
-
-We use the `py library`_ for filesystem path manipulations, terminal
-writing, logging and some other support  functionality.
-
-You don't necessarily need to install these two libraries because
-we also ship them inlined in the PyPy source tree.
-
-.. _py library: http://pylib.readthedocs.org/
-
-
-Getting involved
-----------------
-
-PyPy employs an open development process.  You are invited to join our
-`pypy-dev mailing list`_ or look at the other :ref:`contact
-possibilities <contact>`.  Usually we give out commit rights fairly liberally, so if you
-want to do something with PyPy, you can become a committer. We also run frequent
-coding sprints which are separately announced and often happen around Python
-conferences such as EuroPython or PyCon. Upcoming events are usually announced
-on `the blog`_.
-
-.. _the blog: http://morepypy.blogspot.com
-.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
-
-
-.. _start-reading-sources:
-
-Where to start reading the sources
-----------------------------------
-
-PyPy is made from parts that are relatively independent of each other.
-You should start looking at the part that attracts you most (all paths are
-relative to the PyPy top level directory).  You may look at our :doc:`directory reference <dir-reference>`
-or start off at one of the following points:
-
-*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
-   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
-   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
-   function objects and argument passing in :source:`pypy/interpreter/function.py`
-   and :source:`pypy/interpreter/argument.py`, the object space interface
-   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
-   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
-   Core types supporting the bytecode interpreter are defined in :source:`pypy/interpreter/typedef.py`.
-
-*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
-   and grammar files that allow it to parse the syntax of various Python
-   versions. Once the grammar has been processed, the parser can be
-   translated by the above machinery into efficient code.
-
-*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
-   contains a modified version of the compiler package from CPython
-   that fixes some bugs and is translatable.
-
-*  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
-   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
-   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
-   as a first approximation.  (Some types have multiple implementations.)
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
deleted file mode 100644
--- a/pypy/doc/how-to-contribute.rst
+++ /dev/null
@@ -1,93 +0,0 @@
-How to contribute to PyPy
-=========================
-
-This page describes how to contribute to the PyPy project. The first thing
-to remember is that PyPy project is very different than most projects out there.
-It's also different from a classic compiler project, so academic courses
-about compilers often don't apply or lead in the wrong direction.
-
-
-Don't just hack
----------------
-
-The first and most important rule how not to contribute to PyPy is
-"just hacking". This won't work. There are two major reasons why not
--- build times are large and PyPy has very thick layer separation which
-make it harder to "just hack a feature".
-
-
-Test driven development
------------------------
-
-Instead, we practice a lot of test driven development. This is partly because
-of very high quality requirements for compilers and partly because there is
-simply no other way to get around such complex project, that will keep you sane.
-There are probably people out there who are smart enough not to need it, we're
-not one of those. You may consider familiarizing yourself with `pytest`_,
-since this is a tool we use for tests.
-This leads to the next issue:
-
-.. _pytest: http://pytest.org/
-
-
-Layers
-------
-
-PyPy has layers. Just like Ogres or onions.
-Those layers help us keep the respective parts separated enough
-to be worked on independently and make the complexity manageable. This is,
-again, just a sanity requirement for such a complex project. For example writing
-a new optimization for the JIT usually does **not** involve touching a Python
-interpreter at all or the JIT assembler backend or the garbage collector.
-Instead it requires writing small tests in
-``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
-After that, you can just compile PyPy and things should just work.
-
-The short list of layers for further reading. For each of those layers, a good
-entry point is a test subdirectory in respective directories. It usually
-describes (better or worse) the interfaces between the submodules. For the
-``pypy`` subdirectory, most tests are small snippets of python programs that
-check for correctness (calls ``AppTestXxx``) that will call the appropriate
-part of the interpreter. For the ``rpython`` directory, most tests are small
-RPython interpreters that perform certain tasks. To see how they translate
-to low-level graphs, run them with ``--view``. To see small interpreters
-with a JIT compiler, use ``--viewloops`` option.
-
-* **python interpreter** - it's the part implemented in the ``pypy/`` directory.
-  It's implemented in RPython, which is a high level static language with
-  classes, garbage collection, just-in-time compiler generation and the ability
-  to call C. A cool part about it is that it can be run untranslated, so all
-  the tests are runnable without translating PyPy.
-
-  **interpreter** contains the interpreter core
-
-  **objspace** contains implementations of various objects exported to
-  the Python layer
-
-  **module** directory contains extension modules written in RPython
-
-* **rpython compiler** that resides in ``rpython/annotator`` and
-  ``rpython/rtyper`` directories. Consult `Getting Started with RPython`_
-  for further reading
-
-* **JIT generator** lives in ``rpython/jit`` directory. optimizations live
-  in ``rpython/jit/metainterp/optimizeopt``, the main JIT in
-  ``rpython/jit/metainterp`` (runtime part) and
-  ``rpython/jit/codewriter`` (translation-time part). Backends live in
-  ``rpython/jit/backend``.
-
-* **garbage collection** lives in ``rpython/memory``
-
-The rest of directories serve specific niche goal and are unlikely a good
-entry point.
-
-
-More documentation
-------------------
-
-* `Getting Started Developing With PyPy`_
-
-* `Getting Started with RPython`_
-
-.. _`Getting Started Developing With PyPy`: getting-started-dev.html
-.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -9,7 +9,7 @@
 
 * If you're interested in trying PyPy out, check out the :doc:`installation instructions <install>`.
 
-* If you want to help develop PyPy, please have a look at :doc:`how to contribute <how-to-contribute>`
+* If you want to help develop PyPy, please have a look at :doc:`contributing <contributing>`
   and get in touch (:ref:`contact`)!
 
 All of the documentation and source code is available under the MIT license,
@@ -63,9 +63,7 @@
 .. toctree::
   :maxdepth: 1
 
-  getting-started-dev
-  how-to-contribute
-  you-want-to-help
+  contributing
   architecture
   configuration
   project-ideas
diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
deleted file mode 100644
--- a/pypy/doc/you-want-to-help.rst
+++ /dev/null
@@ -1,81 +0,0 @@
-You want to help with PyPy, now what?
-=====================================
-
-PyPy is a very large project that has a reputation of being hard to dive into.
-Some of this fame is warranted, some of it is purely accidental. There are three
-important lessons that everyone willing to contribute should learn:
-
-* PyPy has layers. There are many pieces of architecture that are very well
-  separated from each other. More about this below, but often the manifestation
-  of this is that things are at a different layer than you would expect them
-  to be. For example if you are looking for the JIT implementation, you will
-  not find it in the implementation of the Python programming language.
-
-* Because of the above, we are very serious about Test Driven Development.
-  It's not only what we believe in, but also that PyPy's architecture is
-  working very well with TDD in mind and not so well without it. Often
-  development means progressing in an unrelated corner, one unittest
-  at a time; and then flipping a giant switch, bringing it all together.
-  (It generally works out of the box.  If it doesn't, then we didn't
-  write enough unit tests.)  It's worth repeating - PyPy's
-  approach is great if you do TDD, and not so great otherwise.
-
-* PyPy uses an entirely different set of tools - most of them included
-  in the PyPy repository. There is no Makefile, nor autoconf. More below.
-
-
-Architecture
-------------
-
-PyPy has layers. The 100 miles view:
-
-* :ref:`RPython <rpython:language>` is the language in which we write interpreters. Not the entire
-  PyPy project is written in RPython, only the parts that are compiled in
-  the translation process. The interesting point is that RPython has no parser,
-  it's compiled from the live python objects, which makes it possible to do
-  all kinds of metaprogramming during import time. In short, Python is a meta
-  programming language for RPython.
-
-  The RPython standard library is to be found in the ``rlib`` subdirectory.
-
-* The translation toolchain - this is the part that takes care of translating
-  RPython to flow graphs and then to C. There is more in the :doc:`architecture <architecture>`
-  document written about it.
-
-  It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
-  and ``rtyper``.
-
-* Python Interpreter and modules
-
-  This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
-  interpreter for Python written in RPython.  The fact that it is
-  RPython is not apparent at first.  Built-in modules are written in
-  ``pypy/module/*``.  Some modules that CPython implements in C are
-  simply written in pure Python; they are in the top-level ``lib_pypy``
-  directory.  The standard library of Python (with a few changes to
-  accomodate PyPy) is in ``lib-python``.
-
-* :ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
-  interpreter written in RPython, rather than the user program that it
-  interprets.  As a result it applies to any interpreter, i.e. any
-  language.  But getting it to work correctly is not trivial: it
-  requires a small number of precise "hints" and possibly some small
-  refactorings of the interpreter.  The JIT itself also has several
-  almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
-  optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
-  residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
-  that turns it into machine code.  Writing a new backend is a
-  traditional way to get into the project.
-
-* Garbage Collectors (GC): as you may notice if you are used to CPython's
-  C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
-  :ref:`rpython:garbage-collection` is inserted
-  during translation.  Moreover, this is not reference counting; it is a real
-  GC written as more RPython code.  The best one we have so far is in
-  ``rpython/memory/gc/incminimark.py``.
-
-
-Toolset
--------
-
-xxx

From pypy.commits at gmail.com  Wed May  9 13:29:20 2018
From: pypy.commits at gmail.com (mwjackson)
Date: Wed, 09 May 2018 10:29:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: added first contrib & deduplicated
 layers section
Message-ID: <5af32ff0.1c69fb81.c3ee4.2c8d@mx.google.com>

Author: Matt Jackson <email at mwjackson.net>
Branch: 
Changeset: r94509:1276e6548105
Date: 2018-04-22 08:11 +0100
http://bitbucket.org/pypy/pypy/changeset/1276e6548105/

Log:	added first contrib & deduplicated layers section

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -91,3 +91,5 @@
 ^release/
 ^rpython/_cache$
 
+.venv
+.venv-docs
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
--- a/pypy/doc/contributing.rst
+++ b/pypy/doc/contributing.rst
@@ -32,20 +32,20 @@
 Getting involved
 ----------------
 
-PyPy employs an open development process.  You are invited to join our
-`pypy-dev mailing list`_, our IRC channel (#pypy on freenode) or look at the
-other :ref:`contact possibilities <contact>`.  Usually we give out commit
-rights fairly liberally, so if you want to do something with PyPy, you can
-become a committer. We also run frequent coding sprints which are separately
-announced and often happen around Python conferences such as EuroPython or
-PyCon. Upcoming events are usually announced on `the blog`_.
+PyPy employs relatively standard open-source development process.  You are
+invited to join our `pypy-dev mailing list`_, our IRC channel (#pypy on freenode)
+or look at the other :ref:`contact possibilities <contact>`.  Usually we give
+out commit rights fairly liberally, so if you want to do something with PyPy,
+you can become a committer. We also run frequent coding sprints which are
+separately announced and often happen around Python conferences such as
+EuroPython or PyCon. Upcoming events are usually announced on `the blog`_.
 
 .. _the blog: http://morepypy.blogspot.com
 .. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
 
 
-Hacking
---------
+Your first contribution
+-----------------------
 
 The first and most important rule how _not_ to contribute to PyPy is
 "just hacking". This won't work. There are two major reasons why not
@@ -53,15 +53,28 @@
 make it harder to "just hack a feature". Instead, reach out on the dev mailing
 list or the IRC channel, and we're more than happy to help! :)
 
+Some ideas for first contributions are:
+
+* Documentation - this will give you an understanding of the pypy architecture
+* Test failures - find a failing test in the `nightly builds`_, and fix it
+* Missing language features - these are listed in our `issue tracker`_
+
+.. _nightly builds: http://buildbot.pypy.org/nightly/
+.. _issue tracker: https://bitbucket.org/pypy/pypy/issues
+
 Source Control
 ==============
 
 Using Mercurial
 ---------------
 
-PyPy development is based on Mercurial (hg).  If you are not used to
-version control, the cycle for a new PyPy contributor goes typically
-like this:
+PyPy development is based a typical fork/pull request based workflow, centered
+around Mercurial (hg). If you have not used this workflow before, a good
+can be found here:
+
+    https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow
+
+The cycle for a new PyPy contributor goes typically like this:
 
 * Make an account on bitbucket_.
 
@@ -69,7 +82,7 @@
   icons).  You get a fork of the repository, e.g. in
   https://bitbucket.org/yourname/pypy/.
 
-* Clone this new repo (i.e. the fork) to your local machine with the command
+* Clone your new repo (i.e. the fork) to your local machine with the command
   ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
   operation but only ever needs to be done once.  See also
   http://pypy.org/download.html#building-from-source .
@@ -148,56 +161,6 @@
 Architecture
 ============
 
-PyPy has layers. The 100 miles view:
-
-* :ref:`RPython <rpython:language>` is the language in which we write interpreters. Not the entire
-  PyPy project is written in RPython, only the parts that are compiled in
-  the translation process. The interesting point is that RPython has no parser,
-  it's compiled from the live python objects, which makes it possible to do
-  all kinds of metaprogramming during import time. In short, Python is a meta
-  programming language for RPython.
-
-  The RPython standard library is to be found in the ``rlib`` subdirectory.
-
-* The translation toolchain - this is the part that takes care of translating
-  RPython to flow graphs and then to C. There is more in the :doc:`architecture <architecture>`
-  document written about it.
-
-  It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
-  and ``rtyper``.
-
-* Python Interpreter and modules
-
-  This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
-  interpreter for Python written in RPython.  The fact that it is
-  RPython is not apparent at first.  Built-in modules are written in
-  ``pypy/module/*``.  Some modules that CPython implements in C are
-  simply written in pure Python; they are in the top-level ``lib_pypy``
-  directory.  The standard library of Python (with a few changes to
-  accomodate PyPy) is in ``lib-python``.
-
-* :ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
-  interpreter written in RPython, rather than the user program that it
-  interprets.  As a result it applies to any interpreter, i.e. any
-  language.  But getting it to work correctly is not trivial: it
-  requires a small number of precise "hints" and possibly some small
-  refactorings of the interpreter.  The JIT itself also has several
-  almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
-  optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
-  residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
-  that turns it into machine code.  Writing a new backend is a
-  traditional way to get into the project.
-
-* Garbage Collectors (GC): as you may notice if you are used to CPython's
-  C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
-  :ref:`rpython:garbage-collection` is inserted
-  during translation.  Moreover, this is not reference counting; it is a real
-  GC written as more RPython code.  The best one we have so far is in
-  ``rpython/memory/gc/incminimark.py``.
-
-Layers
-------
-
 PyPy has layers. Just like Ogres or onions.
 Those layers help us keep the respective parts separated enough
 to be worked on independently and make the complexity manageable. This is,
@@ -218,43 +181,74 @@
 to low-level graphs, run them with ``--view``. To see small interpreters
 with a JIT compiler, use ``--viewloops`` option.
 
-* **python interpreter** - it's the part implemented in the ``pypy/`` directory.
-  It's implemented in RPython, which is a high level static language with
-  classes, garbage collection, just-in-time compiler generation and the ability
-  to call C. A cool part about it is that it can be run untranslated, so all
-  the tests are runnable without translating PyPy.
+Layers
+------
 
-  **interpreter** contains the interpreter core
+RPython
+~~~~~~~
+:ref:`RPython <rpython:language>` is the language in which we write interpreters.
+Not the entire PyPy project is written in RPython, only the parts that are
+compiled in the translation process. The interesting point is that RPython
+has no parser, it's compiled from the live python objects, which makes it
+possible to do all kinds of metaprogramming during import time. In short,
+Python is a meta programming language for RPython.
 
-  **objspace** contains implementations of various objects exported to
-  the Python layer
+The RPython standard library is to be found in the ``rlib`` subdirectory.
 
-  **module** directory contains extension modules written in RPython
+Consult `Getting Started with RPython`_ for further reading
 
-* **rpython compiler** that resides in ``rpython/annotator`` and
-  ``rpython/rtyper`` directories. Consult `Getting Started with RPython`_
-  for further reading
+Translation
+~~~~~~~~~~~
+The translation toolchain - this is the part that takes care of translating
+RPython to flow graphs and then to C. There is more in the
+:doc:`architecture <architecture>` document written about it.
 
-* **JIT generator** lives in ``rpython/jit`` directory. optimizations live
-  in ``rpython/jit/metainterp/optimizeopt``, the main JIT in
-  ``rpython/jit/metainterp`` (runtime part) and
-  ``rpython/jit/codewriter`` (translation-time part). Backends live in
-  ``rpython/jit/backend``.
+It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+and ``rtyper``.
 
-* **garbage collection** lives in ``rpython/memory``
+PyPy Interpreter
+~~~~~~~~~~~~~~~~
+This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
+interpreter for Python written in RPython.  The fact that it is
+RPython is not apparent at first.  Built-in modules are written in
+``pypy/module/*``.  Some modules that CPython implements in C are
+simply written in pure Python; they are in the top-level ``lib_pypy``
+directory.  The standard library of Python (with a few changes to
+accomodate PyPy) is in ``lib-python``.
 
-The rest of directories serve specific niche goal and are unlikely a good
-entry point.
+JIT Compiler
+~~~~~~~~~~~~
+:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
+interpreter written in RPython, rather than the user program that it
+interprets.  As a result it applies to any interpreter, i.e. any
+language.  But getting it to work correctly is not trivial: it
+requires a small number of precise "hints" and possibly some small
+refactorings of the interpreter.  The JIT itself also has several
+almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+that turns it into machine code.  Writing a new backend is a
+traditional way to get into the project.
+
+Garbage Collectors
+~~~~~~~~~~~~~~~~~~
+Garbage Collectors (GC): as you may notice if you are used to CPython's
+C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+:ref:`rpython:garbage-collection` is inserted
+during translation.  Moreover, this is not reference counting; it is a real
+GC written as more RPython code.  The best one we have so far is in
+``rpython/memory/gc/incminimark.py``.
 
 .. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
 
-Where to start reading the sources
-----------------------------------
+Where to start?
+---------------
 
 PyPy is made from parts that are relatively independent of each other.
 You should start looking at the part that attracts you most (all paths are
-relative to the PyPy top level directory).  You may look at our :doc:`directory reference <dir-reference>`
-or start off at one of the following points:
+relative to the PyPy top level directory).  You may look at our
+:doc:`directory reference <dir-reference>` or start off at one of the following
+points:
 
 *  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
    in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
@@ -263,7 +257,8 @@
    and :source:`pypy/interpreter/argument.py`, the object space interface
    definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
    :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
-   Core types supporting the bytecode interpreter are defined in :source:`pypy/interpreter/typedef.py`.
+   Core types supporting the bytecode interpreter are defined in
+   :source:`pypy/interpreter/typedef.py`.
 
 *  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
    and grammar files that allow it to parse the syntax of various Python
@@ -274,7 +269,8 @@
    contains a modified version of the compiler package from CPython
    that fixes some bugs and is translatable.
 
-*  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
+*  :source:`pypy/objspace/std` contains the
+   :ref:`Standard object space <standard-object-space>`.  The main file
    is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
    ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
    as a first approximation.  (Some types have multiple implementations.)
@@ -484,7 +480,7 @@
 graphviz and pygame are both necessary if you want to look at generated flow
 graphs:
 
-	graphviz: http://www.graphviz.org/Download.php
+    graphviz: http://www.graphviz.org/Download.php
 
-	pygame: http://www.pygame.org/download.shtml
+    pygame: http://www.pygame.org/download.shtml
 

From pypy.commits at gmail.com  Wed May  9 13:29:22 2018
From: pypy.commits at gmail.com (mwjackson)
Date: Wed, 09 May 2018 10:29:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: moved architecture overview to
 architecture section
Message-ID: <5af32ff2.1c69fb81.79179.1c3a@mx.google.com>

Author: Matt Jackson <email at mwjackson.net>
Branch: 
Changeset: r94510:89bbc53df999
Date: 2018-04-22 08:29 +0100
http://bitbucket.org/pypy/pypy/changeset/89bbc53df999/

Log:	moved architecture overview to architecture section

	added short building & coding style intro

diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -73,3 +73,63 @@
 This division between bytecode evaluator and object space gives a lot of
 flexibility.  One can plug in different :doc:`object spaces <objspace>` to get
 different or enriched behaviours of the Python objects.
+
+Layers
+------
+
+RPython
+~~~~~~~
+:ref:`RPython <rpython:language>` is the language in which we write interpreters.
+Not the entire PyPy project is written in RPython, only the parts that are
+compiled in the translation process. The interesting point is that RPython
+has no parser, it's compiled from the live python objects, which makes it
+possible to do all kinds of metaprogramming during import time. In short,
+Python is a meta programming language for RPython.
+
+The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+Consult `Getting Started with RPython`_ for further reading
+
+Translation
+~~~~~~~~~~~
+The translation toolchain - this is the part that takes care of translating
+RPython to flow graphs and then to C. There is more in the
+:doc:`architecture <architecture>` document written about it.
+
+It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+and ``rtyper``.
+
+PyPy Interpreter
+~~~~~~~~~~~~~~~~
+This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
+interpreter for Python written in RPython.  The fact that it is
+RPython is not apparent at first.  Built-in modules are written in
+``pypy/module/*``.  Some modules that CPython implements in C are
+simply written in pure Python; they are in the top-level ``lib_pypy``
+directory.  The standard library of Python (with a few changes to
+accomodate PyPy) is in ``lib-python``.
+
+JIT Compiler
+~~~~~~~~~~~~
+:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
+interpreter written in RPython, rather than the user program that it
+interprets.  As a result it applies to any interpreter, i.e. any
+language.  But getting it to work correctly is not trivial: it
+requires a small number of precise "hints" and possibly some small
+refactorings of the interpreter.  The JIT itself also has several
+almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+that turns it into machine code.  Writing a new backend is a
+traditional way to get into the project.
+
+Garbage Collectors
+~~~~~~~~~~~~~~~~~~
+Garbage Collectors (GC): as you may notice if you are used to CPython's
+C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+:ref:`rpython:garbage-collection` is inserted
+during translation.  Moreover, this is not reference counting; it is a real
+GC written as more RPython code.  The best one we have so far is in
+``rpython/memory/gc/incminimark.py``.
+
+.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
--- a/pypy/doc/contributing.rst
+++ b/pypy/doc/contributing.rst
@@ -32,7 +32,7 @@
 Getting involved
 ----------------
 
-PyPy employs relatively standard open-source development process.  You are
+PyPy employs a relatively standard open-source development process.  You are
 invited to join our `pypy-dev mailing list`_, our IRC channel (#pypy on freenode)
 or look at the other :ref:`contact possibilities <contact>`.  Usually we give
 out commit rights fairly liberally, so if you want to do something with PyPy,
@@ -69,8 +69,8 @@
 ---------------
 
 PyPy development is based a typical fork/pull request based workflow, centered
-around Mercurial (hg). If you have not used this workflow before, a good
-can be found here:
+around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
+before, a good introduction can be found here:
 
     https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow
 
@@ -161,85 +161,16 @@
 Architecture
 ============
 
-PyPy has layers. Just like Ogres or onions.
-Those layers help us keep the respective parts separated enough
-to be worked on independently and make the complexity manageable. This is,
-again, just a sanity requirement for such a complex project. For example writing
-a new optimization for the JIT usually does **not** involve touching a Python
-interpreter at all or the JIT assembler backend or the garbage collector.
-Instead it requires writing small tests in
+PyPy has layers. Just like ogres or onions. Those layers help us keep the
+respective parts separated enough to be worked on independently and make the
+complexity manageable. This is, again, just a sanity requirement for such
+a complex project. For example writing a new optimization for the JIT usually
+does **not** involve touching a Python interpreter at all or the JIT assembler
+backend or the garbage collector. Instead it requires writing small tests in
 ``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
 After that, you can just compile PyPy and things should just work.
 
-The short list of layers for further reading. For each of those layers, a good
-entry point is a test subdirectory in respective directories. It usually
-describes (better or worse) the interfaces between the submodules. For the
-``pypy`` subdirectory, most tests are small snippets of python programs that
-check for correctness (calls ``AppTestXxx``) that will call the appropriate
-part of the interpreter. For the ``rpython`` directory, most tests are small
-RPython interpreters that perform certain tasks. To see how they translate
-to low-level graphs, run them with ``--view``. To see small interpreters
-with a JIT compiler, use ``--viewloops`` option.
-
-Layers
-------
-
-RPython
-~~~~~~~
-:ref:`RPython <rpython:language>` is the language in which we write interpreters.
-Not the entire PyPy project is written in RPython, only the parts that are
-compiled in the translation process. The interesting point is that RPython
-has no parser, it's compiled from the live python objects, which makes it
-possible to do all kinds of metaprogramming during import time. In short,
-Python is a meta programming language for RPython.
-
-The RPython standard library is to be found in the ``rlib`` subdirectory.
-
-Consult `Getting Started with RPython`_ for further reading
-
-Translation
-~~~~~~~~~~~
-The translation toolchain - this is the part that takes care of translating
-RPython to flow graphs and then to C. There is more in the
-:doc:`architecture <architecture>` document written about it.
-
-It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
-and ``rtyper``.
-
-PyPy Interpreter
-~~~~~~~~~~~~~~~~
-This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
-interpreter for Python written in RPython.  The fact that it is
-RPython is not apparent at first.  Built-in modules are written in
-``pypy/module/*``.  Some modules that CPython implements in C are
-simply written in pure Python; they are in the top-level ``lib_pypy``
-directory.  The standard library of Python (with a few changes to
-accomodate PyPy) is in ``lib-python``.
-
-JIT Compiler
-~~~~~~~~~~~~
-:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
-interpreter written in RPython, rather than the user program that it
-interprets.  As a result it applies to any interpreter, i.e. any
-language.  But getting it to work correctly is not trivial: it
-requires a small number of precise "hints" and possibly some small
-refactorings of the interpreter.  The JIT itself also has several
-almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
-optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
-residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
-that turns it into machine code.  Writing a new backend is a
-traditional way to get into the project.
-
-Garbage Collectors
-~~~~~~~~~~~~~~~~~~
-Garbage Collectors (GC): as you may notice if you are used to CPython's
-C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
-:ref:`rpython:garbage-collection` is inserted
-during translation.  Moreover, this is not reference counting; it is a real
-GC written as more RPython code.  The best one we have so far is in
-``rpython/memory/gc/incminimark.py``.
-
-.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
+Further Reading: :doc:`architecture <architecture>`
 
 Where to start?
 ---------------
@@ -275,6 +206,25 @@
    ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
    as a first approximation.  (Some types have multiple implementations.)
 
+Building
+========
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Further Reading: :doc:`Build <build>`
+
+Coding Guide
+============
+
+As well as the usual pep8 and formatting standards, there are a number of
+naming conventions and coding styles that are important to understand before
+browsing the source.
+
+Further Reading: :doc:`Coding Guide <coding-guide>`
+
 Testing
 =======
 

From pypy.commits at gmail.com  Wed May  9 13:29:24 2018
From: pypy.commits at gmail.com (mwjackson)
Date: Wed, 09 May 2018 10:29:24 -0700 (PDT)
Subject: [pypy-commit] pypy default: improved context,
 and source control sections
Message-ID: <5af32ff4.1c69fb81.e858d.1b17@mx.google.com>

Author: Matt Jackson <email at mwjackson.net>
Branch: 
Changeset: r94511:1605da2b2315
Date: 2018-04-22 08:46 +0100
http://bitbucket.org/pypy/pypy/changeset/1605da2b2315/

Log:	improved context, and source control sections

diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
--- a/pypy/doc/contributing.rst
+++ b/pypy/doc/contributing.rst
@@ -25,21 +25,27 @@
 * PyPy uses an entirely different set of tools - most of them included
   in the PyPy repository. There is no Makefile, nor autoconf. More below.
 
-The first thing to remember is that PyPy project is very different than most projects out there.
-It's also different from a classic compiler project, so academic courses
-about compilers often don't apply or lead in the wrong direction.
+The first thing to remember is that PyPy project is very different than most
+projects out there. It's also different from a classic compiler project,
+so academic courses about compilers often don't apply or lead in the wrong
+direction. However, if you want to understand how designing & building a runtime
+works in the real world then this is a great project!
 
 Getting involved
 ----------------
 
-PyPy employs a relatively standard open-source development process.  You are
-invited to join our `pypy-dev mailing list`_, our IRC channel (#pypy on freenode)
-or look at the other :ref:`contact possibilities <contact>`.  Usually we give
-out commit rights fairly liberally, so if you want to do something with PyPy,
-you can become a committer. We also run frequent coding sprints which are
-separately announced and often happen around Python conferences such as
+PyPy employs a relatively standard open-source development process. You are
+encouraged as a first step to join our `pypy-dev mailing list`_ and IRC channel,
+details of which can be found in our :ref:`contact <contact>` section. The folks
+there are very friendly, and can point you in the right direction.
+
+We give out commit rights usually fairly liberally, so if you want to do something
+with PyPy, you can become a committer. We also run frequent coding sprints which
+are separately announced and often happen around Python conferences such as
 EuroPython or PyCon. Upcoming events are usually announced on `the blog`_.
 
+Further Reading: :ref:`Contact <contact>`
+
 .. _the blog: http://morepypy.blogspot.com
 .. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
 
@@ -47,11 +53,16 @@
 Your first contribution
 -----------------------
 
-The first and most important rule how _not_ to contribute to PyPy is
-"just hacking". This won't work. There are two major reasons why not
--- build times are large and PyPy has very thick layer separation which
-make it harder to "just hack a feature". Instead, reach out on the dev mailing
-list or the IRC channel, and we're more than happy to help! :)
+The first and most important rule how **not** to contribute to PyPy is
+"just hacking a feature". This won't work, and you'll find your PR will typically
+require a lot of re-work. There are a few reasons why not:
+
+* build times are large
+* PyPy has very thick layer separation
+* context of the cPython runtime is often required
+
+Instead, reach out on the dev mailing list or the IRC channel, and we're more
+than happy to help! :)
 
 Some ideas for first contributions are:
 
@@ -65,9 +76,6 @@
 Source Control
 ==============
 
-Using Mercurial
----------------
-
 PyPy development is based a typical fork/pull request based workflow, centered
 around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
 before, a good introduction can be found here:
@@ -76,6 +84,9 @@
 
 The cycle for a new PyPy contributor goes typically like this:
 
+Fork & Clone
+------------
+
 * Make an account on bitbucket_.
 
 * Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
@@ -98,6 +109,9 @@
 * Now you have a complete copy of the PyPy repo.  Make a branch
   with a command like ``hg branch name_of_your_branch``.
 
+Edit
+----
+
 * Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
   to make Mercurial aware of new files you added, e.g. new test files.
   Use ``hg status`` to see if there are such files.  Write and run tests!
@@ -133,6 +147,9 @@
   accept it as is for PyPy, asking you instead to improve some things,
   but we are not going to judge you.
 
+Pull Request
+------------
+
 * The final step is to open a pull request, so that we know that you'd
   like to merge that branch back to the original ``pypy/pypy`` repo.
   This can also be done several times if you have interesting
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -57,8 +57,8 @@
 
 .. _developing-pypy:
 
-Development documentation
--------------------------
+Development
+-----------
 
 .. toctree::
   :maxdepth: 1
@@ -70,11 +70,6 @@
   project-documentation
   how-to-release
 
-.. TODO: audit ^^
-
-
-.. TODO: Fill this in
-
 
 Further resources
 -----------------
@@ -89,7 +84,6 @@
   index-of-whatsnew
   contributor
 
-
 .. _contact:
 
 Contact

From pypy.commits at gmail.com  Wed May  9 13:29:30 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 09 May 2018 10:29:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: add man page from pypy3
Message-ID: <5af32ffa.1c69fb81.ce632.31bf@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94514:7513d40a257b
Date: 2018-05-09 19:24 +0300
http://bitbucket.org/pypy/pypy/changeset/7513d40a257b/

Log:	add man page from pypy3

diff --git a/pypy/doc/man/pypy3.1.rst b/pypy/doc/man/pypy3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/man/pypy3.1.rst
@@ -0,0 +1,135 @@
+=======
+ pypy3
+=======
+
+.. note: this is turned into a regular man page "pypy3.1" by
+   doing "make man" in pypy/doc/
+
+SYNOPSIS
+========
+
+``pypy3`` [*options*]
+[``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ]
+[*arg*\ ...]
+
+OPTIONS
+=======
+
+-i
+    Inspect interactively after running script.
+
+-O
+    Skip assert statements.
+
+-OO
+    Remove docstrings when importing modules in addition to ``-O``.
+
+-c CMD
+    Program passed in as ``CMD`` (terminates option list).
+
+-S
+    Do not ``import site`` on initialization.
+
+-s
+    Don't add the user site directory to `sys.path`.
+
+-u
+    Unbuffered binary ``stdout`` and ``stderr``.
+
+-h, --help
+    Show a help message and exit.
+
+-m MOD
+    Library module to be run as a script (terminates option list).
+
+-W ARG
+    Warning control (*arg* is *action*:*message*:*category*:*module*:*lineno*).
+
+-E
+    Ignore environment variables (such as ``PYTHONPATH``).
+
+-B
+    Disable writing bytecode (``.pyc``) files.
+
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
+--version
+    Print the PyPy version.
+
+--info
+    Print translation information about this PyPy executable.
+
+--jit ARG
+    Low level JIT parameters.  Mostly internal.  Run ``--jit help``
+    for more information.
+
+ENVIRONMENT
+===========
+
+``PYTHONPATH``
+    Add directories to pypy3's module search path.
+    The format is the same as shell's ``PATH``.
+
+``PYTHONSTARTUP``
+    A script referenced by this variable will be executed before the
+    first prompt is displayed, in interactive mode.
+
+``PYTHONDONTWRITEBYTECODE``
+    If set to a non-empty value, equivalent to the ``-B`` option.
+    Disable writing ``.pyc`` files.
+
+``PYTHONINSPECT``
+    If set to a non-empty value, equivalent to the ``-i`` option.
+    Inspect interactively after running the specified script.
+
+``PYTHONIOENCODING``
+    If this is set, it overrides the encoding used for
+    *stdin*/*stdout*/*stderr*.
+    The syntax is *encodingname*:*errorhandler*
+    The *errorhandler* part is optional and has the same meaning as in
+    `str.encode`.
+
+``PYTHONNOUSERSITE``
+    If set to a non-empty value, equivalent to the ``-s`` option.
+    Don't add the user site directory to `sys.path`.
+
+``PYTHONWARNINGS``
+    If set, equivalent to the ``-W`` option (warning control).
+    The value should be a comma-separated list of ``-W`` parameters.
+
+``PYPYLOG``
+    If set to a non-empty value, enable logging, the format is:
+
+    *fname* or *+fname*
+        logging for profiling: includes all
+        ``debug_start``/``debug_stop`` but not any nested
+        ``debug_print``.
+        *fname* can be ``-`` to log to *stderr*.
+        The *+fname* form can be used if there is a *:* in fname
+
+    ``:``\ *fname*
+        Full logging, including ``debug_print``.
+
+    *prefix*\ ``:``\ *fname*
+        Conditional logging.
+        Multiple prefixes can be specified, comma-separated.
+        Only sections whose name match the prefix will be logged.
+
+    ``PYPYLOG=jit-log-opt,jit-backend:logfile`` will
+    generate a log suitable for *jitviewer*, a tool for debugging
+    performance issues under PyPy.
+
+``PYPY_IRC_TOPIC``
+    If set to a non-empty value, print a random #pypy IRC
+    topic at startup of interactive mode.
+
+
+.. include:: ../gc_info.rst
+   :start-line: 7
+
+SEE ALSO
+========
+
+**python3**\ (1)

From pypy.commits at gmail.com  Wed May  9 13:29:26 2018
From: pypy.commits at gmail.com (mwjackson)
Date: Wed, 09 May 2018 10:29:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: removed unnecessary hgignore lines
Message-ID: <5af32ff6.1c69fb81.aec57.76a9@mx.google.com>

Author: Matt Jackson <email at mwjackson.net>
Branch: 
Changeset: r94512:23344fc0d549
Date: 2018-04-23 10:09 +0100
http://bitbucket.org/pypy/pypy/changeset/23344fc0d549/

Log:	removed unnecessary hgignore lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -91,5 +91,3 @@
 ^release/
 ^rpython/_cache$
 
-.venv
-.venv-docs

From pypy.commits at gmail.com  Wed May  9 13:29:31 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 09 May 2018 10:29:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: rearrange indexing,
 deprecate older documents
Message-ID: <5af32ffb.8cd51c0a.188a1.a933@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94515:7e57ef5dd229
Date: 2018-05-09 20:24 +0300
http://bitbucket.org/pypy/pypy/changeset/7e57ef5dd229/

Log:	rearrange indexing, deprecate older documents

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -66,9 +66,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '5.8'
+version = '6.0'
 # The full version, including alpha/beta/rc tags.
-release = '5.8.0'
+release = '6.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
--- a/pypy/doc/contributing.rst
+++ b/pypy/doc/contributing.rst
@@ -32,7 +32,7 @@
 works in the real world then this is a great project!
 
 Getting involved
-----------------
+^^^^^^^^^^^^^^^^
 
 PyPy employs a relatively standard open-source development process. You are
 encouraged as a first step to join our `pypy-dev mailing list`_ and IRC channel,
@@ -51,7 +51,7 @@
 
 
 Your first contribution
------------------------
+^^^^^^^^^^^^^^^^^^^^^^^
 
 The first and most important rule how **not** to contribute to PyPy is
 "just hacking a feature". This won't work, and you'll find your PR will typically
@@ -74,7 +74,7 @@
 .. _issue tracker: https://bitbucket.org/pypy/pypy/issues
 
 Source Control
-==============
+--------------
 
 PyPy development is based a typical fork/pull request based workflow, centered
 around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
@@ -176,7 +176,7 @@
 
 
 Architecture
-============
+^^^^^^^^^^^^
 
 PyPy has layers. Just like ogres or onions. Those layers help us keep the
 respective parts separated enough to be worked on independently and make the
@@ -224,7 +224,7 @@
    as a first approximation.  (Some types have multiple implementations.)
 
 Building
-========
+^^^^^^^^
 
 For building PyPy, we recommend installing a pre-built PyPy first (see
 :doc:`install`). It is possible to build PyPy with CPython, but it will take a
@@ -234,7 +234,7 @@
 Further Reading: :doc:`Build <build>`
 
 Coding Guide
-============
+------------
 
 As well as the usual pep8 and formatting standards, there are a number of
 naming conventions and coding styles that are important to understand before
@@ -243,7 +243,7 @@
 Further Reading: :doc:`Coding Guide <coding-guide>`
 
 Testing
-=======
+^^^^^^^
 
 Test driven development
 -----------------------
@@ -326,7 +326,7 @@
 .. _`build requirements`: build.html#install-build-time-dependencies
 
 Tooling & Utilities
-===================
+^^^^^^^^^^^^^^^^^^^
 
 If you are interested in the inner workings of the PyPy Python interpreter,
 there are some features of the untranslated Python interpreter that allow you
@@ -433,7 +433,7 @@
 
 
 Demos
------
+^^^^^
 
 The `example-interpreter`_ repository contains an example interpreter
 written using the RPython translation toolchain.
@@ -442,7 +442,7 @@
 
 
 graphviz & pygame for flow graph viewing (highly recommended)
--------------------------------------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 graphviz and pygame are both necessary if you want to look at generated flow
 graphs:
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -1,5 +1,5 @@
-Embedding PyPy
-==============
+Embedding PyPy (DEPRECATED)
+===========================
 
 PyPy has a very minimal and a very strange embedding interface, based on
 the usage of `cffi`_ and the philosophy that Python is a better language than
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -31,6 +31,7 @@
   introduction
   install
   build
+  windows
   faq
 
 
@@ -44,18 +45,12 @@
 
   cpython_differences
   extending
-  embedding
   gc_info
   jit-hooks
   stackless
   __pypy__-module
-  objspace-proxies
   sandbox
   stm
-  windows
-
-
-.. _developing-pypy:
 
 Development
 -----------
@@ -77,12 +72,10 @@
 .. toctree::
   :maxdepth: 1
 
-  extradoc
-  eventhistory
-  discussions
   index-of-release-notes
   index-of-whatsnew
   contributor
+  glossary
 
 .. _contact:
 
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -1,28 +1,7 @@
-What PyPy can do for your objects
-=================================
-
-.. contents::
-
-
-Thanks to the :doc:`Object Space <objspace>` architecture, any feature that is
-based on proxying, extending, changing or otherwise controlling the
-behavior of objects in a running program is easy to implement on top of PyPy.
-
-Here is what we have implemented so far, in historical order:
-
-* *Dump Object Space*: dumps all operations performed on all the objects
-  into a large log file.  For debugging your applications.
-
-* *Transparent Proxies Extension*: adds new proxy objects to
-  the Standard Object Space that enable applications to
-  control operations on application and builtin objects,
-  e.g lists, dictionaries, tracebacks.
-
-
 .. _tproxy:
 
-Transparent Proxies
--------------------
+Transparent Proxies (DEPRECATED)
+--------------------------------
 
 .. warning::
 
diff --git a/pypy/doc/project-documentation.rst b/pypy/doc/project-documentation.rst
--- a/pypy/doc/project-documentation.rst
+++ b/pypy/doc/project-documentation.rst
@@ -32,10 +32,13 @@
    coding-guide
    sprint-reports
    extradoc
+   eventhistory
    video-index
    index-report
+   discussions
    dev_method
-   glossary
+   embedding
+   objspace-proxies
 
 
 Source Code Documentation

From pypy.commits at gmail.com  Wed May  9 13:29:28 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 09 May 2018 10:29:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix many broken links,
 checked with "sphinx-build -n -b linkcheck"
Message-ID: <5af32ff8.1c69fb81.4fcfb.e1bc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94513:e11138105393
Date: 2018-05-09 19:21 +0300
http://bitbucket.org/pypy/pypy/changeset/e11138105393/

Log:	fix many broken links, checked with "sphinx-build -n -b linkcheck"

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -267,14 +267,14 @@
 * PyPy 2.5.1 or earlier: normal users would see permission errors.
   Installers need to run ``pypy -c "import gdbm"`` and other similar
   commands at install time; the exact list is in
-  :source:`pypy/tool/release/package.py <package.py>`.  Users
+  :source:`pypy/tool/release/package.py`.  Users
   seeing a broken installation of PyPy can fix it after-the-fact if they
   have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
 
 * PyPy 2.6 and later: anyone would get ``ImportError: no module named
   _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
   ``lib_pypy`` directory during the installation process (plus others;
-  see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+  see the exact list in :source:`pypy/tool/release/package.py`).
   Users seeing a broken
   installation of PyPy can fix it after-the-fact, by running ``pypy
   /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -539,7 +539,7 @@
 
     hg help branch
 
-.. _official wiki: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: https://www.mercurial-scm.org/wiki/
 
 
 .. _using-development-tracker:
@@ -547,15 +547,7 @@
 Using the development bug/feature tracker
 -----------------------------------------
 
-We have a `development tracker`_, based on Richard Jones'
-`roundup`_ application.  You can file bugs,
-feature requests or see what's going on
-for the next milestone, both from an E-Mail and from a
-web interface.
-
-.. _development tracker: https://bugs.pypy.org/
-.. _roundup: http://roundup.sourceforge.net/
-
+We use bitbucket for :source:`issues` tracking and :source:`pull-requests`.
 
 .. _testing:
 
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
--- a/pypy/doc/contributing.rst
+++ b/pypy/doc/contributing.rst
@@ -91,7 +91,7 @@
 
 * Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
   icons).  You get a fork of the repository, e.g. in
-  https://bitbucket.org/yourname/pypy/.
+  `https://bitbucket.org/yourname/pypy/`.
 
 * Clone your new repo (i.e. the fork) to your local machine with the command
   ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
diff --git a/pypy/doc/discussion/ctypes-implementation.rst b/pypy/doc/discussion/ctypes-implementation.rst
--- a/pypy/doc/discussion/ctypes-implementation.rst
+++ b/pypy/doc/discussion/ctypes-implementation.rst
@@ -141,28 +141,3 @@
 
 .. _pyglet: http://pyglet.org/
 
-
-ctypes configure
------------------
-
-We also released ``ctypes-configure``, which is an experimental package
-trying to approach the portability issues of ctypes-based code.
-
-idea
-~~~~
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes c
-compiler (via distutils) for various platform-dependent details like
-exact sizes of types (for example size_t), ``#defines``, exact outline of
-structures etc. It replaces in this regard code generator (h2py).
-
-installation
-~~~~~~~~~~~~
-
-``easy_install ctypes_configure``
-
-usage
-~~~~~
-
-:source:`ctypes_configure/doc/sample.py` explains in details how to use it.
diff --git a/pypy/doc/eventhistory.rst b/pypy/doc/eventhistory.rst
--- a/pypy/doc/eventhistory.rst
+++ b/pypy/doc/eventhistory.rst
@@ -40,11 +40,9 @@
 Main focus of the sprint will be on the goals of the upcoming June 0.9
 release.
 
-Read more in `the sprint announcement`__, see who is  planning to attend
-on the `people page`_.
+Read more about `the sprint`__
 
-__ https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/announce.html
-.. _people page: https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/people.txt
+__ https://bitbucket.org/pypy/extradoc/src/extradoc/sprintinfo/ddorf2006/
 
 
 PyPy sprint at Akihabara (Tokyo, Japan)
diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst
--- a/pypy/doc/extradoc.rst
+++ b/pypy/doc/extradoc.rst
@@ -75,12 +75,12 @@
 .. _A Way Forward in Parallelising Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2014/position-paper.pdf
 .. _Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf
 .. _Allocation Removal by Partial Evaluation in a Tracing JIT: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/pepm2011/bolz-allocation-removal.pdf
-.. _Towards a Jitting VM for Prolog Execution: http://www.stups.uni-duesseldorf.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
+.. _Towards a Jitting VM for Prolog Execution: http://stups.hhu.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
 .. _High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages: http://buildbot.pypy.org/misc/antocuni-thesis.pdf
 .. _How to *not* write Virtual Machines for Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dyla2007/dyla.pdf
 .. _`Tracing the Meta-Level: PyPy's Tracing JIT Compiler`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009/bolz-tracing-jit.pdf
 .. _`Faster than C#: Efficient Implementation of Dynamic Languages on .NET`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009-dotnet/cli-jit.pdf
-.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: http://stups.hhu.de/mediawiki/images/b/b9/Master_bolz.pdf
+.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: https://www.researchgate.net/profile/Davide_Ancona/publication/252023163_Automatic_generation_of_JIT_compilers_for_dynamic_languages_in_NET/links/53f2098e0cf2bc0c40e70023/Automatic-generation-of-JIT-compilers-for-dynamic-languages-in-NET.pdf
 .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/DynamicLanguages_abstracts.html#AACM-DLS07
 .. _EU Reports: index-report.html
 .. _Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution: http://sabi.net/nriley/pubs/dls6-riley.pdf
@@ -368,6 +368,6 @@
 .. _LLVM: http://llvm.org/
 .. _IronPython: http://ironpython.codeplex.com/
 .. _Dynamic Native Optimization of Native Interpreters: http://people.csail.mit.edu/gregs/dynamorio.html
-.. _JikesRVM: http://jikesrvm.org/
+.. _JikesRVM: http://www.jikesrvm.org/
 .. _Tunes: http://tunes.org
 .. _old Tunes Wiki: http://buildbot.pypy.org/misc/cliki.tunes.org/
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -67,7 +67,7 @@
 you may need to run the command with `sudo` for a global installation.
 The other commands of ``setup.py`` are available too, like ``build``.
 
-.. _PyPI: https://pypi.python.org/pypi
+.. _PyPI: https://pypi.org
 .. _`use virtualenv (as documented here)`: install.html#installing-using-virtualenv
 
 
@@ -360,7 +360,7 @@
 (produced during a sprint).  On the `PyPy bitbucket page`_ there is also a
 Scheme and an Io implementation; both of these are unfinished at the moment.
 
-.. _Topaz: http://topazruby.com/
+.. _Topaz: http://docs.topazruby.com/en/latest/
 .. _Hippy: http://morepypy.blogspot.ch/2012/07/hello-everyone.html
 .. _JavaScript interpreter: https://bitbucket.org/pypy/lang-js/
 .. _Prolog interpreter: https://bitbucket.org/cfbolz/pyrolog/
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -110,7 +110,7 @@
     the `development mailing list`_.
 
 .. _#pypy on irc.freenode.net: irc://irc.freenode.net/pypy
-.. _here: http://www.tismer.com/pypy/irc-logs/pypy/
+.. _here: https://botbot.me/freenode/pypy/
 .. _Development mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
 .. _Commit mailing list: http://mail.python.org/mailman/listinfo/pypy-commit
 .. _Development bug/feature tracker: https://bitbucket.org/pypy/pypy/issues
diff --git a/pypy/doc/interpreter.rst b/pypy/doc/interpreter.rst
--- a/pypy/doc/interpreter.rst
+++ b/pypy/doc/interpreter.rst
@@ -102,7 +102,7 @@
 program flows with homogeneous name-value assignments on
 function invocations.
 
-.. _how-to guide for descriptors: http://users.rcn.com/python/download/Descriptor.htm
+.. _how-to guide for descriptors: https://docs.python.org/3/howto/descriptor.html
 
 
 Bytecode Interpreter Implementation Classes
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -194,7 +194,7 @@
 application-level code.
 
 Transparent proxies are implemented on top of the :ref:`standard object
-space <standard-object-space>`, in :source:`pypy/objspace/std/proxy_helpers.py`,
+space <standard-object-space>`, in :source:`pypy/objspace/std/proxyobject.py`,
 :source:`pypy/objspace/std/proxyobject.py` and :source:`pypy/objspace/std/transparent.py`.
 To use them you will need to pass a `--objspace-std-withtproxy`_ option to ``pypy``
 or ``translate.py``.  This registers implementations named :py:class:`W_TransparentXxx`
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -474,8 +474,8 @@
 :source:`pypy/objspace/std/bytesobject.py` defines ``W_AbstractBytesObject``,
 which contains everything needed to build the ``str`` app-level type;
 and there are subclasses ``W_BytesObject`` (the usual string) and
-``W_StringBufferObject`` (a special implementation tweaked for repeated
-additions, in :source:`pypy/objspace/std/strbufobject.py`).  For mutable data
+``W_Buffer`` (a special implementation tweaked for repeated
+additions, in :source:`pypy/objspace/std/bufferobject.py`).  For mutable data
 types like lists and dictionaries, we have a single class
 ``W_ListObject`` or ``W_DictMultiObject`` which has an indirection to
 the real data and a strategy; the strategy can change as the content of
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -25,7 +25,7 @@
 
 This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has
 made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
-was checked in Nov 2016). Note that the compiler suite may be installed in
+was checked in May 2018). Note that the compiler suite may be installed in
 ``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``
 or in
 ``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``.
@@ -37,7 +37,7 @@
 by running the mt.exe command by hand from a DOS window (that is how the author
 discovered the problem).
 
-.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
+.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/EN-US/DOWNLOAD/DETAILS.ASPX?ID=44266
 
 Installing "Build Tools for Visual Studio 2017" (for Python 3)
 --------------------------------------------------------------

From pypy.commits at gmail.com  Wed May  9 13:46:35 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 May 2018 10:46:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document the purpose of _hash_int() more
Message-ID: <5af333fb.1c69fb81.42eaf.1fd3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94516:662b52963394
Date: 2018-05-09 19:45 +0200
http://bitbucket.org/pypy/pypy/changeset/662b52963394/

Log:	Document the purpose of _hash_int() more

diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -895,4 +895,9 @@
     # The complete list of built-in types whose hash should be
     # consistent is: int, long, bool, float, complex.
     #
+    # Note: the same function in PyPy3 does far more computations.
+    # So you should call _hash_int() only when you want to get the exact
+    # same result as hash(integer) does on app-level, and not merely to
+    # adjust some unrelated hash result from -1 to -2.
+    #
     return a - (a == -1)  # No explicit condition, to avoid JIT bridges

From pypy.commits at gmail.com  Wed May  9 13:46:38 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 09 May 2018 10:46:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5af333fe.1c69fb81.88063.5702@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94517:6aceee434fd0
Date: 2018-05-09 19:45 +0200
http://bitbucket.org/pypy/pypy/changeset/6aceee434fd0/

Log:	hg merge default

diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -73,3 +73,63 @@
 This division between bytecode evaluator and object space gives a lot of
 flexibility.  One can plug in different :doc:`object spaces <objspace>` to get
 different or enriched behaviours of the Python objects.
+
+Layers
+------
+
+RPython
+~~~~~~~
+:ref:`RPython <rpython:language>` is the language in which we write interpreters.
+Not the entire PyPy project is written in RPython, only the parts that are
+compiled in the translation process. The interesting point is that RPython
+has no parser, it's compiled from the live python objects, which makes it
+possible to do all kinds of metaprogramming during import time. In short,
+Python is a meta programming language for RPython.
+
+The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+Consult `Getting Started with RPython`_ for further reading
+
+Translation
+~~~~~~~~~~~
+The translation toolchain - this is the part that takes care of translating
+RPython to flow graphs and then to C. There is more in the
+:doc:`architecture <architecture>` document written about it.
+
+It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+and ``rtyper``.
+
+PyPy Interpreter
+~~~~~~~~~~~~~~~~
+This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
+interpreter for Python written in RPython.  The fact that it is
+RPython is not apparent at first.  Built-in modules are written in
+``pypy/module/*``.  Some modules that CPython implements in C are
+simply written in pure Python; they are in the top-level ``lib_pypy``
+directory.  The standard library of Python (with a few changes to
+accomodate PyPy) is in ``lib-python``.
+
+JIT Compiler
+~~~~~~~~~~~~
+:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
+interpreter written in RPython, rather than the user program that it
+interprets.  As a result it applies to any interpreter, i.e. any
+language.  But getting it to work correctly is not trivial: it
+requires a small number of precise "hints" and possibly some small
+refactorings of the interpreter.  The JIT itself also has several
+almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+that turns it into machine code.  Writing a new backend is a
+traditional way to get into the project.
+
+Garbage Collectors
+~~~~~~~~~~~~~~~~~~
+Garbage Collectors (GC): as you may notice if you are used to CPython's
+C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+:ref:`rpython:garbage-collection` is inserted
+during translation.  Moreover, this is not reference counting; it is a real
+GC written as more RPython code.  The best one we have so far is in
+``rpython/memory/gc/incminimark.py``.
+
+.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -267,14 +267,14 @@
 * PyPy 2.5.1 or earlier: normal users would see permission errors.
   Installers need to run ``pypy -c "import gdbm"`` and other similar
   commands at install time; the exact list is in
-  :source:`pypy/tool/release/package.py <package.py>`.  Users
+  :source:`pypy/tool/release/package.py`.  Users
   seeing a broken installation of PyPy can fix it after-the-fact if they
   have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
 
 * PyPy 2.6 and later: anyone would get ``ImportError: no module named
   _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
   ``lib_pypy`` directory during the installation process (plus others;
-  see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+  see the exact list in :source:`pypy/tool/release/package.py`).
   Users seeing a broken
   installation of PyPy can fix it after-the-fact, by running ``pypy
   /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -539,7 +539,7 @@
 
     hg help branch
 
-.. _official wiki: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: https://www.mercurial-scm.org/wiki/
 
 
 .. _using-development-tracker:
@@ -547,15 +547,7 @@
 Using the development bug/feature tracker
 -----------------------------------------
 
-We have a `development tracker`_, based on Richard Jones'
-`roundup`_ application.  You can file bugs,
-feature requests or see what's going on
-for the next milestone, both from an E-Mail and from a
-web interface.
-
-.. _development tracker: https://bugs.pypy.org/
-.. _roundup: http://roundup.sourceforge.net/
-
+We use bitbucket for :source:`issues` tracking and :source:`pull-requests`.
 
 .. _testing:
 
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -8,3 +8,4 @@
    :maxdepth: 1
 
    man/pypy.1.rst
+   man/pypy3.1.rst
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -66,9 +66,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '5.8'
+version = '6.0'
 # The full version, including alpha/beta/rc tags.
-release = '5.8.0'
+release = '6.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/contributing.rst
@@ -0,0 +1,453 @@
+Contributing Guidelines
+===========================
+
+.. contents::
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy's
+  approach is great if you do TDD, and not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below.
+
+The first thing to remember is that PyPy project is very different than most
+projects out there. It's also different from a classic compiler project,
+so academic courses about compilers often don't apply or lead in the wrong
+direction. However, if you want to understand how designing & building a runtime
+works in the real world then this is a great project!
+
+Getting involved
+^^^^^^^^^^^^^^^^
+
+PyPy employs a relatively standard open-source development process. You are
+encouraged as a first step to join our `pypy-dev mailing list`_ and IRC channel,
+details of which can be found in our :ref:`contact <contact>` section. The folks
+there are very friendly, and can point you in the right direction.
+
+We give out commit rights usually fairly liberally, so if you want to do something
+with PyPy, you can become a committer. We also run frequent coding sprints which
+are separately announced and often happen around Python conferences such as
+EuroPython or PyCon. Upcoming events are usually announced on `the blog`_.
+
+Further Reading: :ref:`Contact <contact>`
+
+.. _the blog: http://morepypy.blogspot.com
+.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
+
+
+Your first contribution
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The first and most important rule how **not** to contribute to PyPy is
+"just hacking a feature". This won't work, and you'll find your PR will typically
+require a lot of re-work. There are a few reasons why not:
+
+* build times are large
+* PyPy has very thick layer separation
+* context of the cPython runtime is often required
+
+Instead, reach out on the dev mailing list or the IRC channel, and we're more
+than happy to help! :)
+
+Some ideas for first contributions are:
+
+* Documentation - this will give you an understanding of the pypy architecture
+* Test failures - find a failing test in the `nightly builds`_, and fix it
+* Missing language features - these are listed in our `issue tracker`_
+
+.. _nightly builds: http://buildbot.pypy.org/nightly/
+.. _issue tracker: https://bitbucket.org/pypy/pypy/issues
+
+Source Control
+--------------
+
+PyPy development is based a typical fork/pull request based workflow, centered
+around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
+before, a good introduction can be found here:
+
+    https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow
+
+The cycle for a new PyPy contributor goes typically like this:
+
+Fork & Clone
+------------
+
+* Make an account on bitbucket_.
+
+* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
+  icons).  You get a fork of the repository, e.g. in
+  `https://bitbucket.org/yourname/pypy/`.
+
+* Clone your new repo (i.e. the fork) to your local machine with the command
+  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
+  operation but only ever needs to be done once.  See also
+  http://pypy.org/download.html#building-from-source .
+  If you already cloned
+  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
+  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
+  your clone to contain the line ``default =
+  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
+  up``.  If you already have such a clone but don't want to change it,
+  you can clone that copy with ``hg clone /path/to/other/copy``, and
+  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
+
+* Now you have a complete copy of the PyPy repo.  Make a branch
+  with a command like ``hg branch name_of_your_branch``.
+
+Edit
+----
+
+* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
+  to make Mercurial aware of new files you added, e.g. new test files.
+  Use ``hg status`` to see if there are such files.  Write and run tests!
+  (See the rest of this page.)
+
+* Commit regularly with ``hg commit``.  A one-line commit message is
+  fine.  We love to have tons of commits; make one as soon as you have
+  some progress, even if it is only some new test that doesn't pass yet,
+  or fixing things even if not all tests pass.  Step by step, you are
+  building the history of your changes, which is the point of a version
+  control system.  (There are commands like ``hg log`` and ``hg up``
+  that you should read about later, to learn how to navigate this
+  history.)
+
+* The commits stay on your machine until you do ``hg push`` to "push"
+  them back to the repo named in the file ``.hg/hgrc``.  Repos are
+  basically just collections of commits (a commit is also called a
+  changeset): there is one repo per url, plus one for each local copy on
+  each local machine.  The commands ``hg push`` and ``hg pull`` copy
+  commits around, with the goal that all repos in question end up with
+  the exact same set of commits.  By opposition, ``hg up`` only updates
+  the "working copy" by reading the local repository, i.e. it makes the
+  files that you see correspond to the latest (or any other) commit
+  locally present.
+
+* You should push often; there is no real reason not to.  Remember that
+  even if they are pushed, with the setup above, the commits are (1)
+  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
+  named.  Yes, they are publicly visible, but don't worry about someone
+  walking around the thousands of repos on bitbucket saying "hah, look
+  at the bad coding style of that guy".  Try to get into the mindset
+  that your work is not secret and it's fine that way.  We might not
+  accept it as is for PyPy, asking you instead to improve some things,
+  but we are not going to judge you.
+
+Pull Request
+------------
+
+* The final step is to open a pull request, so that we know that you'd
+  like to merge that branch back to the original ``pypy/pypy`` repo.
+  This can also be done several times if you have interesting
+  intermediate states, but if you get there, then we're likely to
+  proceed to the next stage, which is...
+
+* Get a regular account for pushing directly to
+  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
+  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
+  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
+  then be pushed directly to the official repo, but (if you follow these
+  rules) they are still on a branch, and we can still review the
+  branches you want to merge.
+
+* If you get closer to the regular day-to-day development, you'll notice
+  that we generally push small changes as one or a few commits directly
+  to the branch ``default``.  Also, we often collaborate even if we are
+  on other branches, which do not really "belong" to anyone.  At this
+  point you'll need ``hg merge`` and learn how to resolve conflicts that
+  sometimes occur when two people try to push different commits in
+  parallel on the same branch.  But it is likely an issue for later ``:-)``
+
+.. _bitbucket: https://bitbucket.org/
+
+
+Architecture
+^^^^^^^^^^^^
+
+PyPy has layers. Just like ogres or onions. Those layers help us keep the
+respective parts separated enough to be worked on independently and make the
+complexity manageable. This is, again, just a sanity requirement for such
+a complex project. For example writing a new optimization for the JIT usually
+does **not** involve touching a Python interpreter at all or the JIT assembler
+backend or the garbage collector. Instead it requires writing small tests in
+``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
+After that, you can just compile PyPy and things should just work.
+
+Further Reading: :doc:`architecture <architecture>`
+
+Where to start?
+---------------
+
+PyPy is made from parts that are relatively independent of each other.
+You should start looking at the part that attracts you most (all paths are
+relative to the PyPy top level directory).  You may look at our
+:doc:`directory reference <dir-reference>` or start off at one of the following
+points:
+
+*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
+   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
+   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
+   function objects and argument passing in :source:`pypy/interpreter/function.py`
+   and :source:`pypy/interpreter/argument.py`, the object space interface
+   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
+   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
+   Core types supporting the bytecode interpreter are defined in
+   :source:`pypy/interpreter/typedef.py`.
+
+*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
+   and grammar files that allow it to parse the syntax of various Python
+   versions. Once the grammar has been processed, the parser can be
+   translated by the above machinery into efficient code.
+
+*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
+   contains a modified version of the compiler package from CPython
+   that fixes some bugs and is translatable.
+
+*  :source:`pypy/objspace/std` contains the
+   :ref:`Standard object space <standard-object-space>`.  The main file
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)
+
+Building
+^^^^^^^^
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Further Reading: :doc:`Build <build>`
+
+Coding Guide
+------------
+
+As well as the usual pep8 and formatting standards, there are a number of
+naming conventions and coding styles that are important to understand before
+browsing the source.
+
+Further Reading: :doc:`Coding Guide <coding-guide>`
+
+Testing
+^^^^^^^
+
+Test driven development
+-----------------------
+
+Instead, we practice a lot of test driven development. This is partly because
+of very high quality requirements for compilers and partly because there is
+simply no other way to get around such complex project, that will keep you sane.
+There are probably people out there who are smart enough not to need it, we're
+not one of those. You may consider familiarizing yourself with `pytest`_,
+since this is a tool we use for tests.
+This leads to the next issue:
+
+.. _pytest: http://pytest.org/
+
+py.test and the py lib
+----------------------
+
+The `py.test testing tool`_ drives all our testing needs.
+
+We use the `py library`_ for filesystem path manipulations, terminal
+writing, logging and some other support  functionality.
+
+You don't necessarily need to install these two libraries because
+we also ship them inlined in the PyPy source tree.
+
+.. _py library: http://pylib.readthedocs.org/
+
+Running PyPy's unit tests
+-------------------------
+
+PyPy development always was and is still thoroughly test-driven.
+We use the flexible `py.test testing tool`_ which you can `install independently
+<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
+
+The PyPy source tree comes with an inlined version of ``py.test``
+which you can invoke by typing::
+
+    python pytest.py -h
+
+This is usually equivalent to using an installed version::
+
+    py.test -h
+
+If you encounter problems with the installed version
+make sure you have the correct version installed which
+you can find out with the ``--version`` switch.
+
+You will need the `build requirements`_ to run tests successfully, since many of
+them compile little pieces of PyPy and then run the tests inside that minimal
+interpreter
+
+Now on to running some tests.  PyPy has many different test directories
+and you can use shell completion to point at directories or files::
+
+    py.test pypy/interpreter/test/test_pyframe.py
+
+    # or for running tests of a whole subdirectory
+    py.test pypy/interpreter/
+
+See `py.test usage and invocations`_ for some more generic info
+on how you can run tests.
+
+Beware trying to run "all" pypy tests by pointing to the root
+directory or even the top level subdirectory ``pypy``.  It takes
+hours and uses huge amounts of RAM and is not recommended.
+
+To run CPython regression tests you can point to the ``lib-python``
+directory::
+
+    py.test lib-python/2.7/test/test_datetime.py
+
+This will usually take a long time because this will run
+the PyPy Python interpreter on top of CPython.  On the plus
+side, it's usually still faster than doing a full translation
+and running the regression test with the translated PyPy Python
+interpreter.
+
+.. _py.test testing tool: http://pytest.org
+.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
+.. _`build requirements`: build.html#install-build-time-dependencies
+
+Tooling & Utilities
+^^^^^^^^^^^^^^^^^^^
+
+If you are interested in the inner workings of the PyPy Python interpreter,
+there are some features of the untranslated Python interpreter that allow you
+to introspect its internals.
+
+
+Interpreter-level console
+-------------------------
+
+To start interpreting Python with PyPy, install a C compiler that is
+supported by distutils and use Python 2.7 or greater to run PyPy::
+
+    cd pypy
+    python bin/pyinteractive.py
+
+After a few seconds (remember: this is running on top of CPython), you should
+be at the PyPy prompt, which is the same as the Python prompt, but with an
+extra ">".
+
+If you press
+<Ctrl-C> on the console you enter the interpreter-level console, a
+usual CPython console.  You can then access internal objects of PyPy
+(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
+prompt with the prefix ``w_``::
+
+    >>>> a = 123
+    >>>> <Ctrl-C>
+    *** Entering interpreter-level console ***
+    >>> w_a
+    W_IntObject(123)
+
+The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
+
+    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
+    >>> <Ctrl-D>
+    *** Leaving interpreter-level console ***
+
+    KeyboardInterrupt
+    >>>> l
+    [1, 'abc']
+
+Note that the prompt of the interpreter-level console is only '>>>' since
+it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
+Linux) or <Ctrl-Z>, <Enter> (under Windows).
+
+Also note that not all modules are available by default in this mode (for
+example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
+``--withmod-...`` command line options.
+
+You may be interested in reading more about the distinction between
+:ref:`interpreter-level and app-level <interpreter-level>`.
+
+pyinteractive.py options
+------------------------
+
+To list the PyPy interpreter command line options, type::
+
+    cd pypy
+    python bin/pyinteractive.py --help
+
+pyinteractive.py supports most of the options that CPython supports too (in addition to a
+large amount of options that can be used to customize pyinteractive.py).
+As an example of using PyPy from the command line, you could type::
+
+    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
+
+Alternatively, as with regular Python, you can simply give a
+script name on the command line::
+
+    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
+
+See our :doc:`configuration sections <config/index>` for details about what all the commandline
+options do.
+
+
+.. _trace example:
+
+Tracing bytecode and operations on objects
+------------------------------------------
+
+You can use a simple tracing mode to monitor the interpretation of
+bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
+PyPy console::
+
+    >>>> __pytrace__ = 1
+    Tracing enabled
+    >>>> x = 5
+            <module>:           LOAD_CONST    0 (5)
+            <module>:           STORE_NAME    0 (x)
+            <module>:           LOAD_CONST    1 (None)
+            <module>:           RETURN_VALUE    0
+    >>>> x
+            <module>:           LOAD_NAME    0 (x)
+            <module>:           PRINT_EXPR    0
+    5
+            <module>:           LOAD_CONST    0 (None)
+            <module>:           RETURN_VALUE    0
+    >>>>
+
+
+Demos
+^^^^^
+
+The `example-interpreter`_ repository contains an example interpreter
+written using the RPython translation toolchain.
+
+.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
+
+
+graphviz & pygame for flow graph viewing (highly recommended)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+graphviz and pygame are both necessary if you want to look at generated flow
+graphs:
+
+    graphviz: http://www.graphviz.org/Download.php
+
+    pygame: http://www.pygame.org/download.shtml
+
diff --git a/pypy/doc/discussion/ctypes-implementation.rst b/pypy/doc/discussion/ctypes-implementation.rst
--- a/pypy/doc/discussion/ctypes-implementation.rst
+++ b/pypy/doc/discussion/ctypes-implementation.rst
@@ -141,28 +141,3 @@
 
 .. _pyglet: http://pyglet.org/
 
-
-ctypes configure
------------------
-
-We also released ``ctypes-configure``, which is an experimental package
-trying to approach the portability issues of ctypes-based code.
-
-idea
-~~~~
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes c
-compiler (via distutils) for various platform-dependent details like
-exact sizes of types (for example size_t), ``#defines``, exact outline of
-structures etc. It replaces in this regard code generator (h2py).
-
-installation
-~~~~~~~~~~~~
-
-``easy_install ctypes_configure``
-
-usage
-~~~~~
-
-:source:`ctypes_configure/doc/sample.py` explains in details how to use it.
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -1,5 +1,5 @@
-Embedding PyPy
-==============
+Embedding PyPy (DEPRECATED)
+===========================
 
 PyPy has a very minimal and a very strange embedding interface, based on
 the usage of `cffi`_ and the philosophy that Python is a better language than
diff --git a/pypy/doc/eventhistory.rst b/pypy/doc/eventhistory.rst
--- a/pypy/doc/eventhistory.rst
+++ b/pypy/doc/eventhistory.rst
@@ -40,11 +40,9 @@
 Main focus of the sprint will be on the goals of the upcoming June 0.9
 release.
 
-Read more in `the sprint announcement`__, see who is  planning to attend
-on the `people page`_.
+Read more about `the sprint`__
 
-__ https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/announce.html
-.. _people page: https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/people.txt
+__ https://bitbucket.org/pypy/extradoc/src/extradoc/sprintinfo/ddorf2006/
 
 
 PyPy sprint at Akihabara (Tokyo, Japan)
diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst
--- a/pypy/doc/extradoc.rst
+++ b/pypy/doc/extradoc.rst
@@ -75,12 +75,12 @@
 .. _A Way Forward in Parallelising Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2014/position-paper.pdf
 .. _Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf
 .. _Allocation Removal by Partial Evaluation in a Tracing JIT: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/pepm2011/bolz-allocation-removal.pdf
-.. _Towards a Jitting VM for Prolog Execution: http://www.stups.uni-duesseldorf.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
+.. _Towards a Jitting VM for Prolog Execution: http://stups.hhu.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
 .. _High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages: http://buildbot.pypy.org/misc/antocuni-thesis.pdf
 .. _How to *not* write Virtual Machines for Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dyla2007/dyla.pdf
 .. _`Tracing the Meta-Level: PyPy's Tracing JIT Compiler`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009/bolz-tracing-jit.pdf
 .. _`Faster than C#: Efficient Implementation of Dynamic Languages on .NET`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009-dotnet/cli-jit.pdf
-.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: http://stups.hhu.de/mediawiki/images/b/b9/Master_bolz.pdf
+.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: https://www.researchgate.net/profile/Davide_Ancona/publication/252023163_Automatic_generation_of_JIT_compilers_for_dynamic_languages_in_NET/links/53f2098e0cf2bc0c40e70023/Automatic-generation-of-JIT-compilers-for-dynamic-languages-in-NET.pdf
 .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/DynamicLanguages_abstracts.html#AACM-DLS07
 .. _EU Reports: index-report.html
 .. _Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution: http://sabi.net/nriley/pubs/dls6-riley.pdf
@@ -368,6 +368,6 @@
 .. _LLVM: http://llvm.org/
 .. _IronPython: http://ironpython.codeplex.com/
 .. _Dynamic Native Optimization of Native Interpreters: http://people.csail.mit.edu/gregs/dynamorio.html
-.. _JikesRVM: http://jikesrvm.org/
+.. _JikesRVM: http://www.jikesrvm.org/
 .. _Tunes: http://tunes.org
 .. _old Tunes Wiki: http://buildbot.pypy.org/misc/cliki.tunes.org/
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -67,7 +67,7 @@
 you may need to run the command with `sudo` for a global installation.
 The other commands of ``setup.py`` are available too, like ``build``.
 
-.. _PyPI: https://pypi.python.org/pypi
+.. _PyPI: https://pypi.org
 .. _`use virtualenv (as documented here)`: install.html#installing-using-virtualenv
 
 
@@ -360,7 +360,7 @@
 (produced during a sprint).  On the `PyPy bitbucket page`_ there is also a
 Scheme and an Io implementation; both of these are unfinished at the moment.
 
-.. _Topaz: http://topazruby.com/
+.. _Topaz: http://docs.topazruby.com/en/latest/
 .. _Hippy: http://morepypy.blogspot.ch/2012/07/hello-everyone.html
 .. _JavaScript interpreter: https://bitbucket.org/pypy/lang-js/
 .. _Prolog interpreter: https://bitbucket.org/cfbolz/pyrolog/
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
deleted file mode 100644
--- a/pypy/doc/getting-started-dev.rst
+++ /dev/null
@@ -1,345 +0,0 @@
-Getting Started Developing With PyPy
-====================================
-
-.. contents::
-
-
-Using Mercurial
----------------
-
-PyPy development is based on Mercurial (hg).  If you are not used to
-version control, the cycle for a new PyPy contributor goes typically
-like this:
-
-* Make an account on bitbucket_.
-
-* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
-  icons).  You get a fork of the repository, e.g. in
-  https://bitbucket.org/yourname/pypy/.
-
-* Clone this new repo (i.e. the fork) to your local machine with the command 
-  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
-  operation but only ever needs to be done once.  See also 
-  http://pypy.org/download.html#building-from-source .
-  If you already cloned
-  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
-  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
-  your clone to contain the line ``default =
-  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
-  up``.  If you already have such a clone but don't want to change it,
-  you can clone that copy with ``hg clone /path/to/other/copy``, and
-  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
-
-* Now you have a complete copy of the PyPy repo.  Make a branch
-  with a command like ``hg branch name_of_your_branch``.
-
-* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
-  to make Mercurial aware of new files you added, e.g. new test files.
-  Use ``hg status`` to see if there are such files.  Write and run tests!
-  (See the rest of this page.)
-
-* Commit regularly with ``hg commit``.  A one-line commit message is
-  fine.  We love to have tons of commits; make one as soon as you have
-  some progress, even if it is only some new test that doesn't pass yet,
-  or fixing things even if not all tests pass.  Step by step, you are
-  building the history of your changes, which is the point of a version
-  control system.  (There are commands like ``hg log`` and ``hg up``
-  that you should read about later, to learn how to navigate this
-  history.)
-
-* The commits stay on your machine until you do ``hg push`` to "push"
-  them back to the repo named in the file ``.hg/hgrc``.  Repos are
-  basically just collections of commits (a commit is also called a
-  changeset): there is one repo per url, plus one for each local copy on
-  each local machine.  The commands ``hg push`` and ``hg pull`` copy
-  commits around, with the goal that all repos in question end up with
-  the exact same set of commits.  By opposition, ``hg up`` only updates
-  the "working copy" by reading the local repository, i.e. it makes the
-  files that you see correspond to the latest (or any other) commit
-  locally present.
-
-* You should push often; there is no real reason not to.  Remember that
-  even if they are pushed, with the setup above, the commits are (1)
-  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
-  named.  Yes, they are publicly visible, but don't worry about someone
-  walking around the thousands of repos on bitbucket saying "hah, look
-  at the bad coding style of that guy".  Try to get into the mindset
-  that your work is not secret and it's fine that way.  We might not
-  accept it as is for PyPy, asking you instead to improve some things,
-  but we are not going to judge you.
-
-* The final step is to open a pull request, so that we know that you'd
-  like to merge that branch back to the original ``pypy/pypy`` repo.
-  This can also be done several times if you have interesting
-  intermediate states, but if you get there, then we're likely to
-  proceed to the next stage, which is...
-
-* Get a regular account for pushing directly to
-  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
-  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
-  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
-  then be pushed directly to the official repo, but (if you follow these
-  rules) they are still on a branch, and we can still review the
-  branches you want to merge.
-
-* If you get closer to the regular day-to-day development, you'll notice
-  that we generally push small changes as one or a few commits directly
-  to the branch ``default``.  Also, we often collaborate even if we are
-  on other branches, which do not really "belong" to anyone.  At this
-  point you'll need ``hg merge`` and learn how to resolve conflicts that
-  sometimes occur when two people try to push different commits in
-  parallel on the same branch.  But it is likely an issue for later ``:-)``
-
-.. _bitbucket: https://bitbucket.org/
-
-
-Running PyPy's unit tests
--------------------------
-
-PyPy development always was and is still thoroughly test-driven.
-We use the flexible `py.test testing tool`_ which you can `install independently
-<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
-
-The PyPy source tree comes with an inlined version of ``py.test``
-which you can invoke by typing::
-
-    python pytest.py -h
-
-This is usually equivalent to using an installed version::
-
-    py.test -h
-
-If you encounter problems with the installed version
-make sure you have the correct version installed which
-you can find out with the ``--version`` switch.
-
-You will need the `build requirements`_ to run tests successfully, since many of
-them compile little pieces of PyPy and then run the tests inside that minimal
-interpreter
-
-Now on to running some tests.  PyPy has many different test directories
-and you can use shell completion to point at directories or files::
-
-    py.test pypy/interpreter/test/test_pyframe.py
-
-    # or for running tests of a whole subdirectory
-    py.test pypy/interpreter/
-
-See `py.test usage and invocations`_ for some more generic info
-on how you can run tests.
-
-Beware trying to run "all" pypy tests by pointing to the root
-directory or even the top level subdirectory ``pypy``.  It takes
-hours and uses huge amounts of RAM and is not recommended.
-
-To run CPython regression tests you can point to the ``lib-python``
-directory::
-
-    py.test lib-python/2.7/test/test_datetime.py
-
-This will usually take a long time because this will run
-the PyPy Python interpreter on top of CPython.  On the plus
-side, it's usually still faster than doing a full translation
-and running the regression test with the translated PyPy Python
-interpreter.
-
-.. _py.test testing tool: http://pytest.org
-.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
-.. _`build requirements`: build.html#install-build-time-dependencies
-
-Special Introspection Features of the Untranslated Python Interpreter
----------------------------------------------------------------------
-
-If you are interested in the inner workings of the PyPy Python interpreter,
-there are some features of the untranslated Python interpreter that allow you
-to introspect its internals.
-
-
-Interpreter-level console
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To start interpreting Python with PyPy, install a C compiler that is
-supported by distutils and use Python 2.7 or greater to run PyPy::
-
-    cd pypy
-    python bin/pyinteractive.py
-
-After a few seconds (remember: this is running on top of CPython), you should
-be at the PyPy prompt, which is the same as the Python prompt, but with an
-extra ">".
-
-If you press
-<Ctrl-C> on the console you enter the interpreter-level console, a
-usual CPython console.  You can then access internal objects of PyPy
-(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
-prompt with the prefix ``w_``::
-
-    >>>> a = 123
-    >>>> <Ctrl-C>
-    *** Entering interpreter-level console ***
-    >>> w_a
-    W_IntObject(123)
-
-The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
-
-    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
-    >>> <Ctrl-D>
-    *** Leaving interpreter-level console ***
-
-    KeyboardInterrupt
-    >>>> l
-    [1, 'abc']
-
-Note that the prompt of the interpreter-level console is only '>>>' since
-it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
-Linux) or <Ctrl-Z>, <Enter> (under Windows).
-
-Also note that not all modules are available by default in this mode (for
-example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
-``--withmod-...`` command line options.
-
-You may be interested in reading more about the distinction between
-:ref:`interpreter-level and app-level <interpreter-level>`.
-
-pyinteractive.py options
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-To list the PyPy interpreter command line options, type::
-
-    cd pypy
-    python bin/pyinteractive.py --help
-
-pyinteractive.py supports most of the options that CPython supports too (in addition to a
-large amount of options that can be used to customize pyinteractive.py).
-As an example of using PyPy from the command line, you could type::
-
-    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
-
-Alternatively, as with regular Python, you can simply give a
-script name on the command line::
-
-    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
-
-The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
-default almost none of them are, because initializing them takes time.
-If you want anyway to enable all built-in modules, you can use
-``--allworkingmodules``.
-
-See our :doc:`configuration sections <config/index>` for details about what all the commandline
-options do.
-
-
-.. _trace example:
-
-Tracing bytecode and operations on objects
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can use a simple tracing mode to monitor the interpretation of
-bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
-PyPy console::
-
-    >>>> __pytrace__ = 1
-    Tracing enabled
-    >>>> x = 5
-            <module>:           LOAD_CONST    0 (5)
-            <module>:           STORE_NAME    0 (x)
-            <module>:           LOAD_CONST    1 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>> x
-            <module>:           LOAD_NAME    0 (x)
-            <module>:           PRINT_EXPR    0 
-    5
-            <module>:           LOAD_CONST    0 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>>
-
-
-Demos
------
-
-The `example-interpreter`_ repository contains an example interpreter
-written using the RPython translation toolchain.
-
-.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
-
-
-Additional Tools for running (and hacking) PyPy
------------------------------------------------
-
-We use some optional tools for developing PyPy. They are not required to run
-the basic tests or to get an interactive PyPy prompt but they help to
-understand  and debug PyPy especially for the translation process.
-
-
-graphviz & pygame for flow graph viewing (highly recommended)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-graphviz and pygame are both necessary if you
-want to look at generated flow graphs:
-
-	graphviz: http://www.graphviz.org/Download.php
-
-	pygame: http://www.pygame.org/download.shtml
-
-
-py.test and the py lib
-~~~~~~~~~~~~~~~~~~~~~~
-
-The `py.test testing tool`_ drives all our testing needs.
-
-We use the `py library`_ for filesystem path manipulations, terminal
-writing, logging and some other support  functionality.
-
-You don't necessarily need to install these two libraries because
-we also ship them inlined in the PyPy source tree.
-
-.. _py library: http://pylib.readthedocs.org/
-
-
-Getting involved
-----------------
-
-PyPy employs an open development process.  You are invited to join our
-`pypy-dev mailing list`_ or look at the other :ref:`contact
-possibilities <contact>`.  Usually we give out commit rights fairly liberally, so if you
-want to do something with PyPy, you can become a committer. We also run frequent
-coding sprints which are separately announced and often happen around Python
-conferences such as EuroPython or PyCon. Upcoming events are usually announced
-on `the blog`_.
-
-.. _the blog: http://morepypy.blogspot.com
-.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
-
-
-.. _start-reading-sources:
-
-Where to start reading the sources
-----------------------------------
-
-PyPy is made from parts that are relatively independent of each other.
-You should start looking at the part that attracts you most (all paths are
-relative to the PyPy top level directory).  You may look at our :doc:`directory reference <dir-reference>`
-or start off at one of the following points:
-
-*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
-   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
-   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
-   function objects and argument passing in :source:`pypy/interpreter/function.py`
-   and :source:`pypy/interpreter/argument.py`, the object space interface
-   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
-   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
-   Core types supporting the bytecode interpreter are defined in :source:`pypy/interpreter/typedef.py`.
-
-*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
-   and grammar files that allow it to parse the syntax of various Python
-   versions. Once the grammar has been processed, the parser can be
-   translated by the above machinery into efficient code.
-
-*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
-   contains a modified version of the compiler package from CPython
-   that fixes some bugs and is translatable.
-
-*  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
-   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
-   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
-   as a first approximation.  (Some types have multiple implementations.)
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
deleted file mode 100644
--- a/pypy/doc/how-to-contribute.rst
+++ /dev/null
@@ -1,93 +0,0 @@
-How to contribute to PyPy
-=========================
-
-This page describes how to contribute to the PyPy project. The first thing
-to remember is that PyPy project is very different than most projects out there.
-It's also different from a classic compiler project, so academic courses
-about compilers often don't apply or lead in the wrong direction.
-
-
-Don't just hack
----------------
-
-The first and most important rule how not to contribute to PyPy is
-"just hacking". This won't work. There are two major reasons why not
--- build times are large and PyPy has very thick layer separation which
-make it harder to "just hack a feature".
-
-
-Test driven development
------------------------
-
-Instead, we practice a lot of test driven development. This is partly because
-of very high quality requirements for compilers and partly because there is
-simply no other way to get around such complex project, that will keep you sane.
-There are probably people out there who are smart enough not to need it, we're
-not one of those. You may consider familiarizing yourself with `pytest`_,
-since this is a tool we use for tests.
-This leads to the next issue:
-
-.. _pytest: http://pytest.org/
-
-
-Layers
-------
-
-PyPy has layers. Just like Ogres or onions.
-Those layers help us keep the respective parts separated enough
-to be worked on independently and make the complexity manageable. This is,
-again, just a sanity requirement for such a complex project. For example writing
-a new optimization for the JIT usually does **not** involve touching a Python
-interpreter at all or the JIT assembler backend or the garbage collector.
-Instead it requires writing small tests in
-``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
-After that, you can just compile PyPy and things should just work.
-
-The short list of layers for further reading. For each of those layers, a good
-entry point is a test subdirectory in respective directories. It usually
-describes (better or worse) the interfaces between the submodules. For the
-``pypy`` subdirectory, most tests are small snippets of python programs that
-check for correctness (calls ``AppTestXxx``) that will call the appropriate
-part of the interpreter. For the ``rpython`` directory, most tests are small
-RPython interpreters that perform certain tasks. To see how they translate
-to low-level graphs, run them with ``--view``. To see small interpreters
-with a JIT compiler, use ``--viewloops`` option.
-
-* **python interpreter** - it's the part implemented in the ``pypy/`` directory.
-  It's implemented in RPython, which is a high level static language with
-  classes, garbage collection, just-in-time compiler generation and the ability
-  to call C. A cool part about it is that it can be run untranslated, so all
-  the tests are runnable without translating PyPy.
-
-  **interpreter** contains the interpreter core
-
-  **objspace** contains implementations of various objects exported to
-  the Python layer
-
-  **module** directory contains extension modules written in RPython
-
-* **rpython compiler** that resides in ``rpython/annotator`` and
-  ``rpython/rtyper`` directories. Consult `Getting Started with RPython`_
-  for further reading
-
-* **JIT generator** lives in ``rpython/jit`` directory. optimizations live
-  in ``rpython/jit/metainterp/optimizeopt``, the main JIT in
-  ``rpython/jit/metainterp`` (runtime part) and
-  ``rpython/jit/codewriter`` (translation-time part). Backends live in
-  ``rpython/jit/backend``.
-
-* **garbage collection** lives in ``rpython/memory``
-
-The rest of directories serve specific niche goal and are unlikely a good
-entry point.
-
-
-More documentation
-------------------
-
-* `Getting Started Developing With PyPy`_
-
-* `Getting Started with RPython`_
-
-.. _`Getting Started Developing With PyPy`: getting-started-dev.html
-.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -34,6 +34,7 @@
    whatsnew-2.0.0-beta1.rst
    whatsnew-1.9.rst
 
+
 CPython 3.5 compatible versions
 -------------------------------
 
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -9,7 +9,7 @@
 
 * If you're interested in trying PyPy out, check out the :doc:`installation instructions <install>`.
 
-* If you want to help develop PyPy, please have a look at :doc:`how to contribute <how-to-contribute>`
+* If you want to help develop PyPy, please have a look at :doc:`contributing <contributing>`
   and get in touch (:ref:`contact`)!
 
 All of the documentation and source code is available under the MIT license,
@@ -31,6 +31,7 @@
   introduction
   install
   build
+  windows
   faq
 
 
@@ -44,39 +45,26 @@
 
   cpython_differences
   extending
-  embedding
   gc_info
   jit-hooks
   stackless
   __pypy__-module
-  objspace-proxies
   sandbox
   stm
-  windows
 
-
-.. _developing-pypy:
-
-Development documentation
--------------------------
+Development
+-----------
 
 .. toctree::
   :maxdepth: 1
 
-  getting-started-dev
-  how-to-contribute
-  you-want-to-help
+  contributing
   architecture
   configuration
   project-ideas
   project-documentation
   how-to-release
 
-.. TODO: audit ^^
-
-
-.. TODO: Fill this in
-
 
 Further resources
 -----------------
@@ -84,13 +72,10 @@
 .. toctree::
   :maxdepth: 1
 
-  extradoc
-  eventhistory
-  discussions
   index-of-release-notes
   index-of-whatsnew
   contributor
-
+  glossary
 
 .. _contact:
 
@@ -118,7 +103,7 @@
     the `development mailing list`_.
 
 .. _#pypy on irc.freenode.net: irc://irc.freenode.net/pypy
-.. _here: http://www.tismer.com/pypy/irc-logs/pypy/
+.. _here: https://botbot.me/freenode/pypy/
 .. _Development mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
 .. _Commit mailing list: http://mail.python.org/mailman/listinfo/pypy-commit
 .. _Development bug/feature tracker: https://bitbucket.org/pypy/pypy/issues
diff --git a/pypy/doc/interpreter.rst b/pypy/doc/interpreter.rst
--- a/pypy/doc/interpreter.rst
+++ b/pypy/doc/interpreter.rst
@@ -102,7 +102,7 @@
 program flows with homogeneous name-value assignments on
 function invocations.
 
-.. _how-to guide for descriptors: http://users.rcn.com/python/download/Descriptor.htm
+.. _how-to guide for descriptors: https://docs.python.org/3/howto/descriptor.html
 
 
 Bytecode Interpreter Implementation Classes
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -1,28 +1,7 @@
-What PyPy can do for your objects
-=================================
-
-.. contents::
-
-
-Thanks to the :doc:`Object Space <objspace>` architecture, any feature that is
-based on proxying, extending, changing or otherwise controlling the
-behavior of objects in a running program is easy to implement on top of PyPy.
-
-Here is what we have implemented so far, in historical order:
-
-* *Dump Object Space*: dumps all operations performed on all the objects
-  into a large log file.  For debugging your applications.
-
-* *Transparent Proxies Extension*: adds new proxy objects to
-  the Standard Object Space that enable applications to
-  control operations on application and builtin objects,
-  e.g lists, dictionaries, tracebacks.
-
-
 .. _tproxy:
 
-Transparent Proxies
--------------------
+Transparent Proxies (DEPRECATED)
+--------------------------------
 
 .. warning::
 
@@ -194,7 +173,7 @@
 application-level code.
 
 Transparent proxies are implemented on top of the :ref:`standard object
-space <standard-object-space>`, in :source:`pypy/objspace/std/proxy_helpers.py`,
+space <standard-object-space>`, in :source:`pypy/objspace/std/proxyobject.py`,
 :source:`pypy/objspace/std/proxyobject.py` and :source:`pypy/objspace/std/transparent.py`.
 To use them you will need to pass a `--objspace-std-withtproxy`_ option to ``pypy``
 or ``translate.py``.  This registers implementations named :py:class:`W_TransparentXxx`
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -474,8 +474,8 @@
 :source:`pypy/objspace/std/bytesobject.py` defines ``W_AbstractBytesObject``,
 which contains everything needed to build the ``str`` app-level type;
 and there are subclasses ``W_BytesObject`` (the usual string) and
-``W_StringBufferObject`` (a special implementation tweaked for repeated
-additions, in :source:`pypy/objspace/std/strbufobject.py`).  For mutable data
+``W_Buffer`` (a special implementation tweaked for repeated
+additions, in :source:`pypy/objspace/std/bufferobject.py`).  For mutable data
 types like lists and dictionaries, we have a single class
 ``W_ListObject`` or ``W_DictMultiObject`` which has an indirection to
 the real data and a strategy; the strategy can change as the content of
diff --git a/pypy/doc/project-documentation.rst b/pypy/doc/project-documentation.rst
--- a/pypy/doc/project-documentation.rst
+++ b/pypy/doc/project-documentation.rst
@@ -32,10 +32,13 @@
    coding-guide
    sprint-reports
    extradoc
+   eventhistory
    video-index
    index-report
+   discussions
    dev_method
-   glossary
+   embedding
+   objspace-proxies
 
 
 Source Code Documentation
diff --git a/pypy/doc/whatsnew-pypy2-5.10.0.rst b/pypy/doc/whatsnew-pypy2-5.10.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.10.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst
@@ -32,6 +32,7 @@
 
 .. branch: fix-vmprof-stacklet-switch
 .. branch: fix-vmprof-stacklet-switch-2
+
 Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...)
 
 .. branch: win32-vcvars
@@ -39,4 +40,3 @@
 .. branch: rdict-fast-hash
 
 Make it possible to declare that the hash function of an r_dict is fast in RPython.
-
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -25,7 +25,7 @@
 
 This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has
 made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
-was checked in Nov 2016). Note that the compiler suite may be installed in
+was checked in May 2018). Note that the compiler suite may be installed in
 ``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``
 or in
 ``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``.
@@ -37,7 +37,7 @@
 by running the mt.exe command by hand from a DOS window (that is how the author
 discovered the problem).
 
-.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
+.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/EN-US/DOWNLOAD/DETAILS.ASPX?ID=44266
 
 Installing "Build Tools for Visual Studio 2017" (for Python 3)
 --------------------------------------------------------------
@@ -150,7 +150,7 @@
 build dependencies for windows. As part of the `rpython` setup stage, environment
 variables will be set to use these dependencies. The repository has a README
 file on how to replicate, and a branch for each supported platform. You may run
- the `get_externals.py` utility to checkout the proper branch for your platform
+the `get_externals.py` utility to checkout the proper branch for your platform
 and PyPy version.
 
 .. _repository:  https://bitbucket.org/pypy/external
diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
deleted file mode 100644
--- a/pypy/doc/you-want-to-help.rst
+++ /dev/null
@@ -1,81 +0,0 @@
-You want to help with PyPy, now what?
-=====================================
-
-PyPy is a very large project that has a reputation of being hard to dive into.
-Some of this fame is warranted, some of it is purely accidental. There are three
-important lessons that everyone willing to contribute should learn:
-
-* PyPy has layers. There are many pieces of architecture that are very well
-  separated from each other. More about this below, but often the manifestation
-  of this is that things are at a different layer than you would expect them
-  to be. For example if you are looking for the JIT implementation, you will
-  not find it in the implementation of the Python programming language.
-
-* Because of the above, we are very serious about Test Driven Development.
-  It's not only what we believe in, but also that PyPy's architecture is
-  working very well with TDD in mind and not so well without it. Often
-  development means progressing in an unrelated corner, one unittest
-  at a time; and then flipping a giant switch, bringing it all together.
-  (It generally works out of the box.  If it doesn't, then we didn't
-  write enough unit tests.)  It's worth repeating - PyPy's
-  approach is great if you do TDD, and not so great otherwise.
-
-* PyPy uses an entirely different set of tools - most of them included
-  in the PyPy repository. There is no Makefile, nor autoconf. More below.
-
-
-Architecture
-------------
-
-PyPy has layers. The 100 miles view:
-
-* :ref:`RPython <rpython:language>` is the language in which we write interpreters. Not the entire
-  PyPy project is written in RPython, only the parts that are compiled in
-  the translation process. The interesting point is that RPython has no parser,
-  it's compiled from the live python objects, which makes it possible to do
-  all kinds of metaprogramming during import time. In short, Python is a meta
-  programming language for RPython.
-
-  The RPython standard library is to be found in the ``rlib`` subdirectory.
-
-* The translation toolchain - this is the part that takes care of translating
-  RPython to flow graphs and then to C. There is more in the :doc:`architecture <architecture>`
-  document written about it.
-
-  It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
-  and ``rtyper``.
-
-* Python Interpreter and modules
-
-  This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
-  interpreter for Python written in RPython.  The fact that it is
-  RPython is not apparent at first.  Built-in modules are written in
-  ``pypy/module/*``.  Some modules that CPython implements in C are
-  simply written in pure Python; they are in the top-level ``lib_pypy``
-  directory.  The standard library of Python (with a few changes to
-  accomodate PyPy) is in ``lib-python``.
-
-* :ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
-  interpreter written in RPython, rather than the user program that it
-  interprets.  As a result it applies to any interpreter, i.e. any
-  language.  But getting it to work correctly is not trivial: it
-  requires a small number of precise "hints" and possibly some small
-  refactorings of the interpreter.  The JIT itself also has several
-  almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
-  optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
-  residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
-  that turns it into machine code.  Writing a new backend is a
-  traditional way to get into the project.
-
-* Garbage Collectors (GC): as you may notice if you are used to CPython's
-  C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
-  :ref:`rpython:garbage-collection` is inserted
-  during translation.  Moreover, this is not reference counting; it is a real
-  GC written as more RPython code.  The best one we have so far is in
-  ``rpython/memory/gc/incminimark.py``.
-
-
-Toolset
--------
-
-xxx

From pypy.commits at gmail.com  Wed May  9 14:36:27 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 09 May 2018 11:36:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: start to document --runappdirect pytest
 option
Message-ID: <5af33fab.1c69fb81.51e2e.6464@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94518:82758e6d009a
Date: 2018-05-09 21:33 +0300
http://bitbucket.org/pypy/pypy/changeset/82758e6d009a/

Log:	start to document --runappdirect pytest option

diff --git a/pypy/doc/config/objspace.usemodules._cppyy.txt b/pypy/doc/config/objspace.usemodules._cppyy.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules._cppyy.txt
@@ -0,0 +1,1 @@
+The internal backend for cppyy
diff --git a/pypy/doc/config/objspace.usemodules._rawffi.txt b/pypy/doc/config/objspace.usemodules._rawffi.txt
--- a/pypy/doc/config/objspace.usemodules._rawffi.txt
+++ b/pypy/doc/config/objspace.usemodules._rawffi.txt
@@ -1,3 +1,3 @@
-An experimental module providing very low-level interface to
+A module providing very low-level interface to
 C-level libraries, for use when implementing ctypes, not
-intended for a direct use at all.
\ No newline at end of file
+intended for a direct use at all.
diff --git a/pypy/doc/config/objspace.usemodules.cpyext.txt b/pypy/doc/config/objspace.usemodules.cpyext.txt
--- a/pypy/doc/config/objspace.usemodules.cpyext.txt
+++ b/pypy/doc/config/objspace.usemodules.cpyext.txt
@@ -1,1 +1,1 @@
-Use (experimental) cpyext module, that tries to load and run CPython extension modules
+Use cpyext module to load and run CPython extension modules
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
--- a/pypy/doc/contributing.rst
+++ b/pypy/doc/contributing.rst
@@ -325,6 +325,24 @@
 .. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
 .. _`build requirements`: build.html#install-build-time-dependencies
 
+Testing After Translation
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While the usual invocation of `pytest` translates a piece of RPython code and
+runs it, we have a test extension to run tests without translation, directly
+on the host python. This is very convenient for modules such as `cpyext`, to
+compare and contrast test results between CPython and PyPy. Untranslated tests
+are invoked by using the `-A` or `--runappdirect` option to `pytest`::
+
+    python2 pytest.py -A pypy/module/cpyext/test
+
+where `python2` can be either `python2` or `pypy2`. On the `py3` branch, the
+collection phase must be run with `python2` so untranslated tests are run
+with::
+
+    cpython2 pytest.py -A pypy/module/cpyext/test --python=path/to/pypy3
+
+
 Tooling & Utilities
 ^^^^^^^^^^^^^^^^^^^
 
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -41,7 +41,7 @@
 ----------
 
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
   cpython_differences
   extending
@@ -56,7 +56,7 @@
 -----------
 
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
   contributing
   architecture

From pypy.commits at gmail.com  Wed May  9 14:36:29 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 09 May 2018 11:36:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: document more test requirments
Message-ID: <5af33fad.1c69fb81.48dcf.7a6a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94519:d728dfde7dc1
Date: 2018-05-09 21:35 +0300
http://bitbucket.org/pypy/pypy/changeset/d728dfde7dc1/

Log:	document more test requirments

diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
--- a/pypy/doc/contributing.rst
+++ b/pypy/doc/contributing.rst
@@ -293,7 +293,8 @@
 
 You will need the `build requirements`_ to run tests successfully, since many of
 them compile little pieces of PyPy and then run the tests inside that minimal
-interpreter
+interpreter. The `cpyext` tests also require `pycparser`, and many tests build
+cases with `hypothesis`.
 
 Now on to running some tests.  PyPy has many different test directories
 and you can use shell completion to point at directories or files::

From pypy.commits at gmail.com  Wed May  9 21:26:34 2018
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 09 May 2018 18:26:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3tests: fix test
Message-ID: <5af39fca.1c69fb81.968f2.dc81@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3tests
Changeset: r94520:64d59e349b56
Date: 2018-05-10 02:21 +0100
http://bitbucket.org/pypy/pypy/changeset/64d59e349b56/

Log:	fix test

diff --git a/pypy/module/_demo/test/test_import.py b/pypy/module/_demo/test/test_import.py
--- a/pypy/module/_demo/test/test_import.py
+++ b/pypy/module/_demo/test/test_import.py
@@ -1,30 +1,30 @@
-from pypy.module import _demo
+from pypy.module._demo.moduledef import Module
 from pypy.tool.option import make_config, make_objspace
 
 class TestImport:
 
     def setup_method(self, func):
-        _demo.Module.demo_events = []
+        Module.demo_events = []
 
     def test_startup(self):
         config = make_config(None, usemodules=('_demo',))
         space = make_objspace(config)
         w_modules = space.sys.get('modules')
 
-        assert _demo.Module.demo_events == ['setup']
+        assert Module.demo_events == ['setup']
         assert not space.contains_w(w_modules, space.wrap('_demo'))
 
         # first import
         w_import = space.builtin.get('__import__')
         w_demo = space.call(w_import,
                             space.newlist([space.wrap('_demo')]))
-        assert _demo.Module.demo_events == ['setup', 'startup']
+        assert Module.demo_events == ['setup', 'startup']
 
         # reload the module, this should not call startup again
         space.delitem(w_modules,
                       space.wrap('_demo'))
         w_demo = space.call(w_import,
                             space.newlist([space.wrap('_demo')]))
-        assert _demo.Module.demo_events == ['setup', 'startup']
+        assert Module.demo_events == ['setup', 'startup']
 
         assert space.getattr(w_demo, space.wrap('measuretime'))

From pypy.commits at gmail.com  Fri May 11 06:55:59 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 May 2018 03:55:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: don't crash if the locals aren't a dict.
 fix another corner case about
Message-ID: <5af576bf.1c69fb81.81364.52bc@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94521:ef3c287b0add
Date: 2018-05-11 12:55 +0200
http://bitbucket.org/pypy/pypy/changeset/ef3c287b0add/

Log:	don't crash if the locals aren't a dict. fix another corner case
	about annotations.

	(aside: this shows that removing the _settled_ = True feature was
	not a good idea, as it would have caught this bug)

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -849,7 +849,7 @@
             if not e.match(self.space, self.space.w_KeyError):
                 raise
             raise oefmt(self.space.w_NameError,
-                        "name %R is not defined", w_varname)
+                        "__annotations__ not found")
 
     def UNPACK_SEQUENCE(self, itemcount, next_instr):
         w_iterable = self.popvalue()
@@ -958,10 +958,17 @@
             self.space.setitem_str(w_locals, '__annotations__', w_annotations)
 
     def STORE_ANNOTATION(self, varindex, next_instr):
+        space = self.space
         varname = self.getname_u(varindex)
         w_newvalue = self.popvalue()
-        self.space.setitem_str(self.getorcreatedebug().w_locals.getitem_str('__annotations__'), varname,
-                               w_newvalue)
+        w_locals = self.getorcreatedebug().w_locals
+        try:
+            w_annotations = space.getitem(w_locals, space.newtext('__annotations__'))
+        except OperationError as e:
+            if e.match(space, space.w_KeyError):
+                raise oefmt(space.w_NameError, CANNOT_CATCH_MSG)
+            raise
+        self.space.setitem_str(w_annotations, varname, w_newvalue)
 
     def BUILD_TUPLE(self, itemcount, next_instr):
         items = self.popvalues(itemcount)
diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py
--- a/pypy/interpreter/test/test_annotations.py
+++ b/pypy/interpreter/test/test_annotations.py
@@ -111,3 +111,24 @@
             except:
                 assert False
         '''
+
+    def test_locals_arent_dicts(self):
+        class O:
+            def __init__(self):
+                self.dct = {}
+            def __getitem__(self, name):
+                return self.dct[name]
+            def __setitem__(self, name, value):
+                self.dct[name] = value
+        # don't crash if locals aren't just a normal dict
+        exec("a: int; assert __annotations__['a'] == int", {}, O())
+
+    def test_NameError_if_annotations_are_gone(self):
+        exec("""if 1:
+            raises(NameError, '''if 1:
+                class A:
+                    del __annotations__
+                    a: int
+            ''')
+        """)
+

From pypy.commits at gmail.com  Fri May 11 08:17:29 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 May 2018 05:17:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix merge problem
Message-ID: <5af589d9.1c69fb81.b83ec.4a3f@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94522:d09a53f84bea
Date: 2018-05-11 12:58 +0200
http://bitbucket.org/pypy/pypy/changeset/d09a53f84bea/

Log:	fix merge problem

diff --git a/lib-python/3/test/test_inspect.py b/lib-python/3/test/test_inspect.py
--- a/lib-python/3/test/test_inspect.py
+++ b/lib-python/3/test/test_inspect.py
@@ -32,7 +32,7 @@
 from test import support
 
 from test.test_import import _ready_to_import
-if check_impl_detail():
+if support.check_impl_detail():
     import _pickle
 
 

From pypy.commits at gmail.com  Fri May 11 08:17:31 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 11 May 2018 05:17:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix error handling in __set_name__
Message-ID: <5af589db.1c69fb81.9d848.8461@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94523:787eff4a164e
Date: 2018-05-11 14:16 +0200
http://bitbucket.org/pypy/pypy/changeset/787eff4a164e/

Log:	fix error handling in __set_name__

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1556,6 +1556,17 @@
         assert X.a.owner is X
         assert X.a.name == "a"
 
+    def test_set_name_error(self):
+        class Descriptor:
+            __set_name__ = None
+        def make_class():
+            class A:
+                d = Descriptor()
+        excinfo = raises(RuntimeError, make_class)
+        assert isinstance(excinfo.value.__cause__, TypeError)
+        assert str(excinfo.value) == "Error calling __set_name__ on 'Descriptor' instance 'd' in 'A'"
+        print(excinfo.value)
+
     def test_type_init_accepts_kwargs(self):
         type.__init__(type, "a", (object, ), {}, a=1)
 
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -835,8 +835,14 @@
     for key, w_value in w_type.dict_w.iteritems():
         w_meth = space.lookup(w_value, '__set_name__')
         if w_meth is not None:
-            # XXX what happens when the call raises, gets turned into a RuntimeError?
-            space.get_and_call_function(w_meth, w_value, w_type, space.newtext(key))
+            try:
+                space.get_and_call_function(w_meth, w_value, w_type, space.newtext(key))
+            except OperationError as e:
+                e2 = oefmt(space.w_RuntimeError,
+                           "Error calling __set_name__ on '%T' instance '%s' in '%N'",
+                           w_value, key, w_type)
+                e2.chain_exceptions_from_cause(space, e)
+                raise e2
 
 def _init_subclass(space, w_type, __args__):
     # bit of a mess, but I didn't feel like implementing the super logic

From pypy.commits at gmail.com  Fri May 11 09:16:07 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 11 May 2018 06:16:07 -0700 (PDT)
Subject: [pypy-commit] pypy gc-more-logging: log some more info about the
 various GC phases
Message-ID: <5af59797.1c69fb81.203d8.3c87@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-more-logging
Changeset: r94524:f1eb76182c61
Date: 2018-05-11 15:15 +0200
http://bitbucket.org/pypy/pypy/changeset/f1eb76182c61/

Log:	log some more info about the various GC phases

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1836,6 +1836,7 @@
         debug_print("minor collect, total memory used:", total_memory_used)
         debug_print("number of pinned objects:",
                     self.pinned_objects_in_nursery)
+        debug_print("total size of surviving objects:", self.nursery_surviving_size)
         if self.DEBUG >= 2:
             self.debug_check_consistency()     # expensive!
         #
@@ -2401,7 +2402,9 @@
                 # a total object size of at least '3 * nursery_size' bytes
                 # is processed.
                 limit = 3 * self.nursery_size // self.small_request_threshold
-                self.free_unvisited_rawmalloc_objects_step(limit)
+                nobjects = self.free_unvisited_rawmalloc_objects_step(limit)
+                debug_print("freeing raw objects:", limit-nobjects,
+                            "freed, limit was", limit)
                 done = False    # the 2nd half below must still be done
             else:
                 # Ask the ArenaCollection to visit a fraction of the objects.
@@ -2411,6 +2414,8 @@
                 limit = 3 * self.nursery_size // self.ac.page_size
                 done = self.ac.mass_free_incremental(self._free_if_unvisited,
                                                      limit)
+                status = done and "No more pages left." or "More to do."
+                debug_print("freeing GC objects, up to", limit, "pages.", status)
             # XXX tweak the limits above
             #
             if done:

From pypy.commits at gmail.com  Sat May 12 02:52:33 2018
From: pypy.commits at gmail.com (arigo)
Date: Fri, 11 May 2018 23:52:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2818
Message-ID: <5af68f31.1c69fb81.1355a.10ae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94525:5d5b4b989e7b
Date: 2018-05-12 08:51 +0200
http://bitbucket.org/pypy/pypy/changeset/5d5b4b989e7b/

Log:	Issue #2818

	__module__ attribute should be writable on cpyext built-in functions

diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -142,6 +142,17 @@
         else:
             return space.w_None
 
+    def fget_module(self, space):
+        if self.w_module is None:
+            return space.w_None
+        return self.w_module
+
+    def fset_module(self, space, w_module):
+        self.w_module = w_module
+
+    def fdel_module(self, space):
+        self.w_module = space.w_None
+
 class W_PyCMethodObject(W_PyCFunctionObject):
 
     def __init__(self, space, ml, w_type):
@@ -305,7 +316,9 @@
     'builtin_function_or_method',
     __call__ = interp2app(W_PyCFunctionObject.descr_call),
     __doc__ = GetSetProperty(W_PyCFunctionObject.get_doc),
-    __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject),
+    __module__ = GetSetProperty(W_PyCFunctionObject.fget_module,
+                                W_PyCFunctionObject.fset_module,
+                                W_PyCFunctionObject.fdel_module),
     __name__ = interp_attrproperty('name', cls=W_PyCFunctionObject,
         wrapfn="newtext_or_none"),
     )
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py
--- a/pypy/module/cpyext/test/test_methodobject.py
+++ b/pypy/module/cpyext/test/test_methodobject.py
@@ -192,3 +192,17 @@
         assert mod.check(A) == 0
         assert mod.check(A.meth) == 0
         assert mod.check(A.stat) == 0
+
+    def test_module_attribute(self):
+        mod = self.import_extension('MyModule', [
+            ('getarg_NO', 'METH_NOARGS',
+             '''
+                 Py_INCREF(Py_None);
+                 return Py_None;
+             '''
+             ),
+            ])
+        assert mod.getarg_NO() is None
+        assert mod.getarg_NO.__module__ == 'MyModule'
+        mod.getarg_NO.__module__ = 'foobar'
+        assert mod.getarg_NO.__module__ == 'foobar'

From pypy.commits at gmail.com  Sat May 12 03:14:46 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 May 2018 00:14:46 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2820
Message-ID: <5af69466.1c69fb81.34361.b116@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94526:a91492fa1e35
Date: 2018-05-12 09:14 +0200
http://bitbucket.org/pypy/pypy/changeset/a91492fa1e35/

Log:	Issue #2820

	Implement Py_ReprEnter() and Py_ReprLeave()

diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -11,6 +11,7 @@
 from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.executioncontext import ExecutionContext
 import pypy.module.__builtin__.operation as operation
 
 
@@ -411,3 +412,27 @@
 def _PyPyGC_AddMemoryPressure(space, report):
     from rpython.rlib import rgc
     rgc.add_memory_pressure(report)
+
+
+ExecutionContext.cpyext_recursive_repr = None
+
+ at cpython_api([PyObject], rffi.INT_real, error=-1)
+def Py_ReprEnter(space, w_obj):
+    ec = space.getexecutioncontext()
+    d = ec.cpyext_recursive_repr
+    if d is None:
+        d = ec.cpyext_recursive_repr = {}
+    if w_obj in d:
+        return 1
+    d[w_obj] = None
+    return 0
+
+ at cpython_api([PyObject], lltype.Void)
+def Py_ReprLeave(space, w_obj):
+    ec = space.getexecutioncontext()
+    d = ec.cpyext_recursive_repr
+    if d is not None:
+        try:
+            del d[w_obj]
+        except KeyError:
+            pass
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -403,6 +403,47 @@
         module.foo(35000)
         assert self.cur_memory_pressure() == 65536 + 80000 + 70000
 
+    def test_repr_enter_leave(self):
+        module = self.import_extension('foo', [
+            ("enter", "METH_O",
+            """
+                return PyInt_FromLong(Py_ReprEnter(args));
+            """),
+            ("leave", "METH_O",
+            """
+                Py_ReprLeave(args);
+                Py_INCREF(Py_None);
+                return Py_None;
+            """)])
+        obj1 = [42]
+        obj2 = [42]   # another list
+
+        n = module.enter(obj1)
+        assert n == 0
+        module.leave(obj1)
+
+        n = module.enter(obj1)
+        assert n == 0
+        n = module.enter(obj1)
+        assert n == 1
+        n = module.enter(obj1)
+        assert n == 1
+        module.leave(obj1)
+
+        n = module.enter(obj1)
+        assert n == 0
+        n = module.enter(obj2)
+        assert n == 0
+        n = module.enter(obj1)
+        assert n == 1
+        n = module.enter(obj2)
+        assert n == 1
+        module.leave(obj1)
+        n = module.enter(obj2)
+        assert n == 1
+        module.leave(obj2)
+
+
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """
     PyBuffer_FillInfo populates the fields of a Py_buffer from its arguments.

From pypy.commits at gmail.com  Sat May 12 03:48:18 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 12 May 2018 00:48:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2823
Message-ID: <5af69c42.1c69fb81.f6732.4e98@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94527:d18c31e806d6
Date: 2018-05-12 09:47 +0200
http://bitbucket.org/pypy/pypy/changeset/d18c31e806d6/

Log:	Issue #2823

	Implement PyMarshal_ReadObjectFromString and
	PyMarshal_WriteObjectToString

diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -77,6 +77,7 @@
 import pypy.module.cpyext.pystrtod
 import pypy.module.cpyext.pytraceback
 import pypy.module.cpyext.methodobject
+import pypy.module.cpyext.marshal
 
 # now that all rffi_platform.Struct types are registered, configure them
 api.configure_types()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -60,7 +60,7 @@
 
 configure_eci = ExternalCompilationInfo(
         include_dirs=include_dirs,
-        includes=['Python.h', 'stdarg.h', 'structmember.h'],
+        includes=['Python.h', 'stdarg.h', 'structmember.h', 'marshal.h'],
         compile_extra=['-DPy_BUILD_CORE'])
 
 class CConfig:
@@ -118,6 +118,7 @@
 pypy_decl = 'pypy_decl.h'
 udir.join(pypy_decl).write("/* Will be filled later */\n")
 udir.join('pypy_structmember_decl.h').write("/* Will be filled later */\n")
+udir.join('pypy_marshal_decl.h').write("/* Will be filled later */\n")
 udir.join('pypy_macros.h').write("/* Will be filled later */\n")
 
 constant_names = """
@@ -1221,14 +1222,11 @@
         global_objects.append('PyTypeObject _PyExc_%s;' % name)
     global_code = '\n'.join(global_objects)
 
-    prologue = ("#include <Python.h>\n"
-                "#include <structmember.h>\n"
+    prologue = ("#include <Python.h>\n" +
+                "#include <structmember.h>\n" +
+                "#include <marshal.h>\n" +
+                ("#include <pypy_numpy.h>\n" if use_micronumpy else "") +
                 "#include <src/thread.c>\n")
-    if use_micronumpy:
-        prologue = ("#include <Python.h>\n"
-                    "#include <structmember.h>\n"
-                    "#include <pypy_numpy.h>\n"
-                    "#include <src/thread.c>\n")
     code = (prologue +
             struct_declaration_code +
             global_code +
diff --git a/pypy/module/cpyext/include/marshal.h b/pypy/module/cpyext/include/marshal.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/marshal.h
@@ -0,0 +1,13 @@
+#ifndef Py_MARSHAL_H
+#define Py_MARSHAL_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define Py_MARSHAL_VERSION 2
+#include "pypy_marshal_decl.h"
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_MARSHAL_H */
diff --git a/pypy/module/cpyext/marshal.py b/pypy/module/cpyext/marshal.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/marshal.py
@@ -0,0 +1,17 @@
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.module.cpyext.api import cpython_api, Py_ssize_t
+from pypy.module.cpyext.pyobject import PyObject
+
+
+_HEADER = 'pypy_marshal_decl.h'
+
+ at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject, header=_HEADER)
+def PyMarshal_ReadObjectFromString(space, p, size):
+    from pypy.module.marshal.interp_marshal import loads
+    s = rffi.charpsize2str(p, size)
+    return loads(space, space.newbytes(s))
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject, header=_HEADER)
+def PyMarshal_WriteObjectToString(space, w_x, version):
+    from pypy.module.marshal.interp_marshal import dumps
+    return dumps(space, w_x, space.newint(version))
diff --git a/pypy/module/cpyext/test/test_marshal.py b/pypy/module/cpyext/test/test_marshal.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_marshal.py
@@ -0,0 +1,33 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
+class AppTestMarshal(AppTestCpythonExtensionBase):
+    def test_PyMarshal_ReadObjectFromString(self):
+        module = self.import_extension('foo', [
+            ("mloads", "METH_O",
+             """
+                 char *input = PyString_AsString(args);
+                 Py_ssize_t length = PyString_Size(args);
+                 return PyMarshal_ReadObjectFromString(input, length);
+             """)],
+            prologue='#include <marshal.h>')
+        import marshal
+        assert module.mloads(marshal.dumps(42.5)) == 42.5
+        x = [None, True, (4, 5), b"adkj", u"\u1234"]
+        assert module.mloads(marshal.dumps(x)) == x
+
+    def test_PyMarshal_WriteObjectToString(self):
+        module = self.import_extension('foo', [
+            ("mdumps", "METH_VARARGS",
+             """
+                 PyObject *obj;
+                 int version;
+                 if (!PyArg_ParseTuple(args, "Oi", &obj, &version))
+                     return NULL;
+                 return PyMarshal_WriteObjectToString(obj, version);
+             """)],
+            prologue='#include <marshal.h>')
+        import marshal
+        for x in [42, b"foo", u"\u2345", (4, None, False)]:
+            for version in [0, 1, 2]:
+                assert module.mdumps(x, version) == marshal.dumps(x, version)

From pypy.commits at gmail.com  Sat May 12 10:26:40 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 12 May 2018 07:26:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: implement the new way the __class__
 closure is filled (via type.__new__ and the
Message-ID: <5af6f9a0.1c69fb81.878c4.96a9@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94528:92cfcd232b4f
Date: 2018-05-12 16:26 +0200
http://bitbucket.org/pypy/pypy/changeset/92cfcd232b4f/

Log:	implement the new way the __class__ closure is filled (via
	type.__new__ and the new __classcell__ element in the class body
	dict)

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1826,6 +1826,8 @@
         if scope == symtable.SCOPE_CELL_CLASS:
             # Return the cell where to store __class__
             self.emit_op_arg(ops.LOAD_CLOSURE, self.cell_vars["__class__"])
+            self.emit_op(ops.DUP_TOP)
+            self.name_op("__classcell__", ast.Store)
         else:
             # This happens when nobody references the cell
             self.load_const(self.space.w_None)
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -1109,6 +1109,74 @@
         assert f() == (4, 3, 2, 1), repr(f())
         """
 
+    def test_classcell(self):
+        """
+        test_class = None
+        class Meta(type):
+            def __new__(cls, name, bases, namespace):
+                nonlocal test_class
+                self = super().__new__(cls, name, bases, namespace)
+                test_class = self.f()
+                return self
+        class A(metaclass=Meta):
+            @staticmethod
+            def f():
+                return __class__
+        assert test_class is A
+        """
+
+    def test_classcell_missing(self):
+        """
+        # Some metaclasses may not pass the original namespace to type.__new__
+        # We test that case here by forcibly deleting __classcell__
+        class Meta(type):
+            def __new__(cls, name, bases, namespace):
+                namespace.pop('__classcell__', None)
+                return super().__new__(cls, name, bases, namespace)
+
+        class WithClassRef(metaclass=Meta):
+            def f(self):
+                return __class__
+
+        # Check __class__ still gets set despite the warning
+        assert WithClassRef().f() is WithClassRef
+        """
+
+    def test_classcell_overwrite(self):
+        """
+        # Overwriting __classcell__ with nonsense is explicitly prohibited
+        class Meta(type):
+            def __new__(cls, name, bases, namespace, cell):
+                namespace['__classcell__'] = cell
+                return super().__new__(cls, name, bases, namespace)
+
+        raises(TypeError, '''if 1:
+            class A(metaclass=Meta, cell=object()):
+                pass
+        ''')
+        """
+
+    def test_classcell_wrong_cell(self):
+        """
+        # Pointing the cell reference at the wrong class is prohibited
+        class Meta(type):
+            def __new__(cls, name, bases, namespace):
+                cls = super().__new__(cls, name, bases, namespace)
+                B = type("B", (), namespace)
+                return cls
+
+        # works, no __class__
+        class A(metaclass=Meta):
+            pass
+
+        raises(TypeError, '''if 1:
+            class A(metaclass=Meta):
+                def f(self):
+                    return __class__
+        ''')
+
+        """
+
 
 class AppTestOptimizer(object):
     def setup_class(cls):
@@ -1438,3 +1506,5 @@
         assert eval(code) == u'\xc2\u20ac'
         code = b'u"""\\\n# -*- coding: ascii -*-\n\xc2\xa4"""\n'
         assert eval(code) == u'# -*- coding: ascii -*-\n\xa4'
+
+
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -93,6 +93,8 @@
     frame.exec_(w_prog, w_globals, w_locals)
 
 def build_class(space, w_func, w_name, __args__):
+    from pypy.objspace.std.typeobject import _calculate_metaclass, W_TypeObject
+    from pypy.interpreter.nestedscope import Cell
     if not isinstance(w_func, Function):
         raise oefmt(space.w_TypeError, "__build_class__: func must be a function")
     bases_w, kwds_w = __args__.unpack()
@@ -109,7 +111,6 @@
     if isclass:
         # w_meta is really a class, so check for a more derived
         # metaclass, or possible metaclass conflicts
-        from pypy.objspace.std.typeobject import _calculate_metaclass
         w_meta = _calculate_metaclass(space, w_meta, bases_w)
 
     try:
@@ -135,6 +136,22 @@
                      keywords=keywords,
                      keywords_w=kwds_w.values())
     w_class = space.call_args(w_meta, args)
-    if isinstance(w_cell, Cell):
-        w_cell.set(w_class)
+    if isinstance(w_cell, Cell) and isinstance(w_class, W_TypeObject):
+        if w_cell.empty():
+            # will become an error in Python 3.7
+            space.warn(space.newtext(
+                "__class__ not set defining %s as %s ."
+                "Was __classcell__ propagated to type.__new__?" % (
+                    space.text_w(w_name),
+                    space.text_w(space.str(w_class))
+                )),
+                space.w_DeprecationWarning)
+            w_cell.set(w_class)
+        else:
+            w_class_from_cell = w_cell.get()
+            if not space.is_w(w_class, w_class_from_cell):
+                raise oefmt(
+                        space.w_TypeError,
+                        "__class__ set to %s defining %s as %s",
+                        w_class_from_cell, w_name, w_class)
     return w_class
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -805,12 +805,28 @@
     w_type = space.allocate_instance(W_TypeObject, w_typetype)
     W_TypeObject.__init__(w_type, space, name, bases_w or [space.w_object],
                           dict_w, is_heaptype=True)
+
+    # store the w_type in __classcell__
+    w_classcell = dict_w.get("__classcell__")
+    if w_classcell:
+        _store_type_in_classcell(space, w_type, w_classcell, dict_w)
+
     w_type.ready()
 
     _set_names(space, w_type)
     _init_subclass(space, w_type, __args__)
     return w_type
 
+def _store_type_in_classcell(space, w_type, w_classcell, dict_w):
+    from pypy.interpreter.nestedscope import Cell
+    if isinstance(w_classcell, Cell):
+        w_classcell.set(w_type)
+    else:
+        raise oefmt(space.w_TypeError,
+                    "__classcell__ must be a nonlocal cell, not %T",
+                    w_classcell)
+    del dict_w['__classcell__']
+
 def _calculate_metaclass(space, w_metaclass, bases_w):
     """Determine the most derived metatype"""
     w_winner = w_metaclass

From pypy.commits at gmail.com  Sat May 12 10:31:01 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 12 May 2018 07:31:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix translation?
Message-ID: <5af6faa5.1c69fb81.754ae.4253@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94529:54d322f20c77
Date: 2018-05-12 16:30 +0200
http://bitbucket.org/pypy/pypy/changeset/54d322f20c77/

Log:	fix translation?

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -807,7 +807,7 @@
                           dict_w, is_heaptype=True)
 
     # store the w_type in __classcell__
-    w_classcell = dict_w.get("__classcell__")
+    w_classcell = dict_w.get("__classcell__", None)
     if w_classcell:
         _store_type_in_classcell(space, w_type, w_classcell, dict_w)
 

From pypy.commits at gmail.com  Sat May 12 11:37:00 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 12 May 2018 08:37:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix another problem
Message-ID: <5af70a1c.1c69fb81.b4758.4fbb@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94530:b557b77bd013
Date: 2018-05-12 17:29 +0200
http://bitbucket.org/pypy/pypy/changeset/b557b77bd013/

Log:	fix another problem

diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -152,6 +152,6 @@
             if not space.is_w(w_class, w_class_from_cell):
                 raise oefmt(
                         space.w_TypeError,
-                        "__class__ set to %s defining %s as %s",
+                        "__class__ set to %S defining %S as %S",
                         w_class_from_cell, w_name, w_class)
     return w_class

From pypy.commits at gmail.com  Sat May 12 11:48:03 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 12 May 2018 08:48:03 -0700 (PDT)
Subject: [pypy-commit] pypy default: rpython support for dict.get(key) (ie
 without giving a default)
Message-ID: <5af70cb3.1c69fb81.b7a8c.9dab@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94531:85d7ccafed44
Date: 2018-05-12 16:47 +0200
http://bitbucket.org/pypy/pypy/changeset/85d7ccafed44/

Log:	rpython support for dict.get(key) (ie without giving a default)

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -688,6 +688,16 @@
         assert isinstance(dictvalue(s), annmodel.SomeInteger)
         assert not dictvalue(s).nonneg
 
+    def test_dict_get(self):
+        def f1(i, j):
+            d = {i: ''}
+            return d.get(j)
+        a = self.RPythonAnnotator()
+        s = a.build_types(f1, [int, int])
+        assert isinstance(s, annmodel.SomeString)
+        assert s.can_be_None
+
+
     def test_exception_deduction(self):
         a = self.RPythonAnnotator()
         s = a.build_types(snippet.exception_deduction, [])
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -496,7 +496,7 @@
             return SomeTuple((s_key, s_Int))
         raise ValueError(variant)
 
-    def method_get(self, key, dfl):
+    def method_get(self, key, dfl=s_None):
         position = getbookkeeper().position_key
         self.dictdef.generalize_key(key)
         self.dictdef.generalize_value(dfl)
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -283,8 +283,12 @@
         return DictIteratorRepr(self, *variant)
 
     def rtype_method_get(self, hop):
-        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
-                                                 self.value_repr)
+        if hop.nb_args == 3:
+            v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                     self.value_repr)
+        else:
+            v_dict, v_key = hop.inputargs(self, self.key_repr)
+            v_default = hop.inputconst(self.value_repr, None)
         hop.exception_cannot_occur()
         v_res = hop.gendirectcall(ll_dict_get, v_dict, v_key, v_default)
         return self.recast_value(hop.llops, v_res)
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -208,6 +208,16 @@
         res = self.interpret(func, ())
         assert res == 421
 
+    def test_dict_get_no_second_arg(self):
+        def func():
+            dic = self.newdict()
+            x1 = dic.get('hi', 'a')
+            x2 = dic.get('blah')
+            return (x1 == 'a') * 10 + (x2 is None)
+            return x1 * 10 + x2
+        res = self.interpret(func, ())
+        assert res == 11
+
     def test_dict_get_empty(self):
         def func():
             # this time without writing to the dict

From pypy.commits at gmail.com  Sat May 12 13:05:20 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 12 May 2018 10:05:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: backport a couple cosmetic changes from
 branch unicode-utf8
Message-ID: <5af71ed0.1c69fb81.34361.379f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r94532:0a8401a58e9f
Date: 2018-05-12 18:04 +0100
http://bitbucket.org/pypy/pypy/changeset/0a8401a58e9f/

Log:	backport a couple cosmetic changes from branch unicode-utf8

diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -499,6 +499,7 @@
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
+    @staticmethod
     def make_fastfunc(unwrap_spec, func):
         unwrap_info = UnwrapSpec_FastFunc_Unwrap()
         unwrap_info.apply_over(unwrap_spec)
@@ -529,7 +530,6 @@
             exec compile2(source) in unwrap_info.miniglobals, d
             fastfunc = d['fastfunc_%s_%d' % (func.__name__.replace('-', '_'), narg)]
         return narg, fastfunc
-    make_fastfunc = staticmethod(make_fastfunc)
 
 
 def int_unwrapping_space_method(typ):
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -27,11 +27,9 @@
 @unwrap_spec(code=int)
 def unichr(space, code):
     "Return a Unicode string of one character with the given ordinal."
-    # XXX range checking!
-    try:
-        c = UNICHR(code)
-    except ValueError:
+    if code < 0 or code > 0x10FFFF:
         raise oefmt(space.w_ValueError, "unichr() arg out of range")
+    c = UNICHR(code)
     return space.newunicode(c)
 
 def len(space, w_obj):

From pypy.commits at gmail.com  Sat May 12 13:10:34 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 12 May 2018 10:10:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix
Message-ID: <5af7200a.1c69fb81.cf708.e4d6@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94533:6846408f5ccc
Date: 2018-05-12 18:36 +0200
http://bitbucket.org/pypy/pypy/changeset/6846408f5ccc/

Log:	fix

diff --git a/lib-python/3/test/test_inspect.py b/lib-python/3/test/test_inspect.py
--- a/lib-python/3/test/test_inspect.py
+++ b/lib-python/3/test/test_inspect.py
@@ -768,7 +768,7 @@
     @unittest.skipIf(MISSING_C_DOCSTRINGS,
                      "Signature information for builtins requires docstrings")
     def test_getfullargspec_builtin_methods(self):
-        if check_impl_detail():
+        if support.check_impl_detail():
             self.assertFullArgSpecEquals(_pickle.Pickler.dump,
                                         args_e=['self', 'obj'], formatted='(self, obj)')
 
@@ -2012,7 +2012,7 @@
 
         # normal method
         # (PyMethodDescr_Type, "method_descriptor")
-        if check_impl_detail():
+        if support.check_impl_detail():
             test_unbound_method(_pickle.Pickler.dump)
             d = _pickle.Pickler(io.StringIO())
             test_callable(d.dump)

From pypy.commits at gmail.com  Sat May 12 13:10:37 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 12 May 2018 10:10:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fill the classcell even earlier
Message-ID: <5af7200d.1c69fb81.7b8ea.b8a0@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94534:4ea0853cd59f
Date: 2018-05-12 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/4ea0853cd59f/

Log:	fill the classcell even earlier

diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -1109,6 +1109,8 @@
         assert f() == (4, 3, 2, 1), repr(f())
         """
 
+    # the following couple of tests are from test_super.py in the stdlib
+
     def test_classcell(self):
         """
         test_class = None
@@ -1177,6 +1179,24 @@
 
         """
 
+    def test_class_mro(self):
+        """
+        test_class = None
+
+        class Meta(type):
+            def mro(self):
+                # self.f() doesn't work yet...
+                self.__dict__["f"]()
+                return super().mro()
+
+        class A(metaclass=Meta):
+            def f():
+                nonlocal test_class
+                test_class = __class__
+
+        assert test_class is A
+        """
+
 
 class AppTestOptimizer(object):
     def setup_class(cls):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -803,14 +803,16 @@
         key = space.text_w(w_key)
         dict_w[key] = space.getitem(w_dict, w_key)
     w_type = space.allocate_instance(W_TypeObject, w_typetype)
-    W_TypeObject.__init__(w_type, space, name, bases_w or [space.w_object],
-                          dict_w, is_heaptype=True)
 
     # store the w_type in __classcell__
     w_classcell = dict_w.get("__classcell__", None)
     if w_classcell:
         _store_type_in_classcell(space, w_type, w_classcell, dict_w)
 
+    W_TypeObject.__init__(w_type, space, name, bases_w or [space.w_object],
+                          dict_w, is_heaptype=True)
+
+
     w_type.ready()
 
     _set_names(space, w_type)

From pypy.commits at gmail.com  Sat May 12 13:32:36 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 12 May 2018 10:32:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: missing space in error message
Message-ID: <5af72534.1c69fb81.68296.cabf@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94536:2341c48345f8
Date: 2018-05-12 19:23 +0200
http://bitbucket.org/pypy/pypy/changeset/2341c48345f8/

Log:	missing space in error message

diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -140,7 +140,7 @@
         if w_cell.empty():
             # will become an error in Python 3.7
             space.warn(space.newtext(
-                "__class__ not set defining %s as %s ."
+                "__class__ not set defining %s as %s . "
                 "Was __classcell__ propagated to type.__new__?" % (
                     space.text_w(w_name),
                     space.text_w(space.str(w_class))

From pypy.commits at gmail.com  Sat May 12 13:32:34 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 12 May 2018 10:32:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: can't use the one-argument form of
 type.__new__ for subclasses of type
Message-ID: <5af72532.1c69fb81.e5065.371e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94535:7c4bb9c6272f
Date: 2018-05-12 19:19 +0200
http://bitbucket.org/pypy/pypy/changeset/7c4bb9c6272f/

Log:	can't use the one-argument form of type.__new__ for subclasses of
	type

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1607,3 +1607,9 @@
 
         assert Y.kwargs == dict(a=1, b=2)
         """)
+
+    def test_onearg_type_only_for_type(self):
+        class Meta(type):
+            pass
+
+        raises(TypeError, Meta, 5)
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -759,10 +759,14 @@
 
     w_typetype = _precheck_for_new(space, w_typetype)
 
-    # special case for type(x)
-    if (space.is_w(space.type(w_typetype), space.w_type) and
-            len(__args__.arguments_w) == 1):
-        return space.type(w_name)
+    # special case for type(x), but not Metaclass(x)
+    if len(__args__.arguments_w) == 1:
+        if space.is_w(w_typetype, space.w_type):
+            return space.type(w_name)
+        else:
+            raise oefmt(space.w_TypeError,
+                        "%N.__new__() takes 3 arguments (1 given)",
+                        w_typetype)
     w_bases = __args__.arguments_w[1]
     w_dict = __args__.arguments_w[2]
     return _create_new_type(space, w_typetype, w_name, w_bases, w_dict, __args__)

From pypy.commits at gmail.com  Sat May 12 13:36:47 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 12 May 2018 10:36:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: bump magic because of classcell changes
Message-ID: <5af7262f.1c69fb81.a0d11.7819@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94537:ba88f1c25694
Date: 2018-05-12 19:36 +0200
http://bitbucket.org/pypy/pypy/changeset/ba88f1c25694/

Log:	bump magic because of classcell changes

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -39,7 +39,7 @@
 # time you make pyc files incompatible.  This value ends up in the frozen
 # importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
-pypy_incremental_magic = 112 # bump it by 16
+pypy_incremental_magic = 128 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value

From pypy.commits at gmail.com  Sat May 12 14:02:39 2018
From: pypy.commits at gmail.com (mattip)
Date: Sat, 12 May 2018 11:02:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge default into py3.5
Message-ID: <5af72c3f.1c69fb81.73fa4.3f15@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94538:5f017005bd26
Date: 2018-05-12 21:01 +0300
http://bitbucket.org/pypy/pypy/changeset/5f017005bd26/

Log:	merge default into py3.5

diff --git a/pypy/doc/config/objspace.usemodules._cppyy.txt b/pypy/doc/config/objspace.usemodules._cppyy.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules._cppyy.txt
@@ -0,0 +1,1 @@
+The internal backend for cppyy
diff --git a/pypy/doc/config/objspace.usemodules._rawffi.txt b/pypy/doc/config/objspace.usemodules._rawffi.txt
--- a/pypy/doc/config/objspace.usemodules._rawffi.txt
+++ b/pypy/doc/config/objspace.usemodules._rawffi.txt
@@ -1,3 +1,3 @@
-An experimental module providing very low-level interface to
+A module providing very low-level interface to
 C-level libraries, for use when implementing ctypes, not
-intended for a direct use at all.
\ No newline at end of file
+intended for a direct use at all.
diff --git a/pypy/doc/config/objspace.usemodules.cpyext.txt b/pypy/doc/config/objspace.usemodules.cpyext.txt
--- a/pypy/doc/config/objspace.usemodules.cpyext.txt
+++ b/pypy/doc/config/objspace.usemodules.cpyext.txt
@@ -1,1 +1,1 @@
-Use (experimental) cpyext module, that tries to load and run CPython extension modules
+Use cpyext module to load and run CPython extension modules
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
--- a/pypy/doc/contributing.rst
+++ b/pypy/doc/contributing.rst
@@ -293,7 +293,8 @@
 
 You will need the `build requirements`_ to run tests successfully, since many of
 them compile little pieces of PyPy and then run the tests inside that minimal
-interpreter
+interpreter. The `cpyext` tests also require `pycparser`, and many tests build
+cases with `hypothesis`.
 
 Now on to running some tests.  PyPy has many different test directories
 and you can use shell completion to point at directories or files::
@@ -325,6 +326,24 @@
 .. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
 .. _`build requirements`: build.html#install-build-time-dependencies
 
+Testing After Translation
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While the usual invocation of `pytest` translates a piece of RPython code and
+runs it, we have a test extension to run tests without translation, directly
+on the host python. This is very convenient for modules such as `cpyext`, to
+compare and contrast test results between CPython and PyPy. Untranslated tests
+are invoked by using the `-A` or `--runappdirect` option to `pytest`::
+
+    python2 pytest.py -A pypy/module/cpyext/test
+
+where `python2` can be either `python2` or `pypy2`. On the `py3` branch, the
+collection phase must be run with `python2` so untranslated tests are run
+with::
+
+    cpython2 pytest.py -A pypy/module/cpyext/test --python=path/to/pypy3
+
+
 Tooling & Utilities
 ^^^^^^^^^^^^^^^^^^^
 
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -41,7 +41,7 @@
 ----------
 
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
   cpython_differences
   extending
@@ -56,7 +56,7 @@
 -----------
 
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
   contributing
   architecture
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -529,7 +529,8 @@
 
     def visit_kwonly(self, typ):
         raise FastFuncNotSupported
-
+        
+    @staticmethod
     def make_fastfunc(unwrap_spec, func):
         unwrap_info = UnwrapSpec_FastFunc_Unwrap()
         unwrap_info.apply_over(unwrap_spec)
@@ -560,7 +561,6 @@
             exec compile2(source) in unwrap_info.miniglobals, d
             fastfunc = d['fastfunc_%s_%d' % (func.__name__.replace('-', '_'), narg)]
         return narg, fastfunc
-    make_fastfunc = staticmethod(make_fastfunc)
 
 
 def int_unwrapping_space_method(typ):
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -27,10 +27,9 @@
 @unwrap_spec(code=int)
 def chr(space, code):
     "Return a Unicode string of one character with the given ordinal."
-    try:
-        c = UNICHR(code)
-    except ValueError:
+    if code < 0 or code > 0x10FFFF:
         raise oefmt(space.w_ValueError, "chr() arg out of range")
+    c = UNICHR(code)
     return space.newunicode(c)
 
 def len(space, w_obj):
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -76,6 +76,7 @@
 import pypy.module.cpyext.pytraceback
 import pypy.module.cpyext.methodobject
 import pypy.module.cpyext.dictproxyobject
+import pypy.module.cpyext.marshal
 import pypy.module.cpyext.genobject
 import pypy.module.cpyext.namespaceobject
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -61,7 +61,7 @@
 
 configure_eci = ExternalCompilationInfo(
         include_dirs=include_dirs,
-        includes=['Python.h', 'stdarg.h', 'structmember.h'],
+        includes=['Python.h', 'stdarg.h', 'structmember.h', 'marshal.h'],
         compile_extra=['-DPy_BUILD_CORE'])
 
 class CConfig:
@@ -119,6 +119,7 @@
 pypy_decl = 'pypy_decl.h'
 udir.join(pypy_decl).write("/* Will be filled later */\n")
 udir.join('pypy_structmember_decl.h').write("/* Will be filled later */\n")
+udir.join('pypy_marshal_decl.h').write("/* Will be filled later */\n")
 udir.join('pypy_macros.h').write("/* Will be filled later */\n")
 
 constant_names = """
@@ -1233,14 +1234,11 @@
         global_objects.append('PyTypeObject _PyExc_%s;' % name)
     global_code = '\n'.join(global_objects)
 
-    prologue = ("#include <Python.h>\n"
-                "#include <structmember.h>\n"
+    prologue = ("#include <Python.h>\n" +
+                "#include <structmember.h>\n" +
+                "#include <marshal.h>\n" +
+                ("#include <pypy_numpy.h>\n" if use_micronumpy else "") +
                 "#include <src/thread.c>\n")
-    if use_micronumpy:
-        prologue = ("#include <Python.h>\n"
-                    "#include <structmember.h>\n"
-                    "#include <pypy_numpy.h>\n"
-                    "#include <src/thread.c>\n")
     code = (prologue +
             struct_declaration_code +
             global_code +
diff --git a/pypy/module/cpyext/include/marshal.h b/pypy/module/cpyext/include/marshal.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/marshal.h
@@ -0,0 +1,13 @@
+#ifndef Py_MARSHAL_H
+#define Py_MARSHAL_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define Py_MARSHAL_VERSION 2
+#include "pypy_marshal_decl.h"
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_MARSHAL_H */
diff --git a/pypy/module/cpyext/marshal.py b/pypy/module/cpyext/marshal.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/marshal.py
@@ -0,0 +1,17 @@
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.module.cpyext.api import cpython_api, Py_ssize_t
+from pypy.module.cpyext.pyobject import PyObject
+
+
+_HEADER = 'pypy_marshal_decl.h'
+
+ at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject, header=_HEADER)
+def PyMarshal_ReadObjectFromString(space, p, size):
+    from pypy.module.marshal.interp_marshal import loads
+    s = rffi.charpsize2str(p, size)
+    return loads(space, space.newbytes(s))
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject, header=_HEADER)
+def PyMarshal_WriteObjectToString(space, w_x, version):
+    from pypy.module.marshal.interp_marshal import dumps
+    return dumps(space, w_x, space.newint(version))
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -172,6 +172,17 @@
                 return space.newtext(txtsig)
         return space.w_None
 
+    def fget_module(self, space):
+        if self.w_module is None:
+            return space.w_None
+        return self.w_module
+
+    def fset_module(self, space, w_module):
+        self.w_module = w_module
+
+    def fdel_module(self, space):
+        self.w_module = space.w_None
+
 class W_PyCMethodObject(W_PyCFunctionObject):
 
     def __init__(self, space, ml, w_type):
@@ -333,7 +344,9 @@
     __call__ = interp2app(W_PyCFunctionObject.descr_call),
     __doc__ = GetSetProperty(W_PyCFunctionObject.get_doc),
     __text_signature__ = GetSetProperty(W_PyCFunctionObject.get_txtsig),
-    __module__ = interp_attrproperty_w('w_module', cls=W_PyCFunctionObject),
+    __module__ = GetSetProperty(W_PyCFunctionObject.fget_module,
+                                W_PyCFunctionObject.fset_module,
+                                W_PyCFunctionObject.fdel_module),
     __name__ = interp_attrproperty('name', cls=W_PyCFunctionObject,
         wrapfn="newtext_or_none"),
     )
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -12,6 +12,7 @@
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.objspace.std.bytesobject import invoke_bytes_method
 from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.executioncontext import ExecutionContext
 import pypy.module.__builtin__.operation as operation
 
 
@@ -420,3 +421,27 @@
 def _PyPyGC_AddMemoryPressure(space, report):
     from rpython.rlib import rgc
     rgc.add_memory_pressure(report)
+
+
+ExecutionContext.cpyext_recursive_repr = None
+
+ at cpython_api([PyObject], rffi.INT_real, error=-1)
+def Py_ReprEnter(space, w_obj):
+    ec = space.getexecutioncontext()
+    d = ec.cpyext_recursive_repr
+    if d is None:
+        d = ec.cpyext_recursive_repr = {}
+    if w_obj in d:
+        return 1
+    d[w_obj] = None
+    return 0
+
+ at cpython_api([PyObject], lltype.Void)
+def Py_ReprLeave(space, w_obj):
+    ec = space.getexecutioncontext()
+    d = ec.cpyext_recursive_repr
+    if d is not None:
+        try:
+            del d[w_obj]
+        except KeyError:
+            pass
diff --git a/pypy/module/cpyext/test/test_marshal.py b/pypy/module/cpyext/test/test_marshal.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_marshal.py
@@ -0,0 +1,33 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
+class AppTestMarshal(AppTestCpythonExtensionBase):
+    def test_PyMarshal_ReadObjectFromString(self):
+        module = self.import_extension('foo', [
+            ("mloads", "METH_O",
+             """
+                 char *input = PyString_AsString(args);
+                 Py_ssize_t length = PyString_Size(args);
+                 return PyMarshal_ReadObjectFromString(input, length);
+             """)],
+            prologue='#include <marshal.h>')
+        import marshal
+        assert module.mloads(marshal.dumps(42.5)) == 42.5
+        x = [None, True, (4, 5), b"adkj", u"\u1234"]
+        assert module.mloads(marshal.dumps(x)) == x
+
+    def test_PyMarshal_WriteObjectToString(self):
+        module = self.import_extension('foo', [
+            ("mdumps", "METH_VARARGS",
+             """
+                 PyObject *obj;
+                 int version;
+                 if (!PyArg_ParseTuple(args, "Oi", &obj, &version))
+                     return NULL;
+                 return PyMarshal_WriteObjectToString(obj, version);
+             """)],
+            prologue='#include <marshal.h>')
+        import marshal
+        for x in [42, b"foo", u"\u2345", (4, None, False)]:
+            for version in [0, 1, 2]:
+                assert module.mdumps(x, version) == marshal.dumps(x, version)
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py
--- a/pypy/module/cpyext/test/test_methodobject.py
+++ b/pypy/module/cpyext/test/test_methodobject.py
@@ -172,6 +172,20 @@
         assert mod.check(A.meth) == 0
         assert mod.check(A.stat) == 0
 
+    def test_module_attribute(self):
+        mod = self.import_extension('MyModule', [
+            ('getarg_NO', 'METH_NOARGS',
+             '''
+                 Py_INCREF(Py_None);
+                 return Py_None;
+             '''
+             ),
+            ])
+        assert mod.getarg_NO() is None
+        assert mod.getarg_NO.__module__ == 'MyModule'
+        mod.getarg_NO.__module__ = 'foobar'
+        assert mod.getarg_NO.__module__ == 'foobar'
+
     def test_text_signature(self):
         mod = self.import_module('docstrings')
         assert mod.no_doc.__doc__ is None
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -411,6 +411,47 @@
         module.foo(35000)
         assert self.cur_memory_pressure() == 65536 + 80000 + 70000
 
+    def test_repr_enter_leave(self):
+        module = self.import_extension('foo', [
+            ("enter", "METH_O",
+            """
+                return PyInt_FromLong(Py_ReprEnter(args));
+            """),
+            ("leave", "METH_O",
+            """
+                Py_ReprLeave(args);
+                Py_INCREF(Py_None);
+                return Py_None;
+            """)])
+        obj1 = [42]
+        obj2 = [42]   # another list
+
+        n = module.enter(obj1)
+        assert n == 0
+        module.leave(obj1)
+
+        n = module.enter(obj1)
+        assert n == 0
+        n = module.enter(obj1)
+        assert n == 1
+        n = module.enter(obj1)
+        assert n == 1
+        module.leave(obj1)
+
+        n = module.enter(obj1)
+        assert n == 0
+        n = module.enter(obj2)
+        assert n == 0
+        n = module.enter(obj1)
+        assert n == 1
+        n = module.enter(obj2)
+        assert n == 1
+        module.leave(obj1)
+        n = module.enter(obj2)
+        assert n == 1
+        module.leave(obj2)
+
+
 class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase):
     """
     PyBuffer_FillInfo populates the fields of a Py_buffer from its arguments.
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -688,6 +688,16 @@
         assert isinstance(dictvalue(s), annmodel.SomeInteger)
         assert not dictvalue(s).nonneg
 
+    def test_dict_get(self):
+        def f1(i, j):
+            d = {i: ''}
+            return d.get(j)
+        a = self.RPythonAnnotator()
+        s = a.build_types(f1, [int, int])
+        assert isinstance(s, annmodel.SomeString)
+        assert s.can_be_None
+
+
     def test_exception_deduction(self):
         a = self.RPythonAnnotator()
         s = a.build_types(snippet.exception_deduction, [])
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -496,7 +496,7 @@
             return SomeTuple((s_key, s_Int))
         raise ValueError(variant)
 
-    def method_get(self, key, dfl):
+    def method_get(self, key, dfl=s_None):
         position = getbookkeeper().position_key
         self.dictdef.generalize_key(key)
         self.dictdef.generalize_value(dfl)
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -283,8 +283,12 @@
         return DictIteratorRepr(self, *variant)
 
     def rtype_method_get(self, hop):
-        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
-                                                 self.value_repr)
+        if hop.nb_args == 3:
+            v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                     self.value_repr)
+        else:
+            v_dict, v_key = hop.inputargs(self, self.key_repr)
+            v_default = hop.inputconst(self.value_repr, None)
         hop.exception_cannot_occur()
         v_res = hop.gendirectcall(ll_dict_get, v_dict, v_key, v_default)
         return self.recast_value(hop.llops, v_res)
diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py
--- a/rpython/rtyper/test/test_rdict.py
+++ b/rpython/rtyper/test/test_rdict.py
@@ -208,6 +208,16 @@
         res = self.interpret(func, ())
         assert res == 421
 
+    def test_dict_get_no_second_arg(self):
+        def func():
+            dic = self.newdict()
+            x1 = dic.get('hi', 'a')
+            x2 = dic.get('blah')
+            return (x1 == 'a') * 10 + (x2 is None)
+            return x1 * 10 + x2
+        res = self.interpret(func, ())
+        assert res == 11
+
     def test_dict_get_empty(self):
         def func():
             # this time without writing to the dict

From pypy.commits at gmail.com  Sun May 13 04:46:57 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Sun, 13 May 2018 01:46:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Addressing Code Review Feedback
Message-ID: <5af7fb81.1c69fb81.f4c2f.e91f@mx.google.com>

Author: Yusuke Tsutsumi <yusuket at tsutsumi.io>
Branch: py3.5
Changeset: r94540:b3a56b6a0940
Date: 2018-04-27 22:30 -0700
http://bitbucket.org/pypy/pypy/changeset/b3a56b6a0940/

Log:	Addressing Code Review Feedback

	Refactoring to move the init_posix code to a central location,
	inside of sysconfigdata.

	Including an improved clause for detecting if gcc is installed on
	the machine, and using a correct build argument.

diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -63,39 +63,9 @@
 
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
-    so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
-
-    g = {}
-    g['CC'] = "cc -pthread"
-    g['CXX'] = "c++ -pthread"
-    g['OPT'] = "-DNDEBUG -O2"
-    g['CFLAGS'] = "-DNDEBUG -O2"
-    g['CCSHARED'] = "-fPIC"
-    g['LDSHARED'] = "cc -pthread -shared"
-    g['EXT_SUFFIX'] = so_ext
-    g['SHLIB_SUFFIX'] = ".so"
-    g['SO'] = so_ext  # deprecated in Python 3, for backward compatibility
-    g['AR'] = "ar"
-    g['ARFLAGS'] = "rc"
-    g['EXE'] = ""
-    g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['VERSION'] = get_python_version()
-
-    if sys.platform[:6] == "darwin":
-        import platform
-        if platform.machine() == 'i386':
-            if platform.architecture()[0] == '32bit':
-                arch = 'i386'
-            else:
-                arch = 'x86_64'
-        else:
-            # just a guess
-            arch = platform.machine()
-        g['LDSHARED'] += ' -undefined dynamic_lookup'
-        g['CC'] += ' -arch %s' % (arch,)
-
+    from _sysconfigdata import build_time_vars
     global _config_vars
-    _config_vars = g
+    _config_vars.update(build_time_vars)
 
 
 def _init_nt():
@@ -221,4 +191,3 @@
 
 from .sysconfig_cpython import (
     parse_makefile, _variable_rx, expand_makefile_vars)
-
diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -1,5 +1,6 @@
 import _imp
 import os
+import sys
 from distutils.spawn import find_executable
 
 so_ext = _imp.extension_suffixes()[0]
@@ -8,13 +9,40 @@
     "EXT_SUFFIX": so_ext,
     "SHLIB_SUFFIX": so_ext,
     "SOABI": '-'.join(so_ext.split('.')[1].split('-')[:2]),
-    "SO": so_ext  # deprecated in Python 3, for backward compatibility
+    "SO": so_ext,  # deprecated in Python 3, for backward compatibility
+    'CC': "cc -pthread",
+    'CXX': "c++ -pthread",
+    'OPT': "-DNDEBUG -O2",
+    'CFLAGS': "-DNDEBUG -O2",
+    'CCSHARED': "-fPIC",
+    'LDSHARED': "cc -pthread -shared",
+    'EXT_SUFFIX': so_ext,
+    'SHLIB_SUFFIX': ".so",
+    'AR': "ar",
+    'ARFLAGS': "rc",
+    'EXE': "",
+    'LIBDIR': os.path.join(sys.prefix, 'lib'),
+    'VERSION': sys.version[:3]
 }
 
-cc_compiler_path = os.path.realpath(find_executable("cc"))
-cc_compiler = os.path.basename(cc_compiler_path)
-build_time_vars["CC"] = cc_compiler
-if "gcc" in cc_compiler or "g++" in cc_compiler:
-    # If we used the gnu compiler, we can safely assume we are using the gnu
-    # linker
-    build_time_vars["GNULD"] = "yes"
+if sys.platform[:6] == "darwin":
+    import platform
+    if platform.machine() == 'i386':
+        if platform.architecture()[0] == '32bit':
+            arch = 'i386'
+        else:
+            arch = 'x86_64'
+    else:
+        # just a guess
+        arch = platform.machine()
+    build_time_vars['LDSHARED'] += ' -undefined dynamic_lookup'
+    build_time_vars['CC'] += ' -arch %s' % (arch,)
+
+if find_executable("gcc"):
+    build_time_vars.update({
+        "CC": "gcc -pthread",
+        "GNULD": "yes",
+        "LDSHARED": "gcc -pthread -shared",
+    })
+    if find_executable("g++"):
+        build_time_vars["CXX"] = "g++ -pthread"

From pypy.commits at gmail.com  Sun May 13 04:46:51 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 May 2018 01:46:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Merged in toumorokoshi/pypy/py3.5 (pull
 request #608)
Message-ID: <5af7fb7b.1c69fb81.40609.4313@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: py3.5
Changeset: r94544:7f62e74d2342
Date: 2018-05-13 08:46 +0000
http://bitbucket.org/pypy/pypy/changeset/7f62e74d2342/

Log:	Merged in toumorokoshi/pypy/py3.5 (pull request #608)

	Adding detection of gcc to sysconfig_pypy (fixes #2809)

diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -63,39 +63,9 @@
 
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
-    so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
-
-    g = {}
-    g['CC'] = "cc -pthread"
-    g['CXX'] = "c++ -pthread"
-    g['OPT'] = "-DNDEBUG -O2"
-    g['CFLAGS'] = "-DNDEBUG -O2"
-    g['CCSHARED'] = "-fPIC"
-    g['LDSHARED'] = "cc -pthread -shared"
-    g['EXT_SUFFIX'] = so_ext
-    g['SHLIB_SUFFIX'] = ".so"
-    g['SO'] = so_ext  # deprecated in Python 3, for backward compatibility
-    g['AR'] = "ar"
-    g['ARFLAGS'] = "rc"
-    g['EXE'] = ""
-    g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['VERSION'] = get_python_version()
-
-    if sys.platform[:6] == "darwin":
-        import platform
-        if platform.machine() == 'i386':
-            if platform.architecture()[0] == '32bit':
-                arch = 'i386'
-            else:
-                arch = 'x86_64'
-        else:
-            # just a guess
-            arch = platform.machine()
-        g['LDSHARED'] += ' -undefined dynamic_lookup'
-        g['CC'] += ' -arch %s' % (arch,)
-
+    from _sysconfigdata import build_time_vars
     global _config_vars
-    _config_vars = g
+    _config_vars.update(build_time_vars)
 
 
 def _init_nt():
@@ -221,4 +191,3 @@
 
 from .sysconfig_cpython import (
     parse_makefile, _variable_rx, expand_makefile_vars)
-
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -4,6 +4,7 @@
 import subprocess
 import shutil
 from copy import copy
+from distutils.spawn import find_executable
 
 from test.support import (run_unittest, TESTFN, unlink, check_warnings,
                           captured_stdout, impl_detail, import_module,
@@ -297,6 +298,30 @@
 
         self.assertIn(ldflags, ldshared)
 
+    @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows")
+    def test_cc_values(self):
+        """ CC and CXX should be set for pypy """
+        for var in ["CC", "CXX"]:
+            assert sysconfig.get_config_var(var) is not None
+
+    @unittest.skipIf(not find_executable("gcc"),
+        "Does not apply to machines without gcc installed"
+    )
+    def test_gcc_values(self):
+        """ if gcc is installed on the box, gcc values should be set. """
+        assert "gcc" in sysconfig.get_config_var("CC")
+        assert sysconfig.get_config_var("GNULD") == "yes"
+        assert "gcc" in sysconfig.get_config_var("LDSHARED")
+
+
+    @unittest.skipIf(not find_executable("g++"),
+        "Does not apply to machines without g++ installed"
+    )
+    def test_gplusplus_values(self):
+        """ if g++ is installed on the box, g++ values should be set. """
+        assert "g++" in sysconfig.get_config_var("CXX")
+
+
     @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX")
     def test_platform_in_subprocess(self):
         my_platform = sysconfig.get_platform()
diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -1,10 +1,49 @@
 import _imp
+import os
+import sys
+from distutils.spawn import find_executable
 
 so_ext = _imp.extension_suffixes()[0]
 
+
 build_time_vars = {
     "EXT_SUFFIX": so_ext,
     "SHLIB_SUFFIX": so_ext,
     "SOABI": '-'.join(so_ext.split('.')[1].split('-')[:2]),
-    "SO": so_ext  # deprecated in Python 3, for backward compatibility
+    "SO": so_ext,  # deprecated in Python 3, for backward compatibility
+    'CC': "cc -pthread",
+    'CXX': "c++ -pthread",
+    'OPT': "-DNDEBUG -O2",
+    'CFLAGS': "-DNDEBUG -O2",
+    'CCSHARED': "-fPIC",
+    'LDSHARED': "cc -pthread -shared",
+    'EXT_SUFFIX': so_ext,
+    'SHLIB_SUFFIX': ".so",
+    'AR': "ar",
+    'ARFLAGS': "rc",
+    'EXE': "",
+    'LIBDIR': os.path.join(sys.prefix, 'lib'),
+    'VERSION': sys.version[:3]
 }
+
+if sys.platform[:6] == "darwin":
+    import platform
+    if platform.machine() == 'i386':
+        if platform.architecture()[0] == '32bit':
+            arch = 'i386'
+        else:
+            arch = 'x86_64'
+    else:
+        # just a guess
+        arch = platform.machine()
+    build_time_vars['LDSHARED'] += ' -undefined dynamic_lookup'
+    build_time_vars['CC'] += ' -arch %s' % (arch,)
+
+if find_executable("gcc"):
+    build_time_vars.update({
+        "CC": "gcc -pthread",
+        "GNULD": "yes",
+        "LDSHARED": "gcc -pthread -shared",
+    })
+    if find_executable("g++"):
+        build_time_vars["CXX"] = "g++ -pthread"

From pypy.commits at gmail.com  Sun May 13 04:46:55 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Sun, 13 May 2018 01:46:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Adding detection of cc and gcc to build
 variables
Message-ID: <5af7fb7f.1c69fb81.9d848.6cb2@mx.google.com>

Author: Yusuke Tsutsumi <yusuket at tsutsumi.io>
Branch: py3.5
Changeset: r94539:4c61881c4c06
Date: 2018-04-27 08:32 -0700
http://bitbucket.org/pypy/pypy/changeset/4c61881c4c06/

Log:	Adding detection of cc and gcc to build variables

diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -1,4 +1,6 @@
 import _imp
+import os
+from distutils.spawn import find_executable
 
 so_ext = _imp.extension_suffixes()[0]
 
@@ -8,3 +10,11 @@
     "SOABI": '-'.join(so_ext.split('.')[1].split('-')[:2]),
     "SO": so_ext  # deprecated in Python 3, for backward compatibility
 }
+
+cc_compiler_path = os.path.realpath(find_executable("cc"))
+cc_compiler = os.path.basename(cc_compiler_path)
+build_time_vars["CC"] = cc_compiler
+if "gcc" in cc_compiler or "g++" in cc_compiler:
+    # If we used the gnu compiler, we can safely assume we are using the gnu
+    # linker
+    build_time_vars["GNULD"] = "yes"

From pypy.commits at gmail.com  Sun May 13 04:47:01 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Sun, 13 May 2018 01:47:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: minor change to recover branch.
Message-ID: <5af7fb85.1c69fb81.cf708.774d@mx.google.com>

Author: Yusuke Tsutsumi <yusuket at tsutsumi.io>
Branch: py3.5
Changeset: r94542:c1a718b1710d
Date: 2018-04-27 23:17 -0700
http://bitbucket.org/pypy/pypy/changeset/c1a718b1710d/

Log:	minor change to recover branch.

diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -5,6 +5,7 @@
 
 so_ext = _imp.extension_suffixes()[0]
 
+
 build_time_vars = {
     "EXT_SUFFIX": so_ext,
     "SHLIB_SUFFIX": so_ext,

From pypy.commits at gmail.com  Sun May 13 04:47:03 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Sun, 13 May 2018 01:47:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Adding unit tests
Message-ID: <5af7fb87.1c69fb81.798de.2dfb@mx.google.com>

Author: Yusuke Tsutsumi <yusuket at tsutsumi.io>
Branch: py3.5
Changeset: r94543:019e1a27a95c
Date: 2018-05-01 22:53 -0700
http://bitbucket.org/pypy/pypy/changeset/019e1a27a95c/

Log:	Adding unit tests

diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -4,6 +4,7 @@
 import subprocess
 import shutil
 from copy import copy
+from distutils.spawn import find_executable
 
 from test.support import (run_unittest, TESTFN, unlink, check_warnings,
                           captured_stdout, impl_detail, import_module,
@@ -297,6 +298,30 @@
 
         self.assertIn(ldflags, ldshared)
 
+    @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows")
+    def test_cc_values(self):
+        """ CC and CXX should be set for pypy """
+        for var in ["CC", "CXX"]:
+            assert sysconfig.get_config_var(var) is not None
+
+    @unittest.skipIf(not find_executable("gcc"),
+        "Does not apply to machines without gcc installed"
+    )
+    def test_gcc_values(self):
+        """ if gcc is installed on the box, gcc values should be set. """
+        assert "gcc" in sysconfig.get_config_var("CC")
+        assert sysconfig.get_config_var("GNULD") == "yes"
+        assert "gcc" in sysconfig.get_config_var("LDSHARED")
+
+
+    @unittest.skipIf(not find_executable("g++"),
+        "Does not apply to machines without g++ installed"
+    )
+    def test_gplusplus_values(self):
+        """ if g++ is installed on the box, g++ values should be set. """
+        assert "g++" in sysconfig.get_config_var("CXX")
+
+
     @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX")
     def test_platform_in_subprocess(self):
         my_platform = sysconfig.get_platform()

From pypy.commits at gmail.com  Sun May 13 04:46:59 2018
From: pypy.commits at gmail.com (toumorokoshi)
Date: Sun, 13 May 2018 01:46:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Close branch py3.5.
Message-ID: <5af7fb83.1c69fb81.4d072.56ec@mx.google.com>

Author: Yusuke Tsutsumi <tsutsumi.yusuke at gmail.com>
Branch: py3.5
Changeset: r94541:29a5d3ceb791
Date: 2018-04-28 06:14 +0000
http://bitbucket.org/pypy/pypy/changeset/29a5d3ceb791/

Log:	Close branch py3.5.


From pypy.commits at gmail.com  Sun May 13 04:48:51 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 May 2018 01:48:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove these two entries because they
 appear also later.
Message-ID: <5af7fbf3.1c69fb81.3e9a0.1985@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94545:0811e990ecee
Date: 2018-05-13 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/0811e990ecee/

Log:	Remove these two entries because they appear also later. In the case
	of SHLIB_SUFFIX, it appears with a different value. Looking at the
	history I think we want to keep the second one.

diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -7,8 +7,6 @@
 
 
 build_time_vars = {
-    "EXT_SUFFIX": so_ext,
-    "SHLIB_SUFFIX": so_ext,
     "SOABI": '-'.join(so_ext.split('.')[1].split('-')[:2]),
     "SO": so_ext,  # deprecated in Python 3, for backward compatibility
     'CC': "cc -pthread",

From pypy.commits at gmail.com  Sun May 13 05:37:01 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 May 2018 02:37:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: Remove an unused line
Message-ID: <5af8073d.1c69fb81.7dd9c.ef63@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94546:b929ba72ac2b
Date: 2018-05-13 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/b929ba72ac2b/

Log:	Remove an unused line

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -430,7 +430,6 @@
             for const1 in switchdict.const_keys_in_order:
                 box = self.execute(rop.INT_EQ, valuebox, const1)
                 assert box.getint() == 0
-                target = switchdict.dict[const1.getint()]
                 self.metainterp.generate_guard(rop.GUARD_FALSE, box,
                                                resumepc=orgpc)
         else:

From pypy.commits at gmail.com  Sun May 13 17:17:42 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 May 2018 14:17:42 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: in-progress (not
 working so far): trying to get
Message-ID: <5af8ab76.1c69fb81.40609.d677@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94548:c8c3a21ba942
Date: 2018-05-13 23:16 +0200
http://bitbucket.org/pypy/pypy/changeset/c8c3a21ba942/

Log:	in-progress (not working so far): trying to get
	get_raw_address_of_string() to never raise ValueError

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -809,6 +809,7 @@
             lltype.free(cp, flavor='raw', track_allocation=True)
         else:
             lltype.free(cp, flavor='raw', track_allocation=False)
+    free_charp._annenforceargs_ = [None, bool]
 
     # str -> already-existing char[maxsize]
     def str2chararray(s, array, maxsize):
@@ -1330,14 +1331,6 @@
         )
 
 
-class RawBytes(object):
-    # literal copy of _cffi_backend/func.py
-    def __init__(self, string):
-        self.ptr = str2charp(string, track_allocation=False)
-    def __del__(self):
-        if free_charp is not None:    # CPython shutdown
-            free_charp(self.ptr, track_allocation=False)
-
 # NOTE: This is not a weak key dictionary, thus keeping a lot of stuff alive.
 TEST_RAW_ADDR_KEEP_ALIVE = {}
 
@@ -1346,8 +1339,6 @@
     """Returns a 'char *' that is valid as long as the rpython string object is alive.
     Two calls to to this function, given the same string parameter,
     are guaranteed to return the same pointer.
-
-    NOTE: may raise ValueError on some GCs, but not the default one.
     """
     assert isinstance(string, str)
     from rpython.rtyper.annlowlevel import llstr
@@ -1357,11 +1348,11 @@
 
     if we_are_translated():
         if rgc.must_split_gc_address_space():
-            raise ValueError("cannot return a pointer in the gcstring")
+            return _get_raw_address_buf_from_string(string)
         if rgc.can_move(string):
             string = rgc.move_out_of_nursery(string)
             if rgc.can_move(string):
-                raise ValueError("cannot make string immovable")
+                return _get_raw_address_buf_from_string(string)
 
         # string cannot move now! return the address
         lldata = llstr(string)
@@ -1374,6 +1365,48 @@
     else:
         global TEST_RAW_ADDR_KEEP_ALIVE
         if string in TEST_RAW_ADDR_KEEP_ALIVE:
-            return TEST_RAW_ADDR_KEEP_ALIVE[string].ptr
-        TEST_RAW_ADDR_KEEP_ALIVE[string] = rb = RawBytes(string)
-        return rb.ptr
+            return TEST_RAW_ADDR_KEEP_ALIVE[string]
+        result = str2charp(string, track_allocation=False)
+        TEST_RAW_ADDR_KEEP_ALIVE[string] = result
+        return result
+
+class _StrFinalizerQueue(rgc.FinalizerQueue):
+    Class = None              # to use GCREFs directly
+    print_debugging = False   # set to True from test_rffi
+    def finalizer_trigger(self):
+        from rpython.rtyper.annlowlevel import hlstr
+        from rpython.rtyper.lltypesystem import rstr
+        from rpython.rlib.debug import debug_print
+        from rpython.rlib import objectmodel
+        debug_print("HI THERE")
+        while True:
+            gcptr = self.next_dead()
+            if not gcptr:
+                break
+            ll_string = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), gcptr)
+            string = hlstr(ll_string)
+            key = objectmodel.compute_unique_id(string)
+            ptr = self.raw_copies.get(key, lltype.nullptr(CCHARP.TO))
+            debug_print(ptr)
+            if ptr:
+                if self.print_debugging:
+                    debug_print("freeing [", ptr, "]")
+                free_charp(ptr, track_allocation=False)
+_fq_addr_from_string = _StrFinalizerQueue()
+_fq_addr_from_string.raw_copies = {}    # {GCREF: CCHARP}
+
+def _get_raw_address_buf_from_string(string):
+    # Slowish but ok because it's not supposed to be used from a
+    # regular PyPy.  It's only used with non-standard GCs like RevDB
+    from rpython.rtyper.annlowlevel import llstr
+    from rpython.rlib import objectmodel
+    key = objectmodel.compute_unique_id(string)
+    try:
+        ptr = _fq_addr_from_string.raw_copies[key]
+    except KeyError:
+        ptr = str2charp(string, track_allocation=False)
+        _fq_addr_from_string.raw_copies[key] = ptr
+        ll_string = llstr(string)
+        gcptr = lltype.cast_opaque_ptr(llmemory.GCREF, ll_string)
+        _fq_addr_from_string.register_finalizer(gcptr)
+    return ptr
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -814,6 +814,35 @@
     def test_generate_return_char_tests(self):
         py.test.skip("GenC does not handle char return values correctly")
 
+    def test__get_raw_address_buf_from_string(self):
+        from rpython.rlib import rgc
+        from rpython.rtyper.lltypesystem import rffi
+
+        def check_content(strings, rawptrs):
+            for i in range(len(strings)):
+                p = rawptrs[i]
+                expected = strings[i] + '\x00'
+                for j in range(len(expected)):
+                    assert p[j] == expected[j]
+
+        def f(n):
+            strings = [str(i) for i in range(n)]
+            rawptrs = [rffi._get_raw_address_buf_from_string(s)
+                       for s in strings]
+            check_content(strings, rawptrs)
+            rgc.collect(); rgc.collect(); rgc.collect()
+            check_content(strings, rawptrs)
+            del strings
+            rgc.collect(); rgc.collect(); rgc.collect()
+            return 42
+
+        rffi._StrFinalizerQueue.print_debugging = True
+        try:
+            xf = self.compile(f, [int], gcpolicy="incminimark")
+            assert xf(10000) == 42
+        finally:
+            rffi._StrFinalizerQueue.print_debugging = False
+
 def test_enforced_args():
     from rpython.annotator.model import s_None
     from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator

From pypy.commits at gmail.com  Sun May 13 17:17:39 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 May 2018 14:17:39 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: hg merge
 release-pypy2.7-6.x
Message-ID: <5af8ab73.1c69fb81.5e49f.83cf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94547:216be18e2a9e
Date: 2018-05-13 22:15 +0200
http://bitbucket.org/pypy/pypy/changeset/216be18e2a9e/

Log:	hg merge release-pypy2.7-6.x

diff too long, truncating to 2000 out of 12937 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -6,36 +6,36 @@
 Except when otherwise stated (look for LICENSE files in directories or
 information at the beginning of each file) all software and documentation in
 the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy',
-'py', and '_pytest' directories is licensed as follows: 
+'py', and '_pytest' directories is licensed as follows:
 
     The MIT License
 
-    Permission is hereby granted, free of charge, to any person 
-    obtaining a copy of this software and associated documentation 
-    files (the "Software"), to deal in the Software without 
-    restriction, including without limitation the rights to use, 
-    copy, modify, merge, publish, distribute, sublicense, and/or 
-    sell copies of the Software, and to permit persons to whom the 
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use,
+    copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the
     Software is furnished to do so, subject to the following conditions:
 
-    The above copyright notice and this permission notice shall be included 
+    The above copyright notice and this permission notice shall be included
     in all copies or substantial portions of the Software.
 
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
     DEALINGS IN THE SOFTWARE.
 
 
 PyPy Copyright holders 2003-2018
------------------------------------ 
+--------------------------------
 
 Except when otherwise stated (look for LICENSE files or information at
 the beginning of each file) the files in the 'pypy' directory are each
-copyrighted by one or more of the following people and organizations:    
+copyrighted by one or more of the following people and organizations:
 
   Armin Rigo
   Maciej Fijalkowski
@@ -89,13 +89,13 @@
   Niko Matsakis
   Alexander Hesse
   Ludovic Aubry
+  stian
   Jacob Hallen
   Jason Creighton
   Mark Young
   Alex Martelli
   Spenser Bauman
   Michal Bendowski
-  stian
   Jan de Mooij
   Tyler Wade
   Vincent Legoll
@@ -123,10 +123,10 @@
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
+  Jeremy Thurgood
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
-  Jeremy Thurgood
   Mark Pearse
   Simon Cross
   Tobias Pape
@@ -145,18 +145,19 @@
   Adrian Kuhn
   tav
   Georg Brandl
+  Joannah Nanjekye
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
-  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
+  Dodan Mihai
   Tim Felgentreff
   Eugene Oden
-  Dodan Mihai
+  Colin Valliant
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -225,12 +226,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Floris Bruynooghe
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Arianna Avanzini
   Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
@@ -244,15 +247,18 @@
   Lukas Vacek
   Omer Katz
   Jacek Generowicz
+  Tomasz Dziopa
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
   Gabriel
+  Thomas Hisch
   Mark Williams
   Kunal Grover
   Nathan Taylor
+  Barry Hart
   Travis Francis Athougies
   Yasir Suhail
   Sergey Kishchenko
@@ -260,6 +266,7 @@
   Lutz Paelike
   Ian Foote
   Philipp Rustemeuer
+  Logan Chien
   Catalin Gabriel Manciu
   Jacob Oscarson
   Ryan Gonzalez
@@ -295,19 +302,20 @@
   Akira Li
   Gustavo Niemeyer
   Rafa&#322; Ga&#322;czy&#324;ski
-  Logan Chien
   Lucas Stadler
   roberto at goyle
   Matt Bogosian
   Yury V. Zaytsev
   florinpapa
   Anders Sigfridsson
+  Matt Jackson
   Nikolay Zinov
   rafalgalczynski at gmail.com
   Joshua Gilbert
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Miro Hron&#269;ok
   Anthony Sottile
   Nate Bragg
   Ben Darnell
@@ -315,7 +323,6 @@
   Godefroid Chappelle
   Julian Berman
   Michael Hudson-Doyle
-  Floris Bruynooghe
   Stephan Busemann
   Dan Colish
   timo
@@ -357,6 +364,7 @@
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Pauli Virtanen
   Petre Vijiac
   Berker Peksag
   Christian Muirhead
@@ -381,6 +389,7 @@
   Graham Markall
   Dan Loewenherz
   werat
+  Andrew Stepanov
   Niclas Olofsson
   Chris Pressey
   Tobias Diaz
@@ -395,14 +404,14 @@
   m at funkyhat.org
   Stefan Marr
 
-  Heinrich-Heine University, Germany 
+  Heinrich-Heine University, Germany
   Open End AB (formerly AB Strakt), Sweden
-  merlinux GmbH, Germany 
-  tismerysoft GmbH, Germany 
-  Logilab Paris, France 
-  DFKI GmbH, Germany 
+  merlinux GmbH, Germany
+  tismerysoft GmbH, Germany
+  Logilab Paris, France
+  DFKI GmbH, Germany
   Impara, Germany
-  Change Maker, Sweden 
+  Change Maker, Sweden
   University of California Berkeley, USA
   Google Inc.
   King's College London
@@ -410,14 +419,14 @@
 The PyPy Logo as used by http://speed.pypy.org and others was created
 by Samuel Reis and is distributed on terms of Creative Commons Share Alike
 License.
- 
-License for 'lib-python/2.7'
-============================
+
+License for 'lib-python/2.7, lib-python/3'
+==========================================
 
 Except when otherwise stated (look for LICENSE files or copyright/license
-information at the beginning of each file) the files in the 'lib-python/2.7'
+information at the beginning of each file) the files in the 'lib-python'
 directory are all copyrighted by the Python Software Foundation and licensed
-under the terms that you can find here: https://docs.python.org/2/license.html
+under the terms that you can find here: https://docs.python.org/3/license.html
 
 License for 'pypy/module/unicodedata/'
 ======================================
@@ -441,9 +450,9 @@
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-  
+
      http://www.apache.org/licenses/LICENSE-2.0
-  
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -4,42 +4,40 @@
 
 Welcome to PyPy!
 
-PyPy is both an implementation of the Python programming language, and
-an extensive compiler framework for dynamic language implementations.
-You can build self-contained Python implementations which execute
-independently from CPython.
+PyPy is an interperter that implements the Python programming language, based
+on the RPython compiler framework for dynamic language implementations.
 
-The home page is:
+The home page for the interpreter is:
 
     http://pypy.org/
 
-If you want to help developing PyPy, this document might help you:
+If you want to help developing PyPy, this documentation might help you:
 
     http://doc.pypy.org/
 
-It will also point you to the rest of the documentation which is generated
-from files in the pypy/doc directory within the source repositories. Enjoy
-and send us feedback!
+More documentation about the RPython framework can be found here
 
-    the pypy-dev team <pypy-dev at python.org>
+    http://rpython.readthedocs.io
 
+The source for the documentation is in the pypy/doc directory 
+
+Using PyPy instead of CPython
+=============================
+
+Please read the information at http://pypy.org to find the correct way to
+download and use PyPy as an alternative to CPython. 
 
 Building
 ========
 
-First switch to or download the correct branch.  The basic choices are
-``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X
-branch (e.g. ``py3.5``).
+Building PyPy is not the recommended way to obtain the PyPy alternative python
+interpreter. It is time-consuming and requires significant computing resources.
+More information can be found here
 
-Build with:
+    http://doc.pypy.org/en/latest/build.html
 
-.. code-block:: console
+Enjoy and send us feedback!
 
-    $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+    the pypy-dev team <pypy-dev at python.org>
 
-This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy
-directory.  We suggest to use virtualenv with the resulting
-pypy-c/pypy3-c as the interpreter; you can find more details about
-various installation schemes here:
 
-    http://doc.pypy.org/en/latest/install.html
diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py
--- a/lib-python/2.7/re.py
+++ b/lib-python/2.7/re.py
@@ -225,7 +225,7 @@
 
 _pattern_type = type(sre_compile.compile("", 0))
 
-_MAXCACHE = 100
+_MAXCACHE = 1000
 
 def _compile(*key):
     # internal: compile pattern
diff --git a/lib-python/2.7/test/test_eof.py b/lib-python/2.7/test/test_eof.py
--- a/lib-python/2.7/test/test_eof.py
+++ b/lib-python/2.7/test/test_eof.py
@@ -5,7 +5,7 @@
 
 class EOFTestCase(unittest.TestCase):
     def test_EOFC(self):
-        expect = "EOL while scanning string literal (<string>, line 1)"
+        expect = "end of line (EOL) while scanning string literal (<string>, line 1)"
         try:
             eval("""'this is a test\
             """)
@@ -15,7 +15,7 @@
             raise test_support.TestFailed
 
     def test_EOFS(self):
-        expect = ("EOF while scanning triple-quoted string literal "
+        expect = ("end of file (EOF) while scanning triple-quoted string literal "
                   "(<string>, line 1)")
         try:
             eval("""'''this is a test""")
diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py
--- a/lib-python/2.7/test/test_generators.py
+++ b/lib-python/2.7/test/test_generators.py
@@ -398,7 +398,10 @@
 0
 >>> type(i.gi_frame)
 <type 'frame'>
->>> i.gi_running = 42
+
+PyPy prints "readonly attribute 'gi_running'" so ignore the exception detail
+
+>>> i.gi_running = 42 # doctest: +IGNORE_EXCEPTION_DETAIL
 Traceback (most recent call last):
   ...
 TypeError: readonly attribute
diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py
--- a/lib-python/2.7/test/test_genexps.py
+++ b/lib-python/2.7/test/test_genexps.py
@@ -87,7 +87,7 @@
     >>> dict(a = i for i in xrange(10))
     Traceback (most recent call last):
        ...
-    SyntaxError: invalid syntax
+    SyntaxError: invalid syntax (expected ')')
 
 Verify that parenthesis are required when used as a keyword argument value
 
diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py
--- a/lib-python/2.7/test/test_traceback.py
+++ b/lib-python/2.7/test/test_traceback.py
@@ -123,10 +123,7 @@
         self.assertEqual(len(err), 4)
         self.assertEqual(err[1].strip(), "print(2)")
         self.assertIn("^", err[2])
-        if check_impl_detail():
-            self.assertEqual(err[1].find("p"), err[2].find("^"))
-        if check_impl_detail(pypy=True):
-            self.assertEqual(err[1].find("2)") + 1, err[2].find("^"))
+        self.assertEqual(err[1].find("p"), err[2].find("^"))
 
     def test_base_exception(self):
         # Test that exceptions derived from BaseException are formatted right
diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py
--- a/lib-python/2.7/threading.py
+++ b/lib-python/2.7/threading.py
@@ -351,6 +351,21 @@
                         # forward-compatibility reasons we do the same.
                         waiter.acquire()
                         gotit = True
+                    except AttributeError:
+                        # someone patched the 'waiter' class, probably.
+                        # Fall back to the standard CPython logic.
+                        # See the CPython lib for the comments about it...
+                        endtime = _time() + timeout
+                        delay = 0.0005 # 500 us -> initial delay of 1 ms
+                        while True:
+                            gotit = waiter.acquire(0)
+                            if gotit:
+                                break
+                            remaining = endtime - _time()
+                            if remaining <= 0:
+                                break
+                            delay = min(delay * 2, remaining, .05)
+                            _sleep(delay)
                 else:
                     gotit = waiter.acquire(False)
                 if not gotit:
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -56,13 +56,13 @@
   Niko Matsakis
   Alexander Hesse
   Ludovic Aubry
+  stian
   Jacob Hallen
   Jason Creighton
   Mark Young
   Alex Martelli
   Spenser Bauman
   Michal Bendowski
-  stian
   Jan de Mooij
   Tyler Wade
   Vincent Legoll
@@ -90,10 +90,10 @@
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
+  Jeremy Thurgood
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
-  Jeremy Thurgood
   Mark Pearse
   Simon Cross
   Tobias Pape
@@ -112,18 +112,19 @@
   Adrian Kuhn
   tav
   Georg Brandl
+  Joannah Nanjekye
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
-  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
+  Dodan Mihai
   Tim Felgentreff
   Eugene Oden
-  Dodan Mihai
+  Colin Valliant
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -192,12 +193,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Floris Bruynooghe
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Arianna Avanzini
   Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
@@ -211,6 +214,7 @@
   Lukas Vacek
   Omer Katz
   Jacek Generowicz
+  Tomasz Dziopa
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -221,6 +225,7 @@
   Mark Williams
   Kunal Grover
   Nathan Taylor
+  Barry Hart
   Travis Francis Athougies
   Yasir Suhail
   Sergey Kishchenko
@@ -228,6 +233,7 @@
   Lutz Paelike
   Ian Foote
   Philipp Rustemeuer
+  Logan Chien
   Catalin Gabriel Manciu
   Jacob Oscarson
   Ryan Gonzalez
@@ -263,19 +269,20 @@
   Akira Li
   Gustavo Niemeyer
   Rafa&#322; Ga&#322;czy&#324;ski
-  Logan Chien
   Lucas Stadler
   roberto at goyle
   Matt Bogosian
   Yury V. Zaytsev
   florinpapa
   Anders Sigfridsson
+  Matt Jackson
   Nikolay Zinov
   rafalgalczynski at gmail.com
   Joshua Gilbert
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Miro Hron&#269;ok
   Anthony Sottile
   Nate Bragg
   Ben Darnell
@@ -283,7 +290,6 @@
   Godefroid Chappelle
   Julian Berman
   Michael Hudson-Doyle
-  Floris Bruynooghe
   Stephan Busemann
   Dan Colish
   timo
@@ -325,6 +331,7 @@
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Pauli Virtanen
   Petre Vijiac
   Berker Peksag
   Christian Muirhead
@@ -349,6 +356,7 @@
   Graham Markall
   Dan Loewenherz
   werat
+  Andrew Stepanov
   Niclas Olofsson
   Chris Pressey
   Tobias Diaz
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -10,89 +10,6 @@
 PyPy.
 
 
-.. _extension-modules:
-
-Extension modules
------------------
-
-List of extension modules that we support:
-
-* Supported as built-in modules (in :source:`pypy/module/`):
-
-    __builtin__
-    :doc:`__pypy__ <__pypy__-module>`
-    _ast
-    _codecs
-    _collections
-    :doc:`_continuation <stackless>`
-    :doc:`_ffi <discussion/ctypes-implementation>`
-    _hashlib
-    _io
-    _locale
-    _lsprof
-    _md5
-    :doc:`_minimal_curses <config/objspace.usemodules._minimal_curses>`
-    _multiprocessing
-    _random
-    :doc:`_rawffi <discussion/ctypes-implementation>`
-    _sha
-    _socket
-    _sre
-    _ssl
-    _warnings
-    _weakref
-    _winreg
-    array
-    binascii
-    bz2
-    cStringIO
-    cmath
-    `cpyext`_
-    crypt
-    errno
-    exceptions
-    fcntl
-    gc
-    imp
-    itertools
-    marshal
-    math
-    mmap
-    operator
-    parser
-    posix
-    pyexpat
-    select
-    signal
-    struct
-    symbol
-    sys
-    termios
-    thread
-    time
-    token
-    unicodedata
-    zipimport
-    zlib
-
-  When translated on Windows, a few Unix-only modules are skipped,
-  and the following module is built instead:
-
-    _winreg
-
-* Supported by being rewritten in pure Python (possibly using ``cffi``):
-  see the :source:`lib_pypy/` directory.  Examples of modules that we
-  support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``...
-  Note that some modules are both in there and in the list above;
-  by default, the built-in module is used (but can be disabled
-  at translation time).
-
-The extension modules (i.e. modules written in C, in the standard CPython)
-that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy.
-(You may have a chance to use them anyway with `cpyext`_.)
-
-.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html
-
 
 Differences related to garbage collection strategies
 ----------------------------------------------------
@@ -559,7 +476,96 @@
   environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above**
   that value.
 
+* SyntaxError_ s try harder to give details about the cause of the failure, so
+  the error messages are not the same as in CPython
+
+
+.. _extension-modules:
+
+Extension modules
+-----------------
+
+List of extension modules that we support:
+
+* Supported as built-in modules (in :source:`pypy/module/`):
+
+    __builtin__
+    :doc:`__pypy__ <__pypy__-module>`
+    _ast
+    _codecs
+    _collections
+    :doc:`_continuation <stackless>`
+    :doc:`_ffi <discussion/ctypes-implementation>`
+    _hashlib
+    _io
+    _locale
+    _lsprof
+    _md5
+    :doc:`_minimal_curses <config/objspace.usemodules._minimal_curses>`
+    _multiprocessing
+    _random
+    :doc:`_rawffi <discussion/ctypes-implementation>`
+    _sha
+    _socket
+    _sre
+    _ssl
+    _warnings
+    _weakref
+    _winreg
+    array
+    binascii
+    bz2
+    cStringIO
+    cmath
+    `cpyext`_
+    crypt
+    errno
+    exceptions
+    fcntl
+    gc
+    imp
+    itertools
+    marshal
+    math
+    mmap
+    operator
+    parser
+    posix
+    pyexpat
+    select
+    signal
+    struct
+    symbol
+    sys
+    termios
+    thread
+    time
+    token
+    unicodedata
+    zipimport
+    zlib
+
+  When translated on Windows, a few Unix-only modules are skipped,
+  and the following module is built instead:
+
+    _winreg
+
+* Supported by being rewritten in pure Python (possibly using ``cffi``):
+  see the :source:`lib_pypy/` directory.  Examples of modules that we
+  support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``...
+  Note that some modules are both in there and in the list above;
+  by default, the built-in module is used (but can be disabled
+  at translation time).
+
+The extension modules (i.e. modules written in C, in the standard CPython)
+that are neither mentioned above nor in :source:`lib_pypy/` are not available in PyPy.
+(You may have a chance to use them anyway with `cpyext`_.)
+
+.. _cpyext: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html
+
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
 .. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/
+.. _SyntaxError: https://morepypy.blogspot.co.il/2018/04/improving-syntaxerror-in-pypy.html
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -121,6 +121,166 @@
   alive by GC objects, but not accounted in the GC
 
 
+GC Hooks
+--------
+
+GC hooks are user-defined functions which are called whenever a specific GC
+event occur, and can be used to monitor GC activity and pauses.  You can
+install the hooks by setting the following attributes:
+
+``gc.hook.on_gc_minor``
+    Called whenever a minor collection occurs. It corresponds to
+    ``gc-minor`` sections inside ``PYPYLOG``.
+
+``gc.hook.on_gc_collect_step``
+    Called whenever an incremental step of a major collection occurs. It
+    corresponds to ``gc-collect-step`` sections inside ``PYPYLOG``.
+
+``gc.hook.on_gc_collect``
+    Called after the last incremental step, when a major collection is fully
+    done. It corresponds to ``gc-collect-done`` sections inside ``PYPYLOG``.
+
+To uninstall a hook, simply set the corresponding attribute to ``None``.  To
+install all hooks at once, you can call ``gc.hooks.set(obj)``, which will look
+for methods ``on_gc_*`` on ``obj``.  To uninstall all the hooks at once, you
+can call ``gc.hooks.reset()``.
+
+The functions called by the hooks receive a single ``stats`` argument, which
+contains various statistics about the event.
+
+Note that PyPy cannot call the hooks immediately after a GC event, but it has
+to wait until it reaches a point in which the interpreter is in a known state
+and calling user-defined code is harmless.  It might happen that multiple
+events occur before the hook is invoked: in this case, you can inspect the
+value ``stats.count`` to know how many times the event occured since the last
+time the hook was called.  Similarly, ``stats.duration`` contains the
+**total** time spent by the GC for this specific event since the last time the
+hook was called.
+
+On the other hand, all the other fields of the ``stats`` object are relative
+only to the **last** event of the series.
+
+The attributes for ``GcMinorStats`` are:
+
+``count``
+    The number of minor collections occured since the last hook call.
+
+``duration``
+    The total time spent inside minor collections since the last hook
+    call. See below for more information on the unit.
+
+``duration_min``
+    The duration of the fastest minor collection since the last hook call.
+    
+``duration_max``
+    The duration of the slowest minor collection since the last hook call.
+
+ ``total_memory_used``
+    The amount of memory used at the end of the minor collection, in
+    bytes. This include the memory used in arenas (for GC-managed memory) and
+    raw-malloced memory (e.g., the content of numpy arrays).
+
+``pinned_objects``
+    the number of pinned objects.
+
+
+The attributes for ``GcCollectStepStats`` are:
+
+``count``, ``duration``, ``duration_min``, ``duration_max``
+    See above.
+
+``oldstate``, ``newstate``
+    Integers which indicate the state of the GC before and after the step.
+
+The value of ``oldstate`` and ``newstate`` is one of these constants, defined
+inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``,
+``STATE_SWEEPING``, ``STATE_FINALIZING``.  It is possible to get a string
+representation of it by indexing the ``GC_STATS`` tuple.
+
+
+The attributes for ``GcCollectStats`` are:
+
+``count``
+    See above.
+
+``num_major_collects``
+    The total number of major collections which have been done since the
+    start. Contrarily to ``count``, this is an always-growing counter and it's
+    not reset between invocations.
+
+``arenas_count_before``, ``arenas_count_after``
+    Number of arenas used before and after the major collection.
+
+``arenas_bytes``
+    Total number of bytes used by GC-managed objects.
+
+``rawmalloc_bytes_before``, ``rawmalloc_bytes_after``
+    Total number of bytes used by raw-malloced objects, before and after the
+    major collection.
+
+Note that ``GcCollectStats`` has **not** got a ``duration`` field. This is
+because all the GC work is done inside ``gc-collect-step``:
+``gc-collect-done`` is used only to give additional stats, but doesn't do any
+actual work.
+
+A note about the ``duration`` field: depending on the architecture and
+operating system, PyPy uses different ways to read timestamps, so ``duration``
+is expressed in varying units. It is possible to know which by calling
+``__pypy__.debug_get_timestamp_unit()``, which can be one of the following
+values:
+
+``tsc``
+    The default on ``x86`` machines: timestamps are expressed in CPU ticks, as
+    read by the `Time Stamp Counter`_.
+
+``ns``
+    Timestamps are expressed in nanoseconds.
+
+``QueryPerformanceCounter``
+    On Windows, in case for some reason ``tsc`` is not available: timestamps
+    are read using the win API ``QueryPerformanceCounter()``.
+
+
+Unfortunately, there does not seem to be a reliable standard way for
+converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs
+it is enough to divide the ticks by the maximum nominal frequency of the CPU.
+For this reason, PyPy gives the raw value, and leaves the job of doing the
+conversion to external libraries.
+
+Here is an example of GC hooks in use::
+
+    import sys
+    import gc
+
+    class MyHooks(object):
+        done = False
+
+        def on_gc_minor(self, stats):
+            print 'gc-minor:        count = %02d, duration = %d' % (stats.count,
+                                                                    stats.duration)
+
+        def on_gc_collect_step(self, stats):
+            old = gc.GcCollectStepStats.GC_STATES[stats.oldstate]
+            new = gc.GcCollectStepStats.GC_STATES[stats.newstate]
+            print 'gc-collect-step: %s --> %s' % (old, new)
+            print '                 count = %02d, duration = %d' % (stats.count,
+                                                                    stats.duration)
+
+        def on_gc_collect(self, stats):
+            print 'gc-collect-done: count = %02d' % stats.count
+            self.done = True
+
+    hooks = MyHooks()
+    gc.hooks.set(hooks)
+
+    # simulate some GC activity
+    lst = []
+    while not hooks.done:
+        lst = [lst, 1, 2, 3]
+
+
+.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter    
+    
 .. _minimark-environment-variables:
 
 Environment variables
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -40,6 +40,8 @@
   sure things are ported back to the trunk and to the branch as
   necessary.
 
+* Make sure the RPython builds on the buildbot pass with no failures
+
 * Maybe bump the SOABI number in module/imp/importing. This has many
   implications, so make sure the PyPy community agrees to the change.
 
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v6.0.0.rst
    release-v5.10.1.rst
    release-v5.10.0.rst
    release-v5.9.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,8 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-6.0.0.rst
+   whatsnew-pypy2-5.10.0.rst
    whatsnew-pypy2-5.10.0.rst
    whatsnew-pypy2-5.9.0.rst
    whatsnew-pypy2-5.8.0.rst
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -17,13 +17,18 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The quickest way to start using PyPy is to download a prebuilt binary for your
-OS and architecture.  You can either use the `most recent release`_ or one of
-our `development nightly build`_.  Please note that the nightly builds are not
+OS and architecture.  You may be able to use either use the
+`most recent release`_ or one of our `development nightly build`_. These
+builds depend on dynamically linked libraries that may not be available on your
+OS. See the section about `Linux binaries` for more info and alternatives that
+may work on your system.
+
+Please note that the nightly builds are not
 guaranteed to be as stable as official releases, use them at your own risk.
 
 .. _most recent release: http://pypy.org/download.html
 .. _development nightly build: http://buildbot.pypy.org/nightly/trunk/
-
+.. _Linux binaries: http://pypy.org/download.html#linux-binaries-and-common-distributions
 
 Installing PyPy
 ~~~~~~~~~~~~~~~
@@ -69,9 +74,9 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 It is often convenient to run pypy inside a virtualenv.  To do this
-you need a recent version of virtualenv -- 1.6.1 or greater.  You can
+you need a version of virtualenv -- 1.6.1 or greater.  You can
 then install PyPy both from a precompiled tarball or from a mercurial
-checkout::
+checkout after translation::
 
 	# from a tarball
 	$ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env
diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v6.0.0.rst
@@ -0,0 +1,117 @@
+======================================
+PyPy2.7 and PyPy3.5 v6.0 dual release
+======================================
+
+The PyPy team is proud to release both PyPy2.7 v6.0 (an interpreter supporting
+Python 2.7 syntax), and a PyPy3.5 v6.0 (an interpreter supporting Python
+3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.
+
+This release is a feature release following our previous 5.10 incremental
+release in late December 2017. Our C-API compatability layer ``cpyext`` is
+now much faster (see the `blog post`_) as well as more complete. We have made
+many other improvements in speed and CPython compatibility. Since the changes
+affect the included python development header files, all c-extension modules must
+be recompiled for this version.
+
+First-time python users are often stumped by silly typos and emissions when
+getting started writing code. We have improved our parser to emit more friendly
+`syntax errors`_,  making PyPy not only faster but more friendly.
+
+The GC now has `hooks`_ to gain more insights into its performance
+
+The Matplotlib TkAgg backend now works with PyPy, as do pygame and pygobject_.
+
+We updated the `cffi`_ module included in PyPy to version 1.11.5, and the
+`cppyy`_ backend to 0.6.0. Please use these to wrap your C and C++ code,
+respectively, for a JIT friendly experience.
+
+As always, this release is 100% compatible with the previous one and fixed
+several issues and bugs raised by the growing community of PyPy users.
+We strongly recommend updating.
+
+The Windows PyPy3.5 release is still considered beta-quality. There are open
+issues with unicode handling especially around system calls and c-extensions.
+
+The utf8 branch that changes internal representation of unicode to utf8 did not
+make it into the release, so there is still more goodness coming. We also
+began working on a Python3.6 implementation, help is welcome.
+
+You can download the v6.0 releases here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project. If PyPy is not quite good enough for your needs, we are available for
+direct consulting work.
+
+We would also like to thank our contributors and encourage new people to join
+the project. PyPy has many layers and we need help with all of them: `PyPy`_
+and `RPython`_ documentation improvements, tweaking popular `modules`_ to run
+on pypy, or general `help`_ with making RPython's JIT even better.
+
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: project-ideas.html
+.. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html
+.. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html
+.. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html
+.. _`hooks`: gc_info.html#gc-hooks
+.. _`cffi`: http://cffi.readthedocs.io
+.. _`cppyy`: https://cppyy.readthedocs.io
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+The PyPy release supports:
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Changelog
+=========
+
+* Speed up C-API method calls, and make most Py*_Check calls C macros
+* Speed up C-API slot method calls
+* Enable TkAgg backend support for matplotlib
+* support ``hastzinfo`` and ``tzinfo`` in the C-API ``PyDateTime*`` structures
+* datetime.h is now more similar to CPython
+* We now support ``PyUnicode_AsUTF{16,32}String``, ``_PyLong_AsByteArray``,
+  ``_PyLong_AsByteArrayO``,
+* PyPy3.5 on Windows is compiled with the Microsoft Visual Compiler v14, like
+  CPython
+* Fix performance of attribute lookup when more than 80 attributes are used
+* Improve performance on passing built-in types to C-API C code
+* Improve the performance of datetime and timedelta by skipping the consistency
+  checks of the datetime values (they are correct by construction)
+* Improve handling of ``bigint`` s, including fixing ``int_divmod``
+* Improve reporting of GC statistics
+* Accept unicode filenames in ``dbm.open()``
+* Improve RPython support for half-floats
+* Added missing attributes to C-API ``instancemethod`` on pypy3
+* Store error state in thread-local storage for C-API.
+* Fix JIT bugs exposed in the sre module
+* Improve speed of Python parser, improve ParseError messages and SyntaxError
+* Handle JIT hooks more efficiently
+* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s
+
+We also refactored many parts of the JIT bridge optimizations, as well as cpyext
+internals, and together with new contributors fixed issues, added new
+documentation, and cleaned up the codebase.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,56 +1,9 @@
-===========================
-What's new in PyPy2.7 5.10+
-===========================
+==========================
+What's new in PyPy2.7 6.0+
+==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.10.0
-.. startrev: 6b024edd9d12
+.. this is a revision shortly after release-pypy-6.0.0
+.. startrev: e50e11af23f1
 
-.. branch: cpyext-avoid-roundtrip
 
-Big refactoring of some cpyext code, which avoids a lot of nonsense when
-calling C from Python and vice-versa: the result is a big speedup in
-function/method calls, up to 6 times faster.
 
-.. branch: cpyext-datetime2
-
-Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD
-
-
-.. branch: mapdict-size-limit
-
-Fix a corner case of mapdict: When an instance is used like a dict (using
-``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are
-added, then the performance using mapdict is linear in the number of
-attributes. This is now fixed (by switching to a regular dict after 80
-attributes).
-
-
-.. branch: cpyext-faster-arg-passing
-
-When using cpyext, improve the speed of passing certain objects from PyPy to C
-code, most notably None, True, False, types, all instances of C-defined types.
-Before, a dict lookup was needed every time such an object crossed over, now it
-is just a field read.
-
-
-.. branch: 2634_datetime_timedelta_performance
-
-Improve datetime + timedelta performance.
-
-.. branch: memory-accounting
-
-Improve way to describe memory
-
-.. branch: msvc14
-
-Allow compilaiton with Visual Studio 2017 compiler suite on windows
-
-.. branch: refactor-slots
-
-Refactor cpyext slots.
-
-
-.. branch: call-loopinvariant-into-bridges
-
-Speed up branchy code that does a lot of function inlining by saving one call
-to read the TLS in most bridges.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-6.0.0.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst
@@ -54,3 +54,75 @@
 
 Speed up branchy code that does a lot of function inlining by saving one call
 to read the TLS in most bridges.
+
+.. branch: rpython-sprint
+
+Refactor in rpython signatures
+
+.. branch: cpyext-tls-operror2
+
+Store error state thread-locally in executioncontext, fixes issue #2764
+
+.. branch: cpyext-fast-typecheck
+
+Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify
+`W_PyCWrapperObject` which is used to call slots from the C-API, greatly
+improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks
+
+
+.. branch: fix-sre-problems
+
+Fix two (unrelated) JIT bugs manifesting in the re module:
+
+- green fields are broken and were thus disabled, plus their usage removed from
+  the _sre implementation
+
+- in rare "trace is too long" situations, the JIT could break behaviour
+  arbitrarily.
+
+.. branch: jit-hooks-can-be-disabled
+
+Be more efficient about JIT hooks. Make it possible for the frontend to declare
+that jit hooks are currently not enabled at all. in that case, the list of ops
+does not have to be created in the case of the on_abort hook (which is
+expensive).
+
+
+.. branch: pyparser-improvements
+
+Improve speed of Python parser, improve ParseError messages slightly.
+
+.. branch: ioctl-arg-size
+
+Work around possible bugs in upstream ioctl users, like CPython allocate at
+least 1024 bytes for the arg in calls to ``ioctl(fd, request, arg)``. Fixes
+issue #2776
+
+.. branch: cpyext-subclass-setattr
+
+Fix for python-level classes that inherit from C-API types, previously the
+`w_obj` was not necessarily preserved throughout the lifetime of the `pyobj`
+which led to cases where instance attributes were lost. Fixes issue #2793
+
+
+.. branch: pyparser-improvements-2
+
+Improve line offsets that are reported by SyntaxError. Improve error messages
+for a few situations, including mismatched parenthesis.
+
+.. branch: issue2752
+
+Fix a rare GC bug that was introduced more than one year ago, but was
+not diagnosed before issue #2752.
+
+.. branch: gc-hooks
+
+Introduce GC hooks, as documented in doc/gc_info.rst
+
+.. branch: gc-hook-better-timestamp
+
+Improve GC hooks
+
+.. branch: cppyy-packaging
+
+Update backend to 0.6.0 and support exceptions through wrappers
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -215,6 +215,7 @@
     usage = SUPPRESS_USAGE
 
     take_options = True
+    space = None
 
     def opt_parser(self, config):
         parser = to_optparse(config, useoptions=["objspace.*"],
@@ -364,15 +365,21 @@
         from pypy.module.pypyjit.hooks import pypy_hooks
         return PyPyJitPolicy(pypy_hooks)
 
+    def get_gchooks(self):
+        from pypy.module.gc.hook import LowLevelGcHooks
+        if self.space is None:
+            raise Exception("get_gchooks must be called afeter get_entry_point")
+        return self.space.fromcache(LowLevelGcHooks)
+
     def get_entry_point(self, config):
-        space = make_objspace(config)
+        self.space = make_objspace(config)
 
         # manually imports app_main.py
         filename = os.path.join(pypydir, 'interpreter', 'app_main.py')
         app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main')
         app.hidden_applevel = False
-        w_dict = app.getwdict(space)
-        entry_point, _ = create_entry_point(space, w_dict)
+        w_dict = app.getwdict(self.space)
+        entry_point, _ = create_entry_point(self.space, w_dict)
 
         return entry_point, None, PyPyAnnotatorPolicy()
 
@@ -381,7 +388,7 @@
                      'jitpolicy', 'get_entry_point',
                      'get_additional_config_options']:
             ns[name] = getattr(self, name)
-
+        ns['get_gchooks'] = self.get_gchooks
 
 PyPyTarget().interface(globals())
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -413,7 +413,7 @@
         self._periodic_actions = []
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
-        self.fired_actions = None
+        self._fired_actions_reset()
         # the default value is not 100, unlike CPython 2.7, but a much
         # larger value, because we use a technique that not only allows
         # but actually *forces* another thread to run whenever the counter
@@ -425,13 +425,28 @@
         """Request for the action to be run before the next opcode."""
         if not action._fired:
             action._fired = True
-            if self.fired_actions is None:
-                self.fired_actions = []
-            self.fired_actions.append(action)
+            self._fired_actions_append(action)
             # set the ticker to -1 in order to force action_dispatcher()
             # to run at the next possible bytecode
             self.reset_ticker(-1)
 
+    def _fired_actions_reset(self):
+        # linked list of actions. We cannot use a normal RPython list because
+        # we want AsyncAction.fire() to be marked as @rgc.collect: this way,
+        # we can call it from e.g. GcHooks or cpyext's dealloc_trigger.
+        self._fired_actions_first = None
+        self._fired_actions_last = None
+
+    @rgc.no_collect
+    def _fired_actions_append(self, action):
+        assert action._next is None
+        if self._fired_actions_first is None:
+            self._fired_actions_first = action
+            self._fired_actions_last = action
+        else:
+            self._fired_actions_last._next = action
+            self._fired_actions_last = action
+
     @not_rpython
     def register_periodic_action(self, action, use_bytecode_counter):
         """
@@ -476,19 +491,26 @@
                 action.perform(ec, frame)
 
             # nonperiodic actions
-            list = self.fired_actions
-            if list is not None:
-                self.fired_actions = None
+            action = self._fired_actions_first
+            if action:
+                self._fired_actions_reset()
                 # NB. in case there are several actions, we reset each
                 # 'action._fired' to false only when we're about to call
                 # 'action.perform()'.  This means that if
                 # 'action.fire()' happens to be called any time before
                 # the corresponding perform(), the fire() has no
                 # effect---which is the effect we want, because
-                # perform() will be called anyway.
-                for action in list:
+                # perform() will be called anyway.  All such pending
+                # actions with _fired == True are still inside the old
+                # chained list.  As soon as we reset _fired to False,
+                # we also reset _next to None and we are ready for
+                # another fire().
+                while action is not None:
+                    next_action = action._next
+                    action._next = None
                     action._fired = False
                     action.perform(ec, frame)
+                    action = next_action
 
         self.action_dispatcher = action_dispatcher
 
@@ -522,10 +544,12 @@
     to occur between two opcodes, not at a completely random time.
     """
     _fired = False
+    _next = None
 
     def __init__(self, space):
         self.space = space
 
+    @rgc.no_collect
     def fire(self):
         """Request for the action to be run before the next opcode.
         The action must have been registered at space initalization time."""
diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py
--- a/pypy/interpreter/pyparser/error.py
+++ b/pypy/interpreter/pyparser/error.py
@@ -6,6 +6,7 @@
                  lastlineno=0):
         self.msg = msg
         self.lineno = lineno
+        # NB: offset is a 1-based index!
         self.offset = offset
         self.text = text
         self.filename = filename
diff --git a/pypy/interpreter/pyparser/metaparser.py b/pypy/interpreter/pyparser/metaparser.py
--- a/pypy/interpreter/pyparser/metaparser.py
+++ b/pypy/interpreter/pyparser/metaparser.py
@@ -147,8 +147,10 @@
                 for label, next in state.arcs.iteritems():
                     arcs.append((self.make_label(gram, label), dfa.index(next)))
                 states.append((arcs, state.is_final))
-            gram.dfas.append((states, self.make_first(gram, name)))
-            assert len(gram.dfas) - 1 == gram.symbol_ids[name] - 256
+            symbol_id = gram.symbol_ids[name]
+            dfa = parser.DFA(symbol_id, states, self.make_first(gram, name))
+            gram.dfas.append(dfa)
+            assert len(gram.dfas) - 1 == symbol_id - 256
         gram.start = gram.symbol_ids[self.start_symbol]
         return gram
 
@@ -162,6 +164,13 @@
                 else:
                     gram.labels.append(gram.symbol_ids[label])
                     gram.symbol_to_label[label] = label_index
+                    first = self.first[label]
+                    if len(first) == 1:
+                        first, = first
+                        if not first[0].isupper():
+                            first = first.strip("\"'")
+                            assert label_index not in gram.token_to_error_string
+                            gram.token_to_error_string[label_index] = first
                     return label_index
             elif label.isupper():
                 token_index = gram.TOKENS[label]
@@ -183,7 +192,7 @@
                 else:
                     gram.labels.append(gram.KEYWORD_TOKEN)
                     gram.keyword_ids[value] = label_index
-                    return label_index
+                    result = label_index
             else:
                 try:
                     token_index = gram.OPERATOR_MAP[value]
@@ -194,7 +203,10 @@
                 else:
                     gram.labels.append(token_index)
                     gram.token_ids[token_index] = label_index
-                    return label_index
+                    result = label_index
+            assert result not in gram.token_to_error_string
+            gram.token_to_error_string[result] = value
+            return result
 
     def make_first(self, gram, name):
         original_firsts = self.first[name]
diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py
--- a/pypy/interpreter/pyparser/parser.py
+++ b/pypy/interpreter/pyparser/parser.py
@@ -1,6 +1,7 @@
 """
 A CPython inspired RPython parser.
 """
+from rpython.rlib.objectmodel import not_rpython
 
 
 class Grammar(object):
@@ -16,6 +17,7 @@
         self.symbol_names = {}
         self.symbol_to_label = {}
         self.keyword_ids = {}
+        self.token_to_error_string = {}
         self.dfas = []
         self.labels = [0]
         self.token_ids = {}
@@ -41,6 +43,27 @@
             pass
         return True
 
+class DFA(object):
+    def __init__(self, symbol_id, states, first):
+        self.symbol_id = symbol_id
+        self.states = states
+        self.first = self._first_to_string(first)
+
+    def could_match_token(self, label_index):
+        pos = label_index >> 3
+        bit = 1 << (label_index & 0b111)
+        return bool(ord(self.first[label_index >> 3]) & bit)
+
+    @staticmethod
+    @not_rpython
+    def _first_to_string(first):
+        l = sorted(first.keys())
+        b = bytearray(32)
+        for label_index in l:
+            pos = label_index >> 3
+            bit = 1 << (label_index & 0b111)
+            b[pos] |= bit
+        return str(b)
 
 class Node(object):
 
@@ -127,14 +150,17 @@
 
 class Nonterminal(AbstractNonterminal):
     __slots__ = ("_children", )
-    def __init__(self, type, children):
+    def __init__(self, type, children=None):
         Node.__init__(self, type)
+        if children is None:
+            children = []
         self._children = children
 
     def __repr__(self):
         return "Nonterminal(type=%s, children=%r)" % (self.type, self._children)
 
     def get_child(self, i):
+        assert self._children is not None
         return self._children[i]
 
     def num_children(self):
@@ -168,25 +194,50 @@
 class ParseError(Exception):
 
     def __init__(self, msg, token_type, value, lineno, column, line,
-                 expected=-1):
+                 expected=-1, expected_str=None):
         self.msg = msg
         self.token_type = token_type
         self.value = value
         self.lineno = lineno
+        # this is a 0-based index
         self.column = column
         self.line = line
         self.expected = expected
+        self.expected_str = expected_str
 
     def __str__(self):
         return "ParserError(%s, %r)" % (self.token_type, self.value)
 
 
+class StackEntry(object):
+    def __init__(self, next, dfa, state):
+        self.next = next
+        self.dfa = dfa
+        self.state = state
+        self.node = None
+
+    def push(self, dfa, state):
+        return StackEntry(self, dfa, state)
+
+    def pop(self):
+        return self.next
+
+    def node_append_child(self, child):
+        node = self.node
+        if node is None:
+            self.node = Nonterminal1(self.dfa.symbol_id, child)
+        elif isinstance(node, Nonterminal1):
+            newnode = self.node = Nonterminal(
+                    self.dfa.symbol_id, [node._child, child])
+        else:
+            self.node.append_child(child)
+
+
 class Parser(object):
 
     def __init__(self, grammar):
         self.grammar = grammar
         self.root = None
-        self.stack = None
 
     def prepare(self, start=-1):
         """Setup the parser for parsing.
@@ -196,16 +247,15 @@
         if start == -1:
             start = self.grammar.start
         self.root = None
-        current_node = Nonterminal(start, [])
-        self.stack = []
-        self.stack.append((self.grammar.dfas[start - 256], 0, current_node))
+        self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0)
 
     def add_token(self, token_type, value, lineno, column, line):
         label_index = self.classify(token_type, value, lineno, column, line)
         sym_id = 0 # for the annotator
         while True:
-            dfa, state_index, node = self.stack[-1]
-            states, first = dfa
+            dfa = self.stack.dfa
+            state_index = self.stack.state
+            states = dfa.states
             arcs, is_accepting = states[state_index]
             for i, next_state in arcs:
                 sym_id = self.grammar.labels[i]
@@ -217,16 +267,17 @@
                     # the stack.
                     while state[1] and not state[0]:
                         self.pop()
-                        if not self.stack:
+                        if self.stack is None:
                             # Parsing is done.
                             return True
-                        dfa, state_index, node = self.stack[-1]
-                        state = dfa[0][state_index]
+                        dfa = self.stack.dfa
+                        state_index = self.stack.state
+                        state = dfa.states[state_index]
                     return False
                 elif sym_id >= 256:
                     sub_node_dfa = self.grammar.dfas[sym_id - 256]
                     # Check if this token can start a child node.
-                    if label_index in sub_node_dfa[1]:
+                    if sub_node_dfa.could_match_token(label_index):
                         self.push(sub_node_dfa, next_state, sym_id, lineno,
                                   column)
                         break
@@ -235,7 +286,7 @@
                 # state is accepting, it's invalid input.
                 if is_accepting:
                     self.pop()
-                    if not self.stack:
+                    if self.stack is None:
                         raise ParseError("too much input", token_type, value,
                                          lineno, column, line)
                 else:
@@ -243,10 +294,13 @@
                     # error.
                     if len(arcs) == 1:
                         expected = sym_id
+                        expected_str = self.grammar.token_to_error_string.get(
+                                arcs[0][0], None)
                     else:
                         expected = -1
+                        expected_str = None
                     raise ParseError("bad input", token_type, value, lineno,
-                                     column, line, expected)
+                                     column, line, expected, expected_str)
 
     def classify(self, token_type, value, lineno, column, line):
         """Find the label for a token."""
@@ -262,26 +316,22 @@
 
     def shift(self, next_state, token_type, value, lineno, column):
         """Shift a non-terminal and prepare for the next state."""
-        dfa, state, node = self.stack[-1]
         new_node = Terminal(token_type, value, lineno, column)
-        node.append_child(new_node)
-        self.stack[-1] = (dfa, next_state, node)
+        self.stack.node_append_child(new_node)
+        self.stack.state = next_state
 
     def push(self, next_dfa, next_state, node_type, lineno, column):
         """Push a terminal and adjust the current state."""
-        dfa, state, node = self.stack[-1]
-        new_node = Nonterminal(node_type, [])
-        self.stack[-1] = (dfa, next_state, node)
-        self.stack.append((next_dfa, 0, new_node))
+        self.stack.state = next_state
+        self.stack = self.stack.push(next_dfa, 0)
 
     def pop(self):
         """Pop an entry off the stack and make its node a child of the last."""
-        dfa, state, node = self.stack.pop()
+        top = self.stack
+        self.stack = top.pop()
+        node = top.node
+        assert node is not None
         if self.stack:
-            # we are now done with node, so we can store it more efficiently if
-            # it has just one child
-            if node.num_children() == 1:
-                node = Nonterminal1(node.type, node.get_child(0))
-            self.stack[-1][2].append_child(node)
+            self.stack.node_append_child(node)
         else:
             self.root = node
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -132,7 +132,11 @@
                         w_message = space.str(e.get_w_value(space))
                         raise error.SyntaxError(space.text_w(w_message))
                     raise
+        if enc is not None:
+            compile_info.encoding = enc
+        return self._parse(textsrc, compile_info)
 
+    def _parse(self, textsrc, compile_info):
         flags = compile_info.flags
 
         # The tokenizer is very picky about how it wants its input.
@@ -181,13 +185,16 @@
                 else:
                     new_err = error.SyntaxError
                     msg = "invalid syntax"
-                raise new_err(msg, e.lineno, e.column, e.line,
+                    if e.expected_str is not None:
+                        msg += " (expected '%s')" % e.expected_str
+
+                # parser.ParseError(...).column is 0-based, but the offsets in the
+                # exceptions in the error module are 1-based, hence the '+ 1'
+                raise new_err(msg, e.lineno, e.column + 1, e.line,
                               compile_info.filename)
             else:
                 tree = self.root
         finally:
             # Avoid hanging onto the tree.
             self.root = None
-        if enc is not None:
-            compile_info.encoding = enc
         return tree
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -73,14 +73,14 @@
         logical line; continuation lines are included.
     """
     token_list = []
-    lnum = parenlev = continued = 0
+    lnum = continued = 0
     namechars = NAMECHARS
     numchars = NUMCHARS
     contstr, needcont = '', 0
     contline = None
     indents = [0]
     last_comment = ''
-    parenlevstart = (0, 0, "")
+    parenstack = []
 
     # make the annotator happy
     endDFA = DUMMY_DFA
@@ -97,7 +97,7 @@
         if contstr:
             if not line:
                 raise TokenError(
-                    "EOF while scanning triple-quoted string literal",
+                    "end of file (EOF) while scanning triple-quoted string literal",
                     strstart[2], strstart[0], strstart[1]+1,
                     token_list, lnum-1)
             endmatch = endDFA.recognize(line)
@@ -123,7 +123,7 @@
                 contline = contline + line
                 continue
 
-        elif parenlev == 0 and not continued:  # new statement
+        elif not parenstack and not continued:  # new statement
             if not line: break
             column = 0
             while pos < max:                   # measure leading whitespace
@@ -143,21 +143,21 @@
                 token_list.append((tokens.INDENT, line[:pos], lnum, 0, line))
                 last_comment = ''
             while column < indents[-1]:
-                indents = indents[:-1]
+                indents.pop()
                 token_list.append((tokens.DEDENT, '', lnum, pos, line))
                 last_comment = ''
             if column != indents[-1]:
                 err = "unindent does not match any outer indentation level"
-                raise TokenIndentationError(err, line, lnum, 0, token_list)
+                raise TokenIndentationError(err, line, lnum, column+1, token_list)
 
         else:                                  # continued statement
             if not line:
-                if parenlev > 0:
-                    lnum1, start1, line1 = parenlevstart
+                if parenstack:
+                    _, lnum1, start1, line1 = parenstack[0]
                     raise TokenError("parenthesis is never closed", line1,
                                      lnum1, start1 + 1, token_list, lnum)
-                raise TokenError("EOF in multi-line statement", line,
-                                 lnum, 0, token_list)
+                raise TokenError("end of file (EOF) in multi-line statement", line,
+                                 lnum, 0, token_list) # XXX why is the offset 0 here?
             continued = 0
 
         while pos < max:
@@ -180,7 +180,7 @@
                     token_list.append((tokens.NUMBER, token, lnum, start, line))
                     last_comment = ''
                 elif initial in '\r\n':
-                    if parenlev <= 0:
+                    if not parenstack:
                         tok = (tokens.NEWLINE, last_comment, lnum, start, line)
                         token_list.append(tok)
                     last_comment = ''
@@ -226,14 +226,22 @@
                     last_comment = ''
                 else:
                     if initial in '([{':
-                        if parenlev == 0:
-                            parenlevstart = (lnum, start, line)
-                        parenlev = parenlev + 1
+                        parenstack.append((initial, lnum, start, line))
                     elif initial in ')]}':
-                        parenlev = parenlev - 1
-                        if parenlev < 0:
+                        if not parenstack:
                             raise TokenError("unmatched '%s'" % initial, line,
                                              lnum, start + 1, token_list)
+                        opening, lnum1, start1, line1 = parenstack.pop()
+                        if not ((opening == "(" and initial == ")") or
+                                (opening == "[" and initial == "]") or
+                                (opening == "{" and initial == "}")):
+                            msg = "closing parenthesis '%s' does not match opening parenthesis '%s'" % (
+                                        initial, opening)
+
+                            if lnum1 != lnum:
+                                msg += " on line " + str(lnum1)
+                            raise TokenError(
+                                    msg, line, lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
@@ -245,7 +253,7 @@
                 if start < 0:
                     start = pos
                 if start<max and line[start] in single_quoted:
-                    raise TokenError("EOL while scanning string literal",
+                    raise TokenError("end of line (EOL) while scanning string literal",
                              line, lnum, start+1, token_list)
                 tok = (tokens.ERRORTOKEN, line[pos], lnum, pos, line)
                 token_list.append(tok)
diff --git a/pypy/interpreter/pyparser/test/targetparse.py b/pypy/interpreter/pyparser/test/targetparse.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/test/targetparse.py
@@ -0,0 +1,50 @@
+import sys
+import os
+ROOT =  os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+print ROOT
+sys.path.insert(0, str(ROOT))
+import time
+from pypy.interpreter.pyparser import pyparse
+
+
+
+class FakeSpace(object):
+    pass
+
+fakespace = FakeSpace()
+
+def bench(fn, s):
+    a = time.clock()
+    info = pyparse.CompileInfo("<string>", "exec")
+    parser = pyparse.PythonParser(fakespace)
+    tree = parser._parse(s, info)
+    b = time.clock()
+    print fn, (b-a)
+
+
+def entry_point(argv):
+    if len(argv) == 2:
+        fn = argv[1]
+    else:
+        fn = "../../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py"
+    fd = os.open(fn, os.O_RDONLY, 0777)
+    res = []
+    while True:
+        s = os.read(fd, 4096)
+        if not s:
+            break
+        res.append(s)
+    os.close(fd)
+    s = "".join(res)
+    print len(s)
+    bench(fn, s)
+
+    return 0
+
+# _____ Define and setup target ___
+
+def target(*args):
+    return entry_point, None
+
+if __name__ == '__main__':
+    entry_point(sys.argv)
diff --git a/pypy/interpreter/pyparser/test/test_metaparser.py b/pypy/interpreter/pyparser/test/test_metaparser.py
--- a/pypy/interpreter/pyparser/test/test_metaparser.py
+++ b/pypy/interpreter/pyparser/test/test_metaparser.py
@@ -34,8 +34,8 @@
         assert len(g.dfas) == 1
         eval_sym = g.symbol_ids["eval"]
         assert g.start == eval_sym
-        states, first = g.dfas[eval_sym - 256]
-        assert states == [([(1, 1)], False), ([], True)]
+        dfa = g.dfas[eval_sym - 256]
+        assert dfa.states == [([(1, 1)], False), ([], True)]
         assert g.labels[0] == 0
 
     def test_load_python_grammars(self):
@@ -51,7 +51,7 @@
     def test_items(self):
         g = self.gram_for("foo: NAME STRING OP '+'")
         assert len(g.dfas) == 1
-        states = g.dfas[g.symbol_ids["foo"] - 256][0]
+        states = g.dfas[g.symbol_ids["foo"] - 256].states
         last = states[0][0][0][1]
         for state in states[1:-1]:
             assert last < state[0][0][1]
diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py
--- a/pypy/interpreter/pyparser/test/test_parser.py
+++ b/pypy/interpreter/pyparser/test/test_parser.py
@@ -7,6 +7,12 @@
 from pypy.interpreter.pyparser.test.test_metaparser import MyGrammar
 
 
+def test_char_set():
+    first = {5: None, 9: None, 100: None, 255:None}
+    p = parser.DFA(None, None, first)
+    for i in range(256):
+        assert p.could_match_token(i) == (i in first)
+
 class SimpleParser(parser.Parser):
 
     def parse(self, input):
@@ -55,8 +61,7 @@
             n = parser.Terminal(tp, value, 0, 0)
         else:
             tp = gram.symbol_ids[data[0]]
-            children = []
-            n = parser.Nonterminal(tp, children)
+            n = parser.Nonterminal(tp)
         new_indent = count_indent(line)
         if new_indent >= last_indent:
             if new_indent == last_indent and node_stack:
@@ -291,3 +296,37 @@
             NEWLINE
             ENDMARKER"""
         assert tree_from_string(expected, gram) == p.parse("hi 42 end")
+
+
+    def test_optimized_terminal(self):
+        gram = """foo: bar baz 'end' NEWLINE ENDMARKER
+bar: NAME
+baz: NUMBER
+"""
+        p, gram = self.parser_for(gram, False)
+        expected = """
+        foo
+            bar
+                NAME "a_name"
+            baz
+                NUMBER "42"
+            NAME "end"
+            NEWLINE
+            ENDMARKER"""
+        input = "a_name 42 end"
+        tree = p.parse(input)
+        assert tree_from_string(expected, gram) == tree
+        assert isinstance(tree, parser.Nonterminal)
+        assert isinstance(tree.get_child(0), parser.Nonterminal1)
+        assert isinstance(tree.get_child(1), parser.Nonterminal1)
+
+
+    def test_error_string(self):
+        p, gram = self.parser_for(
+            "foo: 'if' NUMBER '+' NUMBER"
+        )
+        info = py.test.raises(parser.ParseError, p.parse, "if 42")
+        info.value.expected_str is None
+        info = py.test.raises(parser.ParseError, p.parse, "if 42 42")
+        info.value.expected_str == '+'
+
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -76,14 +76,14 @@
         exc = py.test.raises(SyntaxError, parse, "name another for").value
         assert exc.msg == "invalid syntax"
         assert exc.lineno == 1
-        assert exc.offset == 5
+        assert exc.offset == 6
         assert exc.text.startswith("name another for")
         exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value
-        assert exc.msg == "EOL while scanning string literal"
+        assert exc.msg == "end of line (EOL) while scanning string literal"
         assert exc.lineno == 1
         assert exc.offset == 5
         exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value
-        assert exc.msg == "EOF while scanning triple-quoted string literal"
+        assert exc.msg == "end of file (EOF) while scanning triple-quoted string literal"
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 3
@@ -112,7 +112,7 @@
         assert exc.msg == "expected an indented block"
         assert exc.lineno == 3
         assert exc.text.startswith("pass")
-        assert exc.offset == 0
+        assert exc.offset == 1
         input = "hi\n    indented"
         exc = py.test.raises(IndentationError, parse, input).value
         assert exc.msg == "unexpected indent"
@@ -120,6 +120,7 @@
         exc = py.test.raises(IndentationError, parse, input).value
         assert exc.msg == "unindent does not match any outer indentation level"
         assert exc.lineno == 3
+        assert exc.offset == 3
 
     def test_mac_newline(self):
         self.parse("this_is\ra_mac\rfile")
@@ -174,3 +175,11 @@
         py.test.raises(SyntaxError, self.parse, '$')
         py.test.raises(SyntaxError, self.parse, '$a')
         py.test.raises(SyntaxError, self.parse, '$.5')
+
+    def test_error_forgotten_chars(self):
+        info = py.test.raises(SyntaxError, self.parse, "if 1\n    print 4")

From pypy.commits at gmail.com  Sun May 13 17:41:10 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 May 2018 14:41:10 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: improve the test
Message-ID: <5af8b0f6.1c69fb81.cb2e2.d085@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94550:265844e2ae4e
Date: 2018-05-13 23:40 +0200
http://bitbucket.org/pypy/pypy/changeset/265844e2ae4e/

Log:	improve the test

diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -832,6 +832,10 @@
             check_content(strings, rawptrs)
             rgc.collect(); rgc.collect(); rgc.collect()
             check_content(strings, rawptrs)
+            for i in range(len(strings)):   # check that it still returns the
+                                            # same raw ptrs
+                p1 = rffi._get_raw_address_buf_from_string(strings[i])
+                assert rawptrs[i] == p1
             del strings
             rgc.collect(); rgc.collect(); rgc.collect()
             return 42

From pypy.commits at gmail.com  Sun May 13 17:38:39 2018
From: pypy.commits at gmail.com (arigo)
Date: Sun, 13 May 2018 14:38:39 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: Test was already
 passing, just confused about debug_print().
Message-ID: <5af8b05f.1c69fb81.f90f0.b0b3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94549:3b526d4fd961
Date: 2018-05-13 23:38 +0200
http://bitbucket.org/pypy/pypy/changeset/3b526d4fd961/

Log:	Test was already passing, just confused about debug_print().

diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1376,9 +1376,7 @@
     def finalizer_trigger(self):
         from rpython.rtyper.annlowlevel import hlstr
         from rpython.rtyper.lltypesystem import rstr
-        from rpython.rlib.debug import debug_print
         from rpython.rlib import objectmodel
-        debug_print("HI THERE")
         while True:
             gcptr = self.next_dead()
             if not gcptr:
@@ -1387,10 +1385,10 @@
             string = hlstr(ll_string)
             key = objectmodel.compute_unique_id(string)
             ptr = self.raw_copies.get(key, lltype.nullptr(CCHARP.TO))
-            debug_print(ptr)
             if ptr:
                 if self.print_debugging:
-                    debug_print("freeing [", ptr, "]")
+                    from rpython.rlib.debug import debug_print
+                    debug_print("freeing str [", ptr, "]")
                 free_charp(ptr, track_allocation=False)
 _fq_addr_from_string = _StrFinalizerQueue()
 _fq_addr_from_string.raw_copies = {}    # {GCREF: CCHARP}
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -826,7 +826,7 @@
                     assert p[j] == expected[j]
 
         def f(n):
-            strings = [str(i) for i in range(n)]
+            strings = ["foo%d" % i for i in range(n)]
             rawptrs = [rffi._get_raw_address_buf_from_string(s)
                        for s in strings]
             check_content(strings, rawptrs)
@@ -838,11 +838,24 @@
 
         rffi._StrFinalizerQueue.print_debugging = True
         try:
-            xf = self.compile(f, [int], gcpolicy="incminimark")
-            assert xf(10000) == 42
+            xf = self.compile(f, [int], gcpolicy="incminimark",
+                              return_stderr=True)
         finally:
             rffi._StrFinalizerQueue.print_debugging = False
 
+        os.environ['PYPYLOG'] = ':-'
+        try:
+            error = xf(10000)
+        finally:
+            del os.environ['PYPYLOG']
+
+        import re
+        r = re.compile(r"freeing str [[] [0-9a-fx]+ []]")
+        matches = r.findall(error)
+        assert len(matches) == 10000        # must be all 10000 strings,
+        assert len(set(matches)) == 10000   # and no duplicates
+
+
 def test_enforced_args():
     from rpython.annotator.model import s_None
     from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator

From pypy.commits at gmail.com  Sun May 13 22:58:18 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 May 2018 19:58:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Separate W_StringBuilder and
 W_UnicodeBuilder
Message-ID: <5af8fb4a.1c69fb81.4751a.b3b8@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r94551:f5577200d77d
Date: 2018-05-14 03:31 +0100
http://bitbucket.org/pypy/pypy/changeset/f5577200d77d/

Log:	Separate W_StringBuilder and W_UnicodeBuilder

diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -6,56 +6,90 @@
 from rpython.tool.sourcetools import func_with_new_name
 
 
-def create_builder(name, strtype, builder_cls, newmethod):
-    if strtype is str:
-        unwrap = 'bytes'
-    else:
-        unwrap = unicode
-    class W_Builder(W_Root):
-        def __init__(self, space, size):
-            if size < 0:
-                self.builder = builder_cls()
-            else:
-                self.builder = builder_cls(size)
+class W_StringBuilder(W_Root):
+    def __init__(self, space, size):
+        if size < 0:
+            self.builder = StringBuilder()
+        else:
+            self.builder = StringBuilder(size)
 
-        @unwrap_spec(size=int)
-        def descr__new__(space, w_subtype, size=-1):
-            return W_Builder(space, size)
+    @unwrap_spec(size=int)
+    def descr__new__(space, w_subtype, size=-1):
+        return W_StringBuilder(space, size)
 
-        @unwrap_spec(s=unwrap)
-        def descr_append(self, space, s):
-            self.builder.append(s)
+    @unwrap_spec(s='bytes')
+    def descr_append(self, space, s):
+        self.builder.append(s)
 
-        @unwrap_spec(s=unwrap, start=int, end=int)
-        def descr_append_slice(self, space, s, start, end):
-            if not 0 <= start <= end <= len(s):
-                raise oefmt(space.w_ValueError, "bad start/stop")
-            self.builder.append_slice(s, start, end)
+    @unwrap_spec(s='bytes', start=int, end=int)
+    def descr_append_slice(self, space, s, start, end):
+        if not 0 <= start <= end <= len(s):
+            raise oefmt(space.w_ValueError, "bad start/stop")
+        self.builder.append_slice(s, start, end)
 
-        def descr_build(self, space):
-            w_s = getattr(space, newmethod)(self.builder.build())
-            # after build(), we can continue to append more strings
-            # to the same builder.  This is supported since
-            # 2ff5087aca28 in RPython.
-            return w_s
+    def descr_build(self, space):
+        w_s = space.newbytes(self.builder.build())
+        # after build(), we can continue to append more strings
+        # to the same builder.  This is supported since
+        # 2ff5087aca28 in RPython.
+        return w_s
 
-        def descr_len(self, space):
-            if self.builder is None:
-                raise oefmt(space.w_ValueError, "no length of built builder")
-            return space.newint(self.builder.getlength())
+    def descr_len(self, space):
+        if self.builder is None:
+            raise oefmt(space.w_ValueError, "no length of built builder")
+        return space.newint(self.builder.getlength())
 
-    W_Builder.__name__ = "W_%s" % name
-    W_Builder.typedef = TypeDef(name,
-        __new__ = interp2app(func_with_new_name(
-                                    W_Builder.descr__new__.im_func,
-                                    '%s_new' % (name,))),
-        append = interp2app(W_Builder.descr_append),
-        append_slice = interp2app(W_Builder.descr_append_slice),
-        build = interp2app(W_Builder.descr_build),
-        __len__ = interp2app(W_Builder.descr_len),
-    )
-    W_Builder.typedef.acceptable_as_base_class = False
-    return W_Builder
+W_StringBuilder.typedef = TypeDef("StringBuilder",
+    __new__ = interp2app(func_with_new_name(
+                                W_StringBuilder.descr__new__.im_func,
+                                'StringBuilder_new')),
+    append = interp2app(W_StringBuilder.descr_append),
+    append_slice = interp2app(W_StringBuilder.descr_append_slice),
+    build = interp2app(W_StringBuilder.descr_build),
+    __len__ = interp2app(W_StringBuilder.descr_len),
+)
+W_StringBuilder.typedef.acceptable_as_base_class = False
 
-W_StringBuilder = create_builder("StringBuilder", str, StringBuilder, "newbytes")
-W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, UnicodeBuilder, "newunicode")
+class W_UnicodeBuilder(W_Root):
+    def __init__(self, space, size):
+        if size < 0:
+            self.builder = UnicodeBuilder()
+        else:
+            self.builder = UnicodeBuilder(size)
+
+    @unwrap_spec(size=int)
+    def descr__new__(space, w_subtype, size=-1):
+        return W_UnicodeBuilder(space, size)
+
+    @unwrap_spec(s=unicode)
+    def descr_append(self, space, s):
+        self.builder.append(s)
+
+    @unwrap_spec(s=unicode, start=int, end=int)
+    def descr_append_slice(self, space, s, start, end):
+        if not 0 <= start <= end <= len(s):
+            raise oefmt(space.w_ValueError, "bad start/stop")
+        self.builder.append_slice(s, start, end)
+
+    def descr_build(self, space):
+        w_s = space.newunicode(self.builder.build())
+        # after build(), we can continue to append more strings
+        # to the same builder.  This is supported since
+        # 2ff5087aca28 in RPython.
+        return w_s
+
+    def descr_len(self, space):
+        if self.builder is None:
+            raise oefmt(space.w_ValueError, "no length of built builder")
+        return space.newint(self.builder.getlength())
+
+W_UnicodeBuilder.typedef = TypeDef("UnicodeBuilder",
+    __new__ = interp2app(func_with_new_name(
+                                W_UnicodeBuilder.descr__new__.im_func,
+                                'UnicodeBuilder_new')),
+    append = interp2app(W_UnicodeBuilder.descr_append),
+    append_slice = interp2app(W_UnicodeBuilder.descr_append_slice),
+    build = interp2app(W_UnicodeBuilder.descr_build),
+    __len__ = interp2app(W_UnicodeBuilder.descr_len),
+)
+W_UnicodeBuilder.typedef.acceptable_as_base_class = False

From pypy.commits at gmail.com  Sun May 13 22:58:20 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 May 2018 19:58:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Separate W_BytesBuilder and
 W_StringBuilder (manual port of f5577200d77d)
Message-ID: <5af8fb4c.1c69fb81.a0188.6930@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r94552:033ff20bdfd6
Date: 2018-05-14 03:53 +0100
http://bitbucket.org/pypy/pypy/changeset/033ff20bdfd6/

Log:	Separate W_BytesBuilder and W_StringBuilder (manual port of
	f5577200d77d)

diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -6,59 +6,90 @@
 from rpython.tool.sourcetools import func_with_new_name
 
 
-def create_builder(name, strtype, builder_cls):
-    if strtype is str:
-        unwrap = 'bytes'
-    else:
-        unwrap = unicode
-    class W_Builder(W_Root):
-        def __init__(self, space, size):
-            if size < 0:
-                self.builder = builder_cls()
-            else:
-                self.builder = builder_cls(size)
+class W_BytesBuilder(W_Root):
+    def __init__(self, space, size):
+        if size < 0:
+            self.builder = StringBuilder()
+        else:
+            self.builder = StringBuilder(size)
 
-        @unwrap_spec(size=int)
-        def descr__new__(space, w_subtype, size=-1):
-            return W_Builder(space, size)
+    @unwrap_spec(size=int)
+    def descr__new__(space, w_subtype, size=-1):
+        return W_BytesBuilder(space, size)
 
-        @unwrap_spec(s=unwrap)
-        def descr_append(self, space, s):
-            self.builder.append(s)
+    @unwrap_spec(s='bytes')
+    def descr_append(self, space, s):
+        self.builder.append(s)
 
-        @unwrap_spec(s=unwrap, start=int, end=int)
-        def descr_append_slice(self, space, s, start, end):
-            if not 0 <= start <= end <= len(s):
-                raise oefmt(space.w_ValueError, "bad start/stop")
-            self.builder.append_slice(s, start, end)
+    @unwrap_spec(s='bytes', start=int, end=int)
+    def descr_append_slice(self, space, s, start, end):
+        if not 0 <= start <= end <= len(s):
+            raise oefmt(space.w_ValueError, "bad start/stop")
+        self.builder.append_slice(s, start, end)
 
-        def descr_build(self, space):
-            s = self.builder.build()
-            # after build(), we can continue to append more strings
-            # to the same builder.  This is supported since
-            # 2ff5087aca28 in RPython.
-            if strtype is str:
-                return space.newbytes(s)
-            else:
-                return space.newunicode(s)
+    def descr_build(self, space):
+        s = self.builder.build()
+        # after build(), we can continue to append more strings
+        # to the same builder.  This is supported since
+        # 2ff5087aca28 in RPython.
+        return space.newbytes(s)
 
-        def descr_len(self, space):
-            if self.builder is None:
-                raise oefmt(space.w_ValueError, "no length of built builder")
-            return space.newint(self.builder.getlength())
+    def descr_len(self, space):
+        if self.builder is None:
+            raise oefmt(space.w_ValueError, "no length of built builder")
+        return space.newint(self.builder.getlength())
 
-    W_Builder.__name__ = "W_%s" % name
-    W_Builder.typedef = TypeDef(name,
-        __new__ = interp2app(func_with_new_name(
-                                    W_Builder.descr__new__.im_func,
-                                    '%s_new' % (name,))),
-        append = interp2app(W_Builder.descr_append),
-        append_slice = interp2app(W_Builder.descr_append_slice),
-        build = interp2app(W_Builder.descr_build),
-        __len__ = interp2app(W_Builder.descr_len),
-    )
-    W_Builder.typedef.acceptable_as_base_class = False
-    return W_Builder
+W_BytesBuilder.typedef = TypeDef('BytesBuilder',
+    __new__ = interp2app(func_with_new_name(
+                                W_BytesBuilder.descr__new__.im_func,
+                                'BytesBuilder_new')),
+    append = interp2app(W_BytesBuilder.descr_append),
+    append_slice = interp2app(W_BytesBuilder.descr_append_slice),
+    build = interp2app(W_BytesBuilder.descr_build),
+    __len__ = interp2app(W_BytesBuilder.descr_len),
+)
+W_BytesBuilder.typedef.acceptable_as_base_class = False
 
-W_StringBuilder = create_builder("StringBuilder", unicode, UnicodeBuilder)
-W_BytesBuilder = create_builder("BytesBuilder", str, StringBuilder)
+class W_StringBuilder(W_Root):
+    def __init__(self, space, size):
+        if size < 0:
+            self.builder = UnicodeBuilder()
+        else:
+            self.builder = UnicodeBuilder(size)
+
+    @unwrap_spec(size=int)
+    def descr__new__(space, w_subtype, size=-1):
+        return W_StringBuilder(space, size)
+
+    @unwrap_spec(s=unicode)
+    def descr_append(self, space, s):
+        self.builder.append(s)
+
+    @unwrap_spec(s=unicode, start=int, end=int)
+    def descr_append_slice(self, space, s, start, end):
+        if not 0 <= start <= end <= len(s):
+            raise oefmt(space.w_ValueError, "bad start/stop")
+        self.builder.append_slice(s, start, end)
+
+    def descr_build(self, space):
+        s = self.builder.build()
+        # after build(), we can continue to append more strings
+        # to the same builder.  This is supported since
+        # 2ff5087aca28 in RPython.
+        return space.newunicode(s)
+
+    def descr_len(self, space):
+        if self.builder is None:
+            raise oefmt(space.w_ValueError, "no length of built builder")
+        return space.newint(self.builder.getlength())
+
+W_StringBuilder.typedef = TypeDef('StringBuilder',
+    __new__ = interp2app(func_with_new_name(
+                                W_StringBuilder.descr__new__.im_func,
+                                'StringBuilder_new')),
+    append = interp2app(W_StringBuilder.descr_append),
+    append_slice = interp2app(W_StringBuilder.descr_append_slice),
+    build = interp2app(W_StringBuilder.descr_build),
+    __len__ = interp2app(W_StringBuilder.descr_len),
+)
+W_StringBuilder.typedef.acceptable_as_base_class = False

From pypy.commits at gmail.com  Sun May 13 22:58:22 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 May 2018 19:58:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5af8fb4e.1c69fb81.8eb55.56cc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r94553:d43acd762a29
Date: 2018-05-14 03:57 +0100
http://bitbucket.org/pypy/pypy/changeset/d43acd762a29/

Log:	hg merge default

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -430,7 +430,6 @@
             for const1 in switchdict.const_keys_in_order:
                 box = self.execute(rop.INT_EQ, valuebox, const1)
                 assert box.getint() == 0
-                target = switchdict.dict[const1.getint()]
                 self.metainterp.generate_guard(rop.GUARD_FALSE, box,
                                                resumepc=orgpc)
         else:

From pypy.commits at gmail.com  Sun May 13 23:29:26 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 May 2018 20:29:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: Use the same internal names for
 builders on py2 and py3
Message-ID: <5af90296.1c69fb81.d1ea4.1987@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r94554:dad86f083d4a
Date: 2018-05-14 04:17 +0100
http://bitbucket.org/pypy/pypy/changeset/dad86f083d4a/

Log:	Use the same internal names for builders on py2 and py3

diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -6,7 +6,7 @@
 from rpython.tool.sourcetools import func_with_new_name
 
 
-class W_StringBuilder(W_Root):
+class W_BytesBuilder(W_Root):
     def __init__(self, space, size):
         if size < 0:
             self.builder = StringBuilder()
@@ -15,7 +15,7 @@
 
     @unwrap_spec(size=int)
     def descr__new__(space, w_subtype, size=-1):
-        return W_StringBuilder(space, size)
+        return W_BytesBuilder(space, size)
 
     @unwrap_spec(s='bytes')
     def descr_append(self, space, s):
@@ -39,16 +39,17 @@
             raise oefmt(space.w_ValueError, "no length of built builder")
         return space.newint(self.builder.getlength())
 
-W_StringBuilder.typedef = TypeDef("StringBuilder",
+W_BytesBuilder.typedef = TypeDef("StringBuilder",
     __new__ = interp2app(func_with_new_name(
-                                W_StringBuilder.descr__new__.im_func,
-                                'StringBuilder_new')),
-    append = interp2app(W_StringBuilder.descr_append),
-    append_slice = interp2app(W_StringBuilder.descr_append_slice),
-    build = interp2app(W_StringBuilder.descr_build),
-    __len__ = interp2app(W_StringBuilder.descr_len),
+                                W_BytesBuilder.descr__new__.im_func,
+                                'BytesBuilder_new')),
+    append = interp2app(W_BytesBuilder.descr_append),
+    append_slice = interp2app(W_BytesBuilder.descr_append_slice),
+    build = interp2app(W_BytesBuilder.descr_build),
+    __len__ = interp2app(W_BytesBuilder.descr_len),
 )
-W_StringBuilder.typedef.acceptable_as_base_class = False
+W_BytesBuilder.typedef.acceptable_as_base_class = False
+W_StringBuilder = W_BytesBuilder
 
 class W_UnicodeBuilder(W_Root):
     def __init__(self, space, size):

From pypy.commits at gmail.com  Sun May 13 23:31:02 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 May 2018 20:31:02 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Use the same internal names for builders
 on py2 and py3
Message-ID: <5af902f6.1c69fb81.d4cfe.2f68@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r94555:f476c1168c34
Date: 2018-05-14 04:26 +0100
http://bitbucket.org/pypy/pypy/changeset/f476c1168c34/

Log:	Use the same internal names for builders on py2 and py3

diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -28,18 +28,18 @@
         self.builder.append_slice(s, start, end)
 
     def descr_build(self, space):
-        s = self.builder.build()
+        w_s = space.newbytes(self.builder.build())
         # after build(), we can continue to append more strings
         # to the same builder.  This is supported since
         # 2ff5087aca28 in RPython.
-        return space.newbytes(s)
+        return w_s
 
     def descr_len(self, space):
         if self.builder is None:
             raise oefmt(space.w_ValueError, "no length of built builder")
         return space.newint(self.builder.getlength())
 
-W_BytesBuilder.typedef = TypeDef('BytesBuilder',
+W_BytesBuilder.typedef = TypeDef("BytesBuilder",
     __new__ = interp2app(func_with_new_name(
                                 W_BytesBuilder.descr__new__.im_func,
                                 'BytesBuilder_new')),
@@ -50,7 +50,7 @@
 )
 W_BytesBuilder.typedef.acceptable_as_base_class = False
 
-class W_StringBuilder(W_Root):
+class W_UnicodeBuilder(W_Root):
     def __init__(self, space, size):
         if size < 0:
             self.builder = UnicodeBuilder()
@@ -59,7 +59,7 @@
 
     @unwrap_spec(size=int)
     def descr__new__(space, w_subtype, size=-1):
-        return W_StringBuilder(space, size)
+        return W_UnicodeBuilder(space, size)
 
     @unwrap_spec(s=unicode)
     def descr_append(self, space, s):
@@ -72,24 +72,25 @@
         self.builder.append_slice(s, start, end)
 
     def descr_build(self, space):
-        s = self.builder.build()
+        w_s = space.newunicode(self.builder.build())
         # after build(), we can continue to append more strings
         # to the same builder.  This is supported since
         # 2ff5087aca28 in RPython.
-        return space.newunicode(s)
+        return w_s
 
     def descr_len(self, space):
         if self.builder is None:
             raise oefmt(space.w_ValueError, "no length of built builder")
         return space.newint(self.builder.getlength())
 
-W_StringBuilder.typedef = TypeDef('StringBuilder',
+W_UnicodeBuilder.typedef = TypeDef("StringBuilder",
     __new__ = interp2app(func_with_new_name(
-                                W_StringBuilder.descr__new__.im_func,
-                                'StringBuilder_new')),
-    append = interp2app(W_StringBuilder.descr_append),
-    append_slice = interp2app(W_StringBuilder.descr_append_slice),
-    build = interp2app(W_StringBuilder.descr_build),
-    __len__ = interp2app(W_StringBuilder.descr_len),
+                                W_UnicodeBuilder.descr__new__.im_func,
+                                'UnicodeBuilder_new')),
+    append = interp2app(W_UnicodeBuilder.descr_append),
+    append_slice = interp2app(W_UnicodeBuilder.descr_append_slice),
+    build = interp2app(W_UnicodeBuilder.descr_build),
+    __len__ = interp2app(W_UnicodeBuilder.descr_len),
 )
-W_StringBuilder.typedef.acceptable_as_base_class = False
+W_UnicodeBuilder.typedef.acceptable_as_base_class = False
+W_StringBuilder = W_UnicodeBuilder

From pypy.commits at gmail.com  Sun May 13 23:31:05 2018
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 13 May 2018 20:31:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5af902f9.1c69fb81.5815a.0fb9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r94556:0c21724e75bd
Date: 2018-05-14 04:28 +0100
http://bitbucket.org/pypy/pypy/changeset/0c21724e75bd/

Log:	hg merge default


From pypy.commits at gmail.com  Mon May 14 03:22:48 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 00:22:48 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: Test fixes
Message-ID: <5af93948.1c69fb81.da13b.1fec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94557:2b804a3bfea0
Date: 2018-05-14 09:22 +0200
http://bitbucket.org/pypy/pypy/changeset/2b804a3bfea0/

Log:	Test fixes

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1237,6 +1237,18 @@
         raise NotImplementedError
     def op_revdb_call_destructor(self, *args):
         raise NotImplementedError
+    def op_revdb_strtod(self, *args):
+        raise NotImplementedError
+    def op_revdb_frexp(self, *args):
+        raise NotImplementedError
+    def op_revdb_modf(self, *args):
+        raise NotImplementedError
+    def op_revdb_dtoa(self, *args):
+        raise NotImplementedError
+    def op_revdb_do_next_call(self, *args):
+        raise NotImplementedError
+    def op_revdb_set_thread_breakpoint(self, *args):
+        raise NotImplementedError
 
 
 class Tracer(object):
diff --git a/rpython/rtyper/lltypesystem/test/test_ztranslated.py b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
--- a/rpython/rtyper/lltypesystem/test/test_ztranslated.py
+++ b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
@@ -65,7 +65,8 @@
 def test_compiled_semispace():
     fn = compile(main, [], gcpolicy="semispace")
     res = fn()
-    assert res == 42    # get_raw_address_of_string() raises ValueError
+    # get_raw_address_of_string() never raise ValueError any more
+    assert res == 0
 
 def test_compiled_boehm():
     fn = compile(main, [], gcpolicy="boehm")

From pypy.commits at gmail.com  Mon May 14 05:06:49 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 02:06:49 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: Try to reduce the diff
 from default
Message-ID: <5af951a9.1c69fb81.2da0a.0b85@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94558:ed5132246276
Date: 2018-05-14 11:06 +0200
http://bitbucket.org/pypy/pypy/changeset/ed5132246276/

Log:	Try to reduce the diff from default

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1204,12 +1204,13 @@
         self._compile_slice(sub.slice, sub.ctx)
 
     def visit_RevDBMetaVar(self, node):
-        if self.space.config.translation.reverse_debugger:
-            from pypy.interpreter.reverse_debugging import dbstate
+        if self.space.reverse_debugging:
+            dbstate = self.space.reverse_debugging.dbstate
             if not dbstate.standard_code:
                 self.emit_op_arg(ops.LOAD_REVDB_VAR, node.metavar)
                 return
-        self.error("$NUM is only valid in the reverse-debugger", node)
+        self.error("Unknown character ('$NUM' is only valid in the "
+                   "reverse-debugger)", node)
 
 
 class TopLevelCodeGenerator(PythonCodeGenerator):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -442,6 +442,16 @@
             config = get_pypy_config(translating=False)
         self.config = config
 
+        if self.config.translation.reverse_debugger:
+            # pre-import and attach to the space.  This avoids a regular
+            # translation seeing and executing the imports even if it
+            # turns out that self.config.translation.reverse_debugger is
+            # False.
+            from pypy.interpreter import reverse_debugging
+            self.reverse_debugging = reverse_debugging
+        else:
+            self.reverse_debugging = None
+
         self.builtin_modules = {}
         self.reloading_modules = {}
 
@@ -458,9 +468,8 @@
 
     def startup(self):
         # To be called before using the space
-        if self.config.translation.reverse_debugger:
-            from pypy.interpreter.reverse_debugging import setup_revdb
-            setup_revdb(self)
+        if self.reverse_debugging:
+            self.reverse_debugging.setup_revdb(self)
 
         self.threadlocals.enter_thread(self)
 
@@ -902,9 +911,8 @@
         #     if not space._side_effects_ok():
         #         don't cache.
         #
-        if self.config.translation.reverse_debugger:
-            from pypy.interpreter.reverse_debugging import dbstate
-            return dbstate.standard_code
+        if self.reverse_debugging:
+            return self.reverse_debugging.dbstate.standard_code
         return True
 
     def is_interned_str(self, s):
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -64,9 +64,8 @@
         return frame
 
     def enter(self, frame):
-        if self.space.config.translation.reverse_debugger:
-            from pypy.interpreter.reverse_debugging import enter_call
-            enter_call(self.topframeref(), frame)
+        if self.space.reverse_debugging:
+            self.space.reverse_debugging.enter_call(self.topframeref(), frame)
         frame.f_backref = self.topframeref
         self.topframeref = jit.virtual_ref(frame)
 
@@ -87,9 +86,9 @@
                 # be accessed also later
                 frame_vref()
             jit.virtual_ref_finish(frame_vref, frame)
-            if self.space.config.translation.reverse_debugger:
-                from pypy.interpreter.reverse_debugging import leave_call
-                leave_call(self.topframeref(), got_exception)
+            if self.space.reverse_debugging:
+                self.space.reverse_debugging.leave_call(self.topframeref(),
+                                                        got_exception)
 
     # ________________________________________________________________
 
@@ -159,9 +158,8 @@
         Like bytecode_trace() but doesn't invoke any other events besides the
         trace function.
         """
-        if self.space.config.translation.reverse_debugger:
-            from pypy.interpreter.reverse_debugging import potential_stop_point
-            potential_stop_point(frame)
+        if self.space.reverse_debugging:
+            self.space.reverse_debugging.potential_stop_point(frame)
         if (frame.get_w_f_trace() is None or self.is_tracing or
             self.gettrace() is None):
             return
@@ -519,7 +517,6 @@
     """The normal class for space.actionflag.  The signal module provides
     a different one."""
     _ticker = 0
-    _ticker_count = -1     # xxx only for reverse_debugger.py
 
     def get_ticker(self):
         return self._ticker
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1053,9 +1053,8 @@
         # this function is overridden by pypy.module.pypyjit.interp_jit
         check_nonneg(jumpto)
         #
-        if self.space.config.translation.reverse_debugger:
-            from pypy.interpreter.reverse_debugging import jump_backward
-            jump_backward(self, jumpto)
+        if self.space.reverse_debugging:
+            self.space.reverse_debugging.jump_backward(self, jumpto)
         #
         return jumpto
 
@@ -1311,9 +1310,8 @@
         self.space.setitem(w_dict, w_key, w_value)
 
     def LOAD_REVDB_VAR(self, oparg, next_instr):
-        if self.space.config.translation.reverse_debugger:
-            from pypy.interpreter.reverse_debugging import load_metavar
-            w_var = load_metavar(oparg)
+        if self.space.reverse_debugging:
+            w_var = self.space.reverse_debugging.load_metavar(oparg)
             self.pushvalue(w_var)
         else:
             self.MISSING_OPCODE(oparg, next_instr)
diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -7,7 +7,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter import gateway, typedef, pycode, pytraceback, pyframe
 from pypy.module.marshal import interp_marshal
-from pypy.interpreter.executioncontext import AbstractActionFlag
+from pypy.interpreter.executioncontext import AbstractActionFlag, ActionFlag
 
 
 class DBState:
@@ -222,10 +222,10 @@
     def __enter__(self):
         dbstate.standard_code = False
         self.t = dbstate.space.actionflag._ticker
-        self.c = dbstate.space.actionflag._ticker_count
+        self.c = dbstate.space.actionflag._ticker_revdb_count
     def __exit__(self, *args):
         dbstate.space.actionflag._ticker = self.t
-        dbstate.space.actionflag._ticker_count = self.c
+        dbstate.space.actionflag._ticker_revdb_count = self.c
         dbstate.standard_code = True
 non_standard_code = NonStandardCode()
 
@@ -802,6 +802,8 @@
 # ____________________________________________________________
 
 
+ActionFlag._ticker_revdb_count = -1
+
 class RDBSignalActionFlag(AbstractActionFlag):
     # Used instead of pypy.module.signal.interp_signal.SignalActionFlag
     # when we have reverse-debugging.  That other class would work too,
@@ -818,7 +820,7 @@
 
     _SIG_TICKER_COUNT = 100
     _ticker = 0
-    _ticker_count = _SIG_TICKER_COUNT * 10
+    _ticker_revdb_count = _SIG_TICKER_COUNT * 10
 
     def get_ticker(self):
         return self._ticker
@@ -831,10 +833,10 @@
 
     def decrement_ticker(self, by):
         if we_are_translated():
-            c = self._ticker_count - 1
+            c = self._ticker_revdb_count - 1
             if c < 0:
                 c = self._update_ticker_from_signals()
-            self._ticker_count = c
+            self._ticker_revdb_count = c
         #if self.has_bytecode_counter:    # this 'if' is constant-folded
         #    print ("RDBSignalActionFlag: has_bytecode_counter: "
         #           "not supported for now")
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -48,10 +48,10 @@
             return f.f_code
         assert g() is g.func_code
 
-    def test_f_trace_del(self): 
+    def test_f_trace_del(self):
         import sys
-        f = sys._getframe() 
-        del f.f_trace 
+        f = sys._getframe()
+        del f.f_trace
         assert f.f_trace is None
 
     def test_f_lineno(self):
@@ -120,7 +120,7 @@
         def f():
             assert sys._getframe().f_code.co_name == g()
         def g():
-            return sys._getframe().f_back.f_code.co_name 
+            return sys._getframe().f_back.f_code.co_name
         f()
 
     def test_f_back_virtualref(self):
@@ -237,7 +237,7 @@
     def test_trace_exc(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
@@ -302,7 +302,7 @@
     def test_trace_return_exc(self):
         import sys
         l = []
-        def trace(a,b,c): 
+        def trace(a,b,c):
             if b in ('exception', 'return'):
                 l.append((b, c))
             return trace
@@ -448,7 +448,7 @@
     def test_dont_trace_on_reraise(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
@@ -470,7 +470,7 @@
     def test_dont_trace_on_raise_with_tb(self):
         import sys
         l = []
-        def ltrace(a,b,c): 
+        def ltrace(a,b,c):
             if b == 'exception':
                 l.append(c)
             return ltrace
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -3,7 +3,7 @@
 from pypy.interpreter import gateway
 from rpython.rlib.jit import non_virtual_ref, vref_None
 
-class AppTestSlow:    
+class AppTestSlow:
     spaceconfig = dict(usemodules=['itertools'])
 
     def setup_class(cls):
@@ -64,7 +64,7 @@
     space.setitem(space.builtin.w_dict,
                   space.wrap('read_exc_type'),
                   space.wrap(read_exc_type_gw))
-    
+
 def _detach_helpers(space):
     space.delitem(space.builtin.w_dict,
                   space.wrap('hide_top_frame'))
@@ -92,7 +92,7 @@
         pckl = pickle.dumps(code)
         result = pickle.loads(pckl)
         assert code == result
-    
+
     def test_pickle_global_func(self):
         import new
         mod = new.module('mod')
@@ -109,7 +109,7 @@
             assert func is result
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_not_imported_module(self):
         import new
         mod = new.module('mod')
@@ -119,13 +119,13 @@
         result = pickle.loads(pckl)
         assert mod.__name__ == result.__name__
         assert mod.__dict__ == result.__dict__
-    
+
     def test_pickle_builtin_func(self):
         import pickle
         pckl = pickle.dumps(map)
         result = pickle.loads(pckl)
         assert map is result
-    
+
     def test_pickle_non_top_reachable_func(self):
         def func():
             return 42
@@ -142,7 +142,7 @@
         assert func.func_dict     == result.func_dict
         assert func.func_doc      == result.func_doc
         assert func.func_globals  == result.func_globals
-    
+
     def test_pickle_cell(self):
         def g():
             x = [42]
@@ -171,7 +171,7 @@
         f1     = f()
         saved = hide_top_frame(f1)
         pckl   = pickle.dumps(f1)
-        restore_top_frame(f1, saved) 
+        restore_top_frame(f1, saved)
         f2     = pickle.loads(pckl)
 
         assert type(f1) is type(f2)
@@ -223,7 +223,7 @@
         f1     = f()
         saved = hide_top_frame(f1)
         pckl   = pickle.dumps(f1)
-        restore_top_frame(f1, saved) 
+        restore_top_frame(f1, saved)
         f2     = pickle.loads(pckl)
 
     def test_frame_setstate_crash(self):
@@ -257,21 +257,21 @@
         pckl   = pickle.dumps(mod)
         result = pickle.loads(pckl)
         assert mod is result
-    
+
     def test_pickle_moduledict(self):
         import pickle
         moddict  = pickle.__dict__
         pckl     = pickle.dumps(moddict)
         result   = pickle.loads(pckl)
         assert moddict is result
-    
+
     def test_pickle_bltins_module(self):
         import pickle
         mod  = __builtins__
         pckl     = pickle.dumps(mod)
         result   = pickle.loads(pckl)
         assert mod is result
-    
+
     def test_pickle_buffer(self):
         skip("Can't pickle buffer objects on top of CPython either.  "
              "Do we really need it?")
@@ -280,14 +280,14 @@
         pckl     = pickle.dumps(a)
         result   = pickle.loads(pckl)
         assert a == result
-    
+
     def test_pickle_complex(self):
         import pickle
         a = complex(1.23,4.567)
         pckl     = pickle.dumps(a)
         result   = pickle.loads(pckl)
         assert a == result
-    
+
     def test_pickle_method(self):
         class myclass(object):
             def f(self):
@@ -308,7 +308,7 @@
             assert method() == result()
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_staticmethod(self):
         class myclass(object):
             def f():
@@ -319,7 +319,7 @@
         pckl     = pickle.dumps(method)
         result   = pickle.loads(pckl)
         assert method() == result()
-    
+
     def test_pickle_classmethod(self):
         class myclass(object):
             def f(cls):
@@ -337,7 +337,7 @@
             assert method() == result()
         finally:
             del sys.modules['mod']
-    
+
     def test_pickle_sequenceiter(self):
         '''
         In PyPy there is no distinction here between listiterator and
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -209,5 +209,4 @@
     return space.newbool(space._side_effects_ok())
 
 def revdb_stop(space):
-    from pypy.interpreter.reverse_debugging import stop_point
-    stop_point()
+    space.reverse_debugging.stop_point()
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -162,7 +162,7 @@
 def _setup(space):
     dn = setup_directory_structure(space)
     return space.appexec([space.wrap(dn)], """
-        (dn): 
+        (dn):
             import sys
             path = list(sys.path)
             sys.path.insert(0, dn)
@@ -1060,7 +1060,7 @@
 
         cpathname = udir.join('test.pyc')
         assert not cpathname.check()
-        
+
     def test_load_source_module_importerror(self):
         # the .pyc file is created before executing the module
         space = self.space
@@ -1169,11 +1169,11 @@
                     stream.close()
 
 
-def test_PYTHONPATH_takes_precedence(space): 
+def test_PYTHONPATH_takes_precedence(space):
     if sys.platform == "win32":
         py.test.skip("unresolved issues with win32 shell quoting rules")
-    from pypy.interpreter.test.test_zpy import pypypath 
-    extrapath = udir.ensure("pythonpath", dir=1) 
+    from pypy.interpreter.test.test_zpy import pypypath
+    extrapath = udir.ensure("pythonpath", dir=1)
     extrapath.join("sched.py").write("print 42\n")
     old = os.environ.get('PYTHONPATH', None)
     oldlang = os.environ.pop('LANG', None)
diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py
--- a/pypy/module/micronumpy/test/test_complex.py
+++ b/pypy/module/micronumpy/test/test_complex.py
@@ -494,8 +494,8 @@
         c = array([1.e+110, 1.e-110], dtype=complex128)
         d = floor_divide(c**2, c)
         assert (d == [1.e+110, 0]).all()
-        
-        
+
+
 
     def test_basic(self):
         import sys
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -372,8 +372,8 @@
         a = np.array(data, dtype=b)
         x = pickle.loads(pickle.dumps(a))
         assert (x == a).all()
-        assert x.dtype == a.dtype 
-        
+        assert x.dtype == a.dtype
+
     def test_index(self):
         import numpy as np
         for dtype in [np.int8, np.int16, np.int32, np.int64]:
@@ -1459,7 +1459,7 @@
                      "'offsets':[0,76800], "
                      "'itemsize':80000, "
                      "'aligned':True}")
-        
+
         assert dt == np.dtype(eval(str(dt)))
 
         dt = np.dtype({'names': ['r', 'g', 'b'], 'formats': ['u1', 'u1', 'u1'],
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1885,7 +1885,7 @@
         assert map(isnan, e) == [False, False, False, True, False]
         assert map(isinf, e) == [False, False, True, False, False]
         assert e.argmax() == 3
-        # numpy preserves value for uint16 -> cast_as_float16 -> 
+        # numpy preserves value for uint16 -> cast_as_float16 ->
         #     convert_to_float64 -> convert_to_float16 -> uint16
         #  even for float16 various float16 nans
         all_f16 = arange(0xfe00, 0xffff, dtype='uint16')
@@ -2615,7 +2615,7 @@
         a = np.arange(6).reshape(2,3)
         i = np.dtype('int32').type(0)
         assert (a[0] == a[i]).all()
-        
+
 
     def test_ellipsis_indexing(self):
         import numpy as np
diff --git a/pypy/module/micronumpy/test/test_object_arrays.py b/pypy/module/micronumpy/test/test_object_arrays.py
--- a/pypy/module/micronumpy/test/test_object_arrays.py
+++ b/pypy/module/micronumpy/test/test_object_arrays.py
@@ -200,7 +200,7 @@
         from numpy import arange, dtype
         from cPickle import loads, dumps
         import sys
-        
+
         a = arange(15).astype(object)
         if '__pypy__' in sys.builtin_module_names:
             raises(NotImplementedError, dumps, a)
@@ -211,4 +211,4 @@
         a = arange(15).astype(object).reshape((3, 5))
         b = loads(dumps(a))
         assert (a == b).all()
-        
+
diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py
--- a/pypy/module/signal/__init__.py
+++ b/pypy/module/signal/__init__.py
@@ -46,8 +46,8 @@
         space.check_signal_action = interp_signal.CheckSignalAction(space)
         space.actionflag.register_periodic_action(space.check_signal_action,
                                                   use_bytecode_counter=False)
-        if space.config.translation.reverse_debugger:
-            from pypy.interpreter.reverse_debugging import RDBSignalActionFlag
+        if space.reverse_debugging:
+            RDBSignalActionFlag = space.reverse_debugging.RDBSignalActionFlag
             space.actionflag.__class__ = RDBSignalActionFlag
         else:
             space.actionflag.__class__ = interp_signal.SignalActionFlag
diff --git a/pypy/objspace/test/test_binop_overriding.py b/pypy/objspace/test/test_binop_overriding.py
--- a/pypy/objspace/test/test_binop_overriding.py
+++ b/pypy/objspace/test/test_binop_overriding.py
@@ -73,7 +73,7 @@
                 if C is not object:
                     setattr(C, name, f)
                 override_in_hier(n-1)
-                if C is not object:        
+                if C is not object:
                     delattr(C, name)
 
         override_in_hier()
@@ -105,7 +105,7 @@
         if not self.appdirect:
             skip("slow test, should be run as appdirect test")
         Base, do_test = self.helpers
-         
+
         class X(Base):
             pass
         class Y(X):
@@ -116,7 +116,7 @@
         assert not fail
 
     def test_binop_combinations_sub(self):
-        Base, do_test = self.helpers        
+        Base, do_test = self.helpers
         class X(Base):
             pass
         class Y(X):
@@ -124,13 +124,13 @@
 
         fail = do_test(X, Y, 'sub', lambda x,y: x-y)
         #print len(fail)
-        assert not fail        
+        assert not fail
 
     def test_binop_combinations_pow(self):
         if not self.appdirect:
             skip("slow test, should be run as appdirect test")
         Base, do_test = self.helpers
-        
+
         class X(Base):
             pass
         class Y(X):
@@ -138,13 +138,13 @@
 
         fail = do_test(X, Y, 'pow', lambda x,y: x**y)
         #print len(fail)
-        assert not fail        
+        assert not fail
 
     def test_binop_combinations_more_exhaustive(self):
         if not self.appdirect:
             skip("very slow test, should be run as appdirect test")
         Base, do_test = self.helpers
-        
+
         class X(Base):
             pass
 
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -272,6 +272,9 @@
     def rewrite_op_unlikely(self, op):
         return None   # "no real effect"
 
+    def rewrite_op_revdb_do_next_call(self, op):
+        return []    # ignored, only for revdb
+
     def rewrite_op_raw_malloc_usage(self, op):
         if self.cpu.translate_support_code or isinstance(op.args[0], Variable):
             return   # the operation disappears
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -373,6 +373,18 @@
         hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, translator.config)
 
+def _import_revdb():
+    from rpython.rlib import revdb
+    return revdb
+
+def revdb_flag_io_disabled():
+    config = fetch_translated_config()
+    if config is not None and config.translation.reverse_debugger:
+        revdb = _import_revdb()
+        if revdb.flag_io_disabled():
+            return revdb
+    return None
+
 # ____________________________________________________________
 
 class FREED_OBJECT(object):
diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py
--- a/rpython/rlib/rdtoa.py
+++ b/rpython/rlib/rdtoa.py
@@ -3,7 +3,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator import cdir
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib import jit, revdb
+from rpython.rlib import jit, objectmodel
 from rpython.rlib.rstring import StringBuilder
 import py, sys
 
@@ -54,7 +54,8 @@
 def strtod(input):
     if len(input) > _INT_LIMIT:
         raise MemoryError
-    if revdb.flag_io_disabled():
+    revdb = objectmodel.revdb_flag_io_disabled()
+    if revdb:
         return revdb.emulate_strtod(input)
     end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
     try:
@@ -240,7 +241,8 @@
          special_strings=lower_special_strings, upper=False):
     if precision > _INT_LIMIT:
         raise MemoryError
-    if revdb.flag_io_disabled():
+    revdb = objectmodel.revdb_flag_io_disabled()
+    if revdb:
         return revdb.emulate_dtoa(value)
     decpt_ptr = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
     try:
diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -1,3 +1,4 @@
+XX X X
 import sys
 from rpython.rlib.objectmodel import we_are_translated, fetch_translated_config
 from rpython.rlib.objectmodel import specialize
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1211,12 +1211,11 @@
         exc_data.exc_value = lltype.typeOf(evalue)._defl()
         return bool(etype)
 
-    def op_revdb_stop_point(self, *args):
-        pass
-
     def op_gc_move_out_of_nursery(self, obj):
         raise NotImplementedError("gc_move_out_of_nursery")
 
+    def op_revdb_stop_point(self, *args):
+        pass
     def op_revdb_send_answer(self, *args):
         raise NotImplementedError
     def op_revdb_breakpoint(self, *args):
@@ -1225,10 +1224,8 @@
         raise NotImplementedError
     def op_revdb_get_unique_id(self, *args):
         raise NotImplementedError
-
     def op_revdb_watch_save_state(self, *args):
         return False
-
     def op_revdb_watch_restore_state(self, *args):
         raise NotImplementedError
     def op_revdb_weakref_create(self, *args):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -402,7 +402,7 @@
     'raw_store':            LLOp(revdb_protect=True, canrun=True),
     'bare_raw_store':       LLOp(revdb_protect=True),
     'gc_load_indexed':      LLOp(sideeffects=False, canrun=True),
-    'gc_store':             LLOp(canrun=True),
+    'gc_store':             LLOp(canrun=True),   # only used by the boehm gc
     'gc_store_indexed':     LLOp(canrun=True),
     'track_alloc_start':    LLOp(),
     'track_alloc_stop':     LLOp(),
diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py
--- a/rpython/rtyper/lltypesystem/module/ll_math.py
+++ b/rpython/rtyper/lltypesystem/module/ll_math.py
@@ -4,7 +4,7 @@
 import sys
 
 from rpython.translator import cdir
-from rpython.rlib import jit, rposix, revdb
+from rpython.rlib import jit, rposix, objectmodel
 from rpython.rlib.rfloat import INFINITY, NAN, isfinite
 from rpython.rlib.rposix import UNDERSCORE_ON_WIN32
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -185,7 +185,8 @@
         mantissa = x
         exponent = 0
     else:
-        if revdb.flag_io_disabled():
+        revdb = objectmodel.revdb_flag_io_disabled()
+        if revdb:
             return revdb.emulate_frexp(x)
         exp_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
         try:
@@ -223,7 +224,8 @@
 def ll_math_modf(x):
     # some platforms don't do the right thing for NaNs and
     # infinities, so we take care of special cases directly.
-    if revdb.flag_io_disabled():
+    revdb = objectmodel.revdb_flag_io_disabled()
+    if revdb:
         return revdb.emulate_modf(x)
     if not isfinite(x):
         if math.isnan(x):

From pypy.commits at gmail.com  Mon May 14 05:10:19 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 02:10:19 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: oops
Message-ID: <5af9527b.1c69fb81.f65a9.24b9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94559:57e6fc1e19e1
Date: 2018-05-14 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/57e6fc1e19e1/

Log:	oops

diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -1,4 +1,3 @@
-XX X X
 import sys
 from rpython.rlib.objectmodel import we_are_translated, fetch_translated_config
 from rpython.rlib.objectmodel import specialize

From pypy.commits at gmail.com  Mon May 14 05:11:15 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 02:11:15 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: Fix some tests
Message-ID: <5af952b3.1c69fb81.9d5a1.4075@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94560:2fe38255fdb8
Date: 2018-05-14 11:10 +0200
http://bitbucket.org/pypy/pypy/changeset/2fe38255fdb8/

Log:	Fix some tests

diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -606,7 +606,7 @@
     'revdb_dtoa':           LLOp(sideeffects=False),
     'revdb_modf':           LLOp(sideeffects=False),
     'revdb_frexp':          LLOp(sideeffects=False),
-    'revdb_do_next_call':   LLOp(),
+    'revdb_do_next_call':   LLOp(canrun=True),
 }
 # ***** Run test_lloperation after changes. *****
 
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -765,6 +765,9 @@
 def op_gc_move_out_of_nursery(obj):
     return obj
 
+def op_revdb_do_next_call():
+    pass
+
 # ____________________________________________________________
 
 def get_op_impl(opname):

From pypy.commits at gmail.com  Mon May 14 05:25:30 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 02:25:30 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: More diff reduction
Message-ID: <5af9560a.1c69fb81.1eef7.d364@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94561:4a2a458b4728
Date: 2018-05-14 11:24 +0200
http://bitbucket.org/pypy/pypy/changeset/4a2a458b4728/

Log:	More diff reduction

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -86,6 +86,8 @@
     w_buffer = W_TypeObject("buffer")
     w_type = W_TypeObject("type")
 
+    reverse_debugging = None
+
     def __init__(self, config=None):
         """NOT_RPYTHON"""
         self.fromcache = InternalSpaceCache(self).getorbuild
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -839,8 +839,11 @@
     def OP_DEBUG_PRINT(self, op):
         # XXX
         from rpython.rtyper.lltypesystem.rstr import STR
-        format = ['{%d} ']
-        argv = ['(int)getpid()']
+        format = []
+        argv = []
+        if self.db.reverse_debugger:
+            format.append('{%d} ')
+            argv.append('(int)getpid()')
         free_line = ""
         for arg in op.args:
             T = arg.concretetype
diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -465,16 +465,12 @@
         self.implementationtypename = db.gettype(
             T, varlength=self.getvarlength())
         parent, parentindex = parentlink(obj)
-        ## mangled = False
         if obj in exports.EXPORTS_obj2name:
             self.name = exports.EXPORTS_obj2name[obj]
             self.globalcontainer = 2    # meh
         elif parent is None:
             self.name = db.namespace.uniquename('g_' + self.basename())
             self.globalcontainer = True
-            ## if db.reverse_debugger and T._gckind != 'gc':
-            ##     from rpython.translator.revdb import gencsupp
-            ##     mangled = gencsupp.mangle_name_prebuilt_raw(db, self, T)
         else:
             self.globalcontainer = False
             parentnode = db.getcontainernode(parent)
@@ -482,7 +478,6 @@
             self.name = defnode.access_expr(parentnode.name, parentindex)
         if self.typename != self.implementationtypename:
             if db.gettypedefnode(T).extra_union_for_varlength:
-                 ## and not mangled:
                 self.name += '.b'
         self._funccodegen_owner = None
 
@@ -503,17 +498,14 @@
         return getattr(self.obj, self.eci_name, None)
 
     def get_declaration(self):
-        name = self.name
-        ## if name.startswith('RPY_RDB_A(') and name.endswith(')'):
-        ##     name = name[len('RPY_RDB_A('):-1]
-        if name[-2:] == '.b':
+        if self.name[-2:] == '.b':
             # xxx fish fish
             assert self.implementationtypename.startswith('struct ')
             assert self.implementationtypename.endswith(' @')
             uniontypename = 'union %su @' % self.implementationtypename[7:-2]
-            return uniontypename, name[:-2], True
+            return uniontypename, self.name[:-2], True
         else:
-            return self.implementationtypename, name, False
+            return self.implementationtypename, self.name, False
 
     def forward_declaration(self):
         if llgroup.member_of_group(self.obj):
@@ -774,14 +766,6 @@
             expr = db.get(value)
             if typeOf(value) is Void:
                 comma = ''
-            ## elif expr.startswith('(&RPY_RDB_A('):
-            ##     # can't use this in static initialization code if we
-            ##     # are inside a GC struct or a static_immutable struct.
-            ##     # (It is not needed inside other raw structs, but we
-            ##     # don't try to optimize that here.)
-            ##     assert db.reverse_debugger
-            ##     db.late_initializations.append(('%s' % access_expr, expr))
-            ##     expr = 'NULL /* patched later with %s */' % (expr,)
         expr += comma
         i = expr.find('\n')
         if i < 0:
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -208,9 +208,9 @@
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
             ('$(TARGET)', '$(OBJECTS)', ['$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)', '$(MAKE) postcompile BIN=$(TARGET)']),
-            ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) $(CFLAGS1) -o $@ -c $< $(INCLUDEDIRS)'),
-            ('%.o', '%.s', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) $(CFLAGS1) -o $@ -c $< $(INCLUDEDIRS)'),
-            ('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) $(CFLAGS1) -o $@ -c $< $(INCLUDEDIRS)'),
+            ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
+            ('%.o', '%.s', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
+            ('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 
         for rule in rules:

From pypy.commits at gmail.com  Mon May 14 05:38:51 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 02:38:51 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: Fix some tests that
 failed with "trying to change a frozen option object"
Message-ID: <5af9592b.1c69fb81.b2982.9e9d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94562:679c88232aae
Date: 2018-05-14 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/679c88232aae/

Log:	Fix some tests that failed with "trying to change a frozen option
	object"

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -373,18 +373,36 @@
         hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, translator.config)
 
-def _import_revdb():
-    from rpython.rlib import revdb
-    return revdb
 
 def revdb_flag_io_disabled():
-    config = fetch_translated_config()
-    if config is not None and config.translation.reverse_debugger:
-        revdb = _import_revdb()
-        if revdb.flag_io_disabled():
-            return revdb
+    revdb = _import_revdb()
+    if revdb and revdb.flag_io_disabled():
+        return revdb
     return None
 
+def _import_revdb():
+    "NOT_RPYTHON"
+    return None
+
+class Entry(ExtRegistryEntry):
+    _about_ = _import_revdb
+
+    def compute_result_annotation(self):
+        revdb = None
+        config = self.bookkeeper.annotator.translator.config
+        if config.translation.reverse_debugger:
+            from rpython.rlib import revdb
+        return self.bookkeeper.immutablevalue(revdb)
+
+    def specialize_call(self, hop):
+        from rpython.rtyper.lltypesystem import lltype
+        revdb = None
+        config = hop.rtyper.annotator.translator.config
+        if config.translation.reverse_debugger:
+            from rpython.rlib import revdb
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Void, revdb)
+
 # ____________________________________________________________
 
 class FREED_OBJECT(object):

From pypy.commits at gmail.com  Mon May 14 05:38:53 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 02:38:53 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: Fix
Message-ID: <5af9592d.1c69fb81.a2116.0cc7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94563:f872fdab3c20
Date: 2018-05-14 11:38 +0200
http://bitbucket.org/pypy/pypy/changeset/f872fdab3c20/

Log:	Fix

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1242,8 +1242,8 @@
         raise NotImplementedError
     def op_revdb_dtoa(self, *args):
         raise NotImplementedError
-    def op_revdb_do_next_call(self, *args):
-        raise NotImplementedError
+    def op_revdb_do_next_call(self):
+        pass
     def op_revdb_set_thread_breakpoint(self, *args):
         raise NotImplementedError
 

From pypy.commits at gmail.com  Mon May 14 06:21:58 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 03:21:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: Make this a pointer instead of a
 Py_ssize_t. Why not, and reduces the
Message-ID: <5af96346.1c69fb81.41354.0983@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94564:0088ece5635b
Date: 2018-05-14 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/0088ece5635b/

Log:	Make this a pointer instead of a Py_ssize_t. Why not, and reduces
	the diff from reverse-debugger.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1164,8 +1164,8 @@
         '_PyPy_tuple_dealloc', [PyObject], lltype.Void,
         compilation_info=eci, _nowrapper=True)
     _, state.C.set_marker = rffi.CExternVariable(
-                   Py_ssize_t, '_pypy_rawrefcount_w_marker_deallocating',
-                   eci, _nowrapper=True, c_type='Py_ssize_t')
+                   rffi.VOIDP, '_pypy_rawrefcount_w_marker_deallocating',
+                   eci, _nowrapper=True, c_type='void *')
     state.C._PyPy_subtype_dealloc = rffi.llexternal(
         '_PyPy_subtype_dealloc', [PyObject], lltype.Void,
         compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -49,7 +49,7 @@
 
 PyAPI_FUNC(void) Py_IncRef(PyObject *);
 PyAPI_FUNC(void) Py_DecRef(PyObject *);
-extern Py_ssize_t _pypy_rawrefcount_w_marker_deallocating;
+extern void *_pypy_rawrefcount_w_marker_deallocating;
 PyAPI_FUNC(void) _Py_Dealloc(PyObject *);
 
 #define Py_CLEAR(op)                        \
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -409,7 +409,7 @@
     if we_are_translated():
         llptr = cast_instance_to_base_ptr(w_marker_deallocating)
         state = space.fromcache(State)
-        state.C.set_marker(rffi.cast(Py_ssize_t, llptr))
+        state.C.set_marker(llptr)
 
 @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL)
 def _Py_HashPointer(space, ptr):
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -14,14 +14,14 @@
  * tests we cannot call set_marker(), so we need to set a special value
  * directly here)
  */
-Py_ssize_t _pypy_rawrefcount_w_marker_deallocating = 0xDEADFFF;
+void* _pypy_rawrefcount_w_marker_deallocating = (void*) 0xDEADFFF;
 
 void
 _Py_Dealloc(PyObject *obj)
 {
     PyTypeObject *pto = obj->ob_type;
     /* this is the same as rawrefcount.mark_deallocating() */
-    obj->ob_pypy_link = _pypy_rawrefcount_w_marker_deallocating;
+    obj->ob_pypy_link = (Py_ssize_t)_pypy_rawrefcount_w_marker_deallocating;
     pto->tp_dealloc(obj);
 }
 

From pypy.commits at gmail.com  Mon May 14 06:24:30 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 03:24:30 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: RPython fix: we can't
 pass around module objects, they must be
Message-ID: <5af963de.1c69fb81.de124.73d7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94565:5fc00945551a
Date: 2018-05-14 12:23 +0200
http://bitbucket.org/pypy/pypy/changeset/5fc00945551a/

Log:	RPython fix: we can't pass around module objects, they must be fully
	resolved by the flow graph

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1203,12 +1203,17 @@
             sub.value.walkabout(self)
         self._compile_slice(sub.slice, sub.ctx)
 
+    def _revdb_metavar(self, node):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import dbstate
+        if not dbstate.standard_code:
+            self.emit_op_arg(ops.LOAD_REVDB_VAR, node.metavar)
+            return True
+        return False
+
     def visit_RevDBMetaVar(self, node):
-        if self.space.reverse_debugging:
-            dbstate = self.space.reverse_debugging.dbstate
-            if not dbstate.standard_code:
-                self.emit_op_arg(ops.LOAD_REVDB_VAR, node.metavar)
-                return
+        if self.space.reverse_debugging and self._revdb_metavar(node):
+            return
         self.error("Unknown character ('$NUM' is only valid in the "
                    "reverse-debugger)", node)
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -430,6 +430,8 @@
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
 
+    reverse_debugging = False
+
     @not_rpython
     def __init__(self, config=None):
         "Basic initialization of objects."
@@ -441,16 +443,7 @@
             from pypy.config.pypyoption import get_pypy_config
             config = get_pypy_config(translating=False)
         self.config = config
-
-        if self.config.translation.reverse_debugger:
-            # pre-import and attach to the space.  This avoids a regular
-            # translation seeing and executing the imports even if it
-            # turns out that self.config.translation.reverse_debugger is
-            # False.
-            from pypy.interpreter import reverse_debugging
-            self.reverse_debugging = reverse_debugging
-        else:
-            self.reverse_debugging = None
+        self.reverse_debugging = config.translation.reverse_debugger
 
         self.builtin_modules = {}
         self.reloading_modules = {}
@@ -469,7 +462,7 @@
     def startup(self):
         # To be called before using the space
         if self.reverse_debugging:
-            self.reverse_debugging.setup_revdb(self)
+            self._revdb_startup()
 
         self.threadlocals.enter_thread(self)
 
@@ -896,6 +889,16 @@
                 self.interned_strings.set(s, w_s1)
         return w_s1
 
+    def _revdb_startup(self):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import setup_revdb
+        setup_revdb(self)
+
+    def _revdb_standard_code(self):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import dbstate
+        return dbstate.standard_code
+
     def _side_effects_ok(self):
         # For the reverse debugger: we run compiled watchpoint
         # expressions in a fast way that will crash if they have
@@ -912,7 +915,7 @@
         #         don't cache.
         #
         if self.reverse_debugging:
-            return self.reverse_debugging.dbstate.standard_code
+            return self._revdb_standard_code()
         return True
 
     def is_interned_str(self, s):
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -65,7 +65,7 @@
 
     def enter(self, frame):
         if self.space.reverse_debugging:
-            self.space.reverse_debugging.enter_call(self.topframeref(), frame)
+            self._revdb_enter(frame)
         frame.f_backref = self.topframeref
         self.topframeref = jit.virtual_ref(frame)
 
@@ -87,8 +87,7 @@
                 frame_vref()
             jit.virtual_ref_finish(frame_vref, frame)
             if self.space.reverse_debugging:
-                self.space.reverse_debugging.leave_call(self.topframeref(),
-                                                        got_exception)
+                self._revdb_leave(got_exception)
 
     # ________________________________________________________________
 
@@ -159,7 +158,7 @@
         trace function.
         """
         if self.space.reverse_debugging:
-            self.space.reverse_debugging.potential_stop_point(frame)
+            self._revdb_potential_stop_point(frame)
         if (frame.get_w_f_trace() is None or self.is_tracing or
             self.gettrace() is None):
             return
@@ -392,6 +391,21 @@
         if self.space.check_signal_action is not None:
             self.space.check_signal_action.perform(self, None)
 
+    def _revdb_enter(self, frame):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import enter_call
+        enter_call(self.topframeref(), frame)
+
+    def _revdb_leave(self, got_exception):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import leave_call
+        leave_call(self.topframeref(), got_exception)
+
+    def _revdb_potential_stop_point(self, frame):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import potential_stop_point
+        potential_stop_point(frame)
+
     def _freeze_(self):
         raise Exception("ExecutionContext instances should not be seen during"
                         " translation.  Now is a good time to inspect the"
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1049,13 +1049,16 @@
     def YIELD_VALUE(self, oparg, next_instr):
         raise Yield
 
+    def _revdb_jump_backward(self, jumpto):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import jump_backward
+        jump_backward(self, jumpto)
+
     def jump_absolute(self, jumpto, ec):
         # this function is overridden by pypy.module.pypyjit.interp_jit
         check_nonneg(jumpto)
-        #
         if self.space.reverse_debugging:
-            self.space.reverse_debugging.jump_backward(self, jumpto)
-        #
+            self._revdb_jump_backward(jumpto)
         return jumpto
 
     def JUMP_FORWARD(self, jumpby, next_instr):
@@ -1309,10 +1312,15 @@
         w_dict = self.peekvalue()
         self.space.setitem(w_dict, w_key, w_value)
 
+    def _revdb_load_var(self, oparg):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import load_metavar
+        w_var = load_metavar(oparg)
+        self.pushvalue(w_var)
+
     def LOAD_REVDB_VAR(self, oparg, next_instr):
         if self.space.reverse_debugging:
-            w_var = self.space.reverse_debugging.load_metavar(oparg)
-            self.pushvalue(w_var)
+            self._revdb_load_var(oparg)
         else:
             self.MISSING_OPCODE(oparg, next_instr)
 
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -209,4 +209,5 @@
     return space.newbool(space._side_effects_ok())
 
 def revdb_stop(space):
-    space.reverse_debugging.stop_point()
+    from pypy.interpreter.reverse_debugging import stop_point
+    stop_point()
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -86,8 +86,6 @@
     w_buffer = W_TypeObject("buffer")
     w_type = W_TypeObject("type")
 
-    reverse_debugging = None
-
     def __init__(self, config=None):
         """NOT_RPYTHON"""
         self.fromcache = InternalSpaceCache(self).getorbuild
diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py
--- a/pypy/module/signal/__init__.py
+++ b/pypy/module/signal/__init__.py
@@ -47,7 +47,7 @@
         space.actionflag.register_periodic_action(space.check_signal_action,
                                                   use_bytecode_counter=False)
         if space.reverse_debugging:
-            RDBSignalActionFlag = space.reverse_debugging.RDBSignalActionFlag
+            from pypy.interpreter.reverse_debugging import RDBSignalActionFlag
             space.actionflag.__class__ = RDBSignalActionFlag
         else:
             space.actionflag.__class__ = interp_signal.SignalActionFlag
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -375,33 +375,34 @@
 
 
 def revdb_flag_io_disabled():
-    revdb = _import_revdb()
-    if revdb and revdb.flag_io_disabled():
-        return revdb
-    return None
+    if not revdb_enabled():
+        return False
+    return _revdb_flag_io_disabled()
 
-def _import_revdb():
-    "NOT_RPYTHON"
-    return None
+def _revdb_flag_io_disabled():
+    # moved in its own function for the import statement
+    from rpython.rlib import revdb
+    return revdb.flag_io_disabled()
+
+ at not_rpython
+def revdb_enabled():
+    return False
 
 class Entry(ExtRegistryEntry):
-    _about_ = _import_revdb
+    _about_ = revdb_enabled
 
     def compute_result_annotation(self):
-        revdb = None
+        from rpython.annotator import model as annmodel
         config = self.bookkeeper.annotator.translator.config
         if config.translation.reverse_debugger:
-            from rpython.rlib import revdb
-        return self.bookkeeper.immutablevalue(revdb)
+            return annmodel.s_True
+        else:
+            return annmodel.s_False
 
     def specialize_call(self, hop):
         from rpython.rtyper.lltypesystem import lltype
-        revdb = None
-        config = hop.rtyper.annotator.translator.config
-        if config.translation.reverse_debugger:
-            from rpython.rlib import revdb
         hop.exception_cannot_occur()
-        return hop.inputconst(lltype.Void, revdb)
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
 
diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py
--- a/rpython/rlib/rdtoa.py
+++ b/rpython/rlib/rdtoa.py
@@ -54,9 +54,8 @@
 def strtod(input):
     if len(input) > _INT_LIMIT:
         raise MemoryError
-    revdb = objectmodel.revdb_flag_io_disabled()
-    if revdb:
-        return revdb.emulate_strtod(input)
+    if objectmodel.revdb_flag_io_disabled():
+        return _revdb_strtod(input)
     end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
     try:
         # note: don't use the class scoped_view_charp here, it
@@ -241,9 +240,8 @@
          special_strings=lower_special_strings, upper=False):
     if precision > _INT_LIMIT:
         raise MemoryError
-    revdb = objectmodel.revdb_flag_io_disabled()
-    if revdb:
-        return revdb.emulate_dtoa(value)
+    if objectmodel.revdb_flag_io_disabled():
+        return _revdb_dtoa(value)
     decpt_ptr = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
     try:
         sign_ptr = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
@@ -307,3 +305,13 @@
 
     return dtoa(value, code, mode=mode, precision=precision, flags=flags,
                 special_strings=special_strings, upper=upper)
+
+def _revdb_strtod(input):
+    # moved in its own function for the import statement
+    from rpython.rlib import revdb
+    return revdb.emulate_strtod(input)
+
+def _revdb_dtoa(value):
+    # moved in its own function for the import statement
+    from rpython.rlib import revdb
+    return revdb.emulate_dtoa(value)
diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py
--- a/rpython/rtyper/lltypesystem/module/ll_math.py
+++ b/rpython/rtyper/lltypesystem/module/ll_math.py
@@ -185,9 +185,8 @@
         mantissa = x
         exponent = 0
     else:
-        revdb = objectmodel.revdb_flag_io_disabled()
-        if revdb:
-            return revdb.emulate_frexp(x)
+        if objectmodel.revdb_flag_io_disabled():
+            return _revdb_frexp(x)
         exp_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
         try:
             mantissa = math_frexp(x, exp_p)
@@ -224,9 +223,8 @@
 def ll_math_modf(x):
     # some platforms don't do the right thing for NaNs and
     # infinities, so we take care of special cases directly.
-    revdb = objectmodel.revdb_flag_io_disabled()
-    if revdb:
-        return revdb.emulate_modf(x)
+    if objectmodel.revdb_flag_io_disabled():
+        return _revdb_modf(x)
     if not isfinite(x):
         if math.isnan(x):
             return (x, x)
@@ -413,6 +411,18 @@
 
     return func_with_new_name(ll_math, 'll_math_' + name)
 
+
+def _revdb_frexp(x):
+    # moved in its own function for the import statement
+    from rpython.rlib import revdb
+    return revdb.emulate_frexp(x)
+
+def _revdb_modf(x):
+    # moved in its own function for the import statement
+    from rpython.rlib import revdb
+    return revdb.emulate_modf(x)
+
+
 # ____________________________________________________________
 
 unary_math_functions = [

From pypy.commits at gmail.com  Mon May 14 07:01:03 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 04:01:03 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: Fix test
Message-ID: <5af96c6f.1c69fb81.a348f.6b23@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94566:3c84fc6a2df5
Date: 2018-05-14 13:00 +0200
http://bitbucket.org/pypy/pypy/changeset/3c84fc6a2df5/

Log:	Fix test

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -956,12 +956,16 @@
     def test_revdb_metavar(self):
         from pypy.interpreter.reverse_debugging import dbstate, setup_revdb
         self.space.config.translation.reverse_debugger = True
-        setup_revdb(self.space)
-        dbstate.standard_code = False
-        dbstate.metavars = [self.space.wrap(6)]
-        self.simple_test("x = 7*$0", "x", 42)
-        dbstate.standard_code = True
-        self.error_test("x = 7*$0", SyntaxError)
+        self.space.reverse_debugging = True
+        try:
+            setup_revdb(self.space)
+            dbstate.standard_code = False
+            dbstate.metavars = [self.space.wrap(6)]
+            self.simple_test("x = 7*$0", "x", 42)
+            dbstate.standard_code = True
+            self.error_test("x = 7*$0", SyntaxError)
+        finally:
+            self.space.reverse_debugging = False
 
 
 class AppTestCompiler:

From pypy.commits at gmail.com  Mon May 14 07:02:59 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 04:02:59 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger-updated: Close branch
Message-ID: <5af96ce3.1c69fb81.30955.e118@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger-updated
Changeset: r94567:883b423d5d72
Date: 2018-05-14 13:02 +0200
http://bitbucket.org/pypy/pypy/changeset/883b423d5d72/

Log:	Close branch


From pypy.commits at gmail.com  Mon May 14 07:10:23 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 14 May 2018 04:10:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix distutils.sysconfig
Message-ID: <5af96e9f.1c69fb81.fa01c.caa2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94569:28c3a0a86d50
Date: 2018-05-14 14:09 +0300
http://bitbucket.org/pypy/pypy/changeset/28c3a0a86d50/

Log:	fix distutils.sysconfig

diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -65,7 +65,7 @@
     """Initialize the module as appropriate for POSIX systems."""
     from _sysconfigdata import build_time_vars
     global _config_vars
-    _config_vars.update(build_time_vars)
+    _config_vars = build_time_vars
 
 
 def _init_nt():

From pypy.commits at gmail.com  Mon May 14 08:00:03 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 14 May 2018 05:00:03 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix test for python3,
 remove implemented functions from stubs.py
Message-ID: <5af97a43.1c69fb81.88878.6a8a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94570:a8e6b2583cbc
Date: 2018-05-14 14:39 +0300
http://bitbucket.org/pypy/pypy/changeset/a8e6b2583cbc/

Log:	fix test for python3, remove implemented functions from stubs.py

diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1261,13 +1261,6 @@
     version indicates the file format."""
     raise NotImplementedError
 
- at cpython_api([PyObject, rffi.INT_real], PyObject)
-def PyMarshal_WriteObjectToString(space, value, version):
-    """Return a string object containing the marshalled representation of value.
-
-    version indicates the file format."""
-    raise NotImplementedError
-
 @cpython_api([FILE], lltype.Signed, error=CANNOT_FAIL)
 def PyMarshal_ReadLongFromFile(space, file):
     """Return a C long from the data stream in a FILE* opened
@@ -1301,17 +1294,6 @@
     (EOFError or TypeError) and returns NULL."""
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject)
-def PyMarshal_ReadObjectFromString(space, string, len):
-    """Return a Python object from the data stream in a character buffer
-    containing len bytes pointed to by string.  On error, sets the
-    appropriate exception (EOFError or TypeError) and returns
-    NULL.
-
-    This function used an int type for len. This might require
-    changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
 def PyMethod_ClearFreeList(space):
     """Clear the free list. Return the total number of freed items.
diff --git a/pypy/module/cpyext/test/test_marshal.py b/pypy/module/cpyext/test/test_marshal.py
--- a/pypy/module/cpyext/test/test_marshal.py
+++ b/pypy/module/cpyext/test/test_marshal.py
@@ -6,8 +6,8 @@
         module = self.import_extension('foo', [
             ("mloads", "METH_O",
              """
-                 char *input = PyString_AsString(args);
-                 Py_ssize_t length = PyString_Size(args);
+                 char *input = PyBytes_AsString(args);
+                 Py_ssize_t length = PyBytes_Size(args);
                  return PyMarshal_ReadObjectFromString(input, length);
              """)],
             prologue='#include <marshal.h>')

From pypy.commits at gmail.com  Mon May 14 08:00:05 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 14 May 2018 05:00:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge default into py3.5
Message-ID: <5af97a45.1c69fb81.935c6.2665@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94571:823fbc19cb92
Date: 2018-05-14 14:56 +0300
http://bitbucket.org/pypy/pypy/changeset/823fbc19cb92/

Log:	merge default into py3.5

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1176,8 +1176,8 @@
         '_PyPy_tuple_dealloc', [PyObject], lltype.Void,
         compilation_info=eci, _nowrapper=True)
     _, state.C.set_marker = rffi.CExternVariable(
-                   Py_ssize_t, '_pypy_rawrefcount_w_marker_deallocating',
-                   eci, _nowrapper=True, c_type='Py_ssize_t')
+                   rffi.VOIDP, '_pypy_rawrefcount_w_marker_deallocating',
+                   eci, _nowrapper=True, c_type='void *')
     state.C._PyPy_subtype_dealloc = rffi.llexternal(
         '_PyPy_subtype_dealloc', [PyObject], lltype.Void,
         compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -61,7 +61,7 @@
 #endif
 PyAPI_FUNC(void) Py_IncRef(PyObject *);
 PyAPI_FUNC(void) Py_DecRef(PyObject *);
-extern Py_ssize_t _pypy_rawrefcount_w_marker_deallocating;
+extern void *_pypy_rawrefcount_w_marker_deallocating;
 PyAPI_FUNC(void) _Py_Dealloc(PyObject *);
 
 
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -410,7 +410,7 @@
     if we_are_translated():
         llptr = cast_instance_to_base_ptr(w_marker_deallocating)
         state = space.fromcache(State)
-        state.C.set_marker(rffi.cast(Py_ssize_t, llptr))
+        state.C.set_marker(llptr)
 
 @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL)
 def _Py_HashPointer(space, ptr):
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -14,14 +14,14 @@
  * tests we cannot call set_marker(), so we need to set a special value
  * directly here)
  */
-Py_ssize_t _pypy_rawrefcount_w_marker_deallocating = 0xDEADFFF;
+void* _pypy_rawrefcount_w_marker_deallocating = (void*) 0xDEADFFF;
 
 void
 _Py_Dealloc(PyObject *obj)
 {
     PyTypeObject *pto = obj->ob_type;
     /* this is the same as rawrefcount.mark_deallocating() */
-    obj->ob_pypy_link = _pypy_rawrefcount_w_marker_deallocating;
+    obj->ob_pypy_link = (Py_ssize_t)_pypy_rawrefcount_w_marker_deallocating;
     pto->tp_dealloc(obj);
 }
 
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1222,13 +1222,6 @@
     version indicates the file format."""
     raise NotImplementedError
 
- at cpython_api([PyObject, rffi.INT_real], PyObject)
-def PyMarshal_WriteObjectToString(space, value, version):
-    """Return a string object containing the marshalled representation of value.
-
-    version indicates the file format."""
-    raise NotImplementedError
-
 @cpython_api([FILE], lltype.Signed, error=-1)
 def PyMarshal_ReadLongFromFile(space, file):
     """Return a C long from the data stream in a FILE* opened
@@ -1262,14 +1255,6 @@
     (EOFError or TypeError) and returns NULL."""
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP, Py_ssize_t], PyObject)
-def PyMarshal_ReadObjectFromString(space, string, len):
-    """Return a Python object from the data stream in a character buffer
-    containing len bytes pointed to by string.  On error, sets the
-    appropriate exception (EOFError or TypeError) and returns
-    NULL."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, rffi.INT_real, lltype.Char], PyObject)
 def PyMemoryView_GetContiguous(space, obj, buffertype, order):
     """Create a memoryview object to a contiguous chunk of memory (in either
diff --git a/pypy/module/cpyext/test/test_marshal.py b/pypy/module/cpyext/test/test_marshal.py
--- a/pypy/module/cpyext/test/test_marshal.py
+++ b/pypy/module/cpyext/test/test_marshal.py
@@ -6,8 +6,8 @@
         module = self.import_extension('foo', [
             ("mloads", "METH_O",
              """
-                 char *input = PyString_AsString(args);
-                 Py_ssize_t length = PyString_Size(args);
+                 char *input = PyBytes_AsString(args);
+                 Py_ssize_t length = PyBytes_Size(args);
                  return PyMarshal_ReadObjectFromString(input, length);
              """)],
             prologue='#include <marshal.h>')

From pypy.commits at gmail.com  Mon May 14 08:02:29 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 05:02:29 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Shouldn't catch an exception
 here, because it remains visible to
Message-ID: <5af97ad5.1c69fb81.81364.921a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r94572:2733778b0c3d
Date: 2018-05-14 14:01 +0200
http://bitbucket.org/pypy/pypy/changeset/2733778b0c3d/

Log:	Shouldn't catch an exception here, because it remains visible to
	user code as the last exception

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -83,10 +83,7 @@
     sys.excepthook(), catching SystemExit, printing a newline after
     sys.stdout if needed, etc.
     """
-    try:
-        from __pypy__ import revdb_stop
-    except ImportError:
-        revdb_stop = None
+    from __pypy__ import revdb_stop
     try:
         # run it
         try:
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -147,5 +147,6 @@
             self.extra_interpdef('jit_backend_features',
                                     'space.wrap(%r)' % features)
         if self.space.config.translation.reverse_debugger:
-            self.extra_interpdef('revdb_stop',
-                                 'interp_magic.revdb_stop')
+            self.extra_interpdef('revdb_stop', 'interp_magic.revdb_stop')
+        else:
+            self.extra_interpdef('revdb_stop', 'space.w_None')

From pypy.commits at gmail.com  Mon May 14 08:04:14 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 05:04:14 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Don't explode if 'hypothesis'
 cannot be imported
Message-ID: <5af97b3e.1c69fb81.54326.8c18@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r94573:e74b65718f40
Date: 2018-05-14 14:03 +0200
http://bitbucket.org/pypy/pypy/changeset/e74b65718f40/

Log:	Don't explode if 'hypothesis' cannot be imported

diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py
--- a/pypy/interpreter/test/test_reverse_debugging.py
+++ b/pypy/interpreter/test/test_reverse_debugging.py
@@ -2,7 +2,6 @@
 from pypy.interpreter.reverse_debugging import *
 from pypy.interpreter import reverse_debugging
 from rpython.rlib import revdb
-from hypothesis import given, strategies, example
 
 
 class FakeCode:
@@ -15,28 +14,34 @@
         self.co_filename = co_filename
 
 
- at given(strategies.binary())
- at example("\x01\x02\x03\x04"
-         "\x00\xFF\x20\x30\x00\xFF\x00\x40"
-         "\xFF\x00\x0A\x0B\xFF\x00\x0C\x00")
-def test_build_co_revdb_linestarts(lnotab):
-    if len(lnotab) & 1:
-        lnotab = lnotab + '\x00'   # make the length even
-    code = FakeCode("?" * sum(map(ord, lnotab[0::2])), lnotab)
-    lstart = build_co_revdb_linestarts(code)
-    assert lstart is code.co_revdb_linestarts
+try:
+    from hypothesis import given, strategies, example
+except ImportError:
+    pass
+else:
+    @given(strategies.binary())
+    @example("\x01\x02\x03\x04"
+             "\x00\xFF\x20\x30\x00\xFF\x00\x40"
+             "\xFF\x00\x0A\x0B\xFF\x00\x0C\x00")
+    def test_build_co_revdb_linestarts(lnotab):
+        if len(lnotab) & 1:
+            lnotab = lnotab + '\x00'   # make the length even
+        code = FakeCode("?" * sum(map(ord, lnotab[0::2])), lnotab)
+        lstart = build_co_revdb_linestarts(code)
+        assert lstart is code.co_revdb_linestarts
 
-    expected_starts = set()
-    for addr, lineno in dis.findlinestarts(code):
-        expected_starts.add(addr)
+        expected_starts = set()
+        for addr, lineno in dis.findlinestarts(code):
+            expected_starts.add(addr)
 
-    next_start = len(code.co_code)
-    for index in range(len(code.co_code), -1, -1):
-        if index in expected_starts:
-            next_start = index
-        assert lstart[index] == chr(next_start - index
-                                    if next_start - index <= 255
-                                    else 255)
+        next_start = len(code.co_code)
+        for index in range(len(code.co_code), -1, -1):
+            if index in expected_starts:
+                next_start = index
+            assert lstart[index] == chr(next_start - index
+                                        if next_start - index <= 255
+                                        else 255)
+
 
 class FakeFrame:
     def __init__(self, code):

From pypy.commits at gmail.com  Mon May 14 08:55:48 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 05:55:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: clean up _resolve_block_targets
Message-ID: <5af98754.1c69fb81.4751a.2ea4@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94575:1154776fb167
Date: 2018-05-14 14:51 +0200
http://bitbucket.org/pypy/pypy/changeset/1154776fb167/

Log:	clean up _resolve_block_targets

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -148,6 +148,8 @@
         code = []
         for instr in self.instructions:
             instr.encode(code)
+        assert len(code) == self.code_size()
+        assert len(code) & 1 == 0
         return ''.join(code)
 
 
@@ -198,6 +200,13 @@
         self.lineno = 0
         self.add_none_to_final_return = True
 
+    def _check_consistency(self, blocks):
+        current_off = 0
+        for block in blocks:
+            assert block.offset == current_off
+            for instr in block.instructions:
+                current_off += instr.size()
+
     def new_block(self):
         return Block()
 
@@ -328,14 +337,12 @@
 
     def _resolve_block_targets(self, blocks):
         """Compute the arguments of jump instructions."""
-        last_extended_arg_count = 0
         # The reason for this loop is extended jumps.  EXTENDED_ARG
-        # extends the bytecode size, so it might invalidate the offsets
-        # we've already given.  Thus we have to loop until the number of
-        # extended args is stable.  Any extended jump at all is
-        # extremely rare, so performance is not too concerning.
+        # extends the bytecode size, so it might invalidate the offsets we've
+        # already given.  Thus we have to loop until the size of all jump
+        # instructions is stable. Any extended jump at all is extremely rare,
+        # so performance should not be too concerning.
         while True:
-            extended_arg_count = 0
             offset = 0
             force_redo = False
             # Calculate the code offset of each block.
@@ -345,7 +352,8 @@
             for block in blocks:
                 offset = block.offset
                 for instr in block.instructions:
-                    offset += instr.size()
+                    size = instr.size()
+                    offset += size
                     if instr.has_jump:
                         target, absolute = instr.jump
                         op = instr.opcode
@@ -364,22 +372,21 @@
                                     instr.opcode = ops.RETURN_VALUE
                                     instr.arg = 0
                                     instr.has_jump = False
-                                    # The size of the code changed,
+                                    # The size of the code maybe have changed,
                                     # we have to trigger another pass
-                                    force_redo = True
+                                    if instr.size() != size:
+                                        force_redo = True
                                     continue
                         if absolute:
                             jump_arg = target.offset
                         else:
                             jump_arg = target.offset - offset
                         instr.arg = jump_arg
-                        if jump_arg > 0xFFFF:
-                            extended_arg_count += 1
-            if (extended_arg_count == last_extended_arg_count and
-                not force_redo):
-                break
-            else:
-                last_extended_arg_count = extended_arg_count
+                        if instr.size() != size:
+                            force_redo = True
+            if not force_redo:
+                self._check_consistency(blocks)
+                return
 
     def _build_consts_array(self):
         """Turn the applevel constants dictionary into a list."""

From pypy.commits at gmail.com  Mon May 14 08:55:45 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 05:55:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: switch encoding of bytecodes in
 the astcompiler to that of cpython 3.6
Message-ID: <5af98751.1c69fb81.5eeb0.5b95@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94574:61471dc3b236
Date: 2018-05-14 13:33 +0200
http://bitbucket.org/pypy/pypy/changeset/61471dc3b236/

Log:	switch encoding of bytecodes in the astcompiler to that of cpython
	3.6

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -21,6 +21,8 @@
     def __init__(self, opcode, arg=0):
         self.opcode = opcode
         self.arg = arg
+        if opcode < ops.HAVE_ARGUMENT:
+            assert arg == 0
         self.lineno = 0
         self.has_jump = False
 
@@ -28,9 +30,33 @@
         """Return the size of bytes of this instruction when it is
         encoded.
         """
-        if self.opcode >= ops.HAVE_ARGUMENT:
-            return (6 if self.arg > 0xFFFF else 3)
-        return 1
+        if self.arg <= 0xff:
+            return 2
+        if self.arg <= 0xffff:
+            return 4
+        if self.arg <= 0xffffff:
+            return 6
+        return 8
+
+    def encode(self, code):
+        opcode = self.opcode
+
+        arg = self.arg
+        size = self.size()
+        if size == 8:
+            code.append(chr(ops.EXTENDED_ARG))
+            code.append(chr((arg >> 24) & 0xff))
+            assert ((arg >> 24) & 0xff) == (arg >> 24)
+        if size >= 6:
+            code.append(chr(ops.EXTENDED_ARG))
+            code.append(chr((arg >> 16) & 0xff))
+        if size >= 4:
+            code.append(chr(ops.EXTENDED_ARG))
+            code.append(chr((arg >> 8) & 0xff))
+        if size >= 2:
+            code.append(chr(opcode))
+            code.append(chr(arg & 0xff))
+
 
     def jump_to(self, target, absolute=False):
         """Indicate the target this jump instruction.
@@ -121,20 +147,7 @@
         """Encode the instructions in this block into bytecode."""
         code = []
         for instr in self.instructions:
-            opcode = instr.opcode
-            if opcode >= ops.HAVE_ARGUMENT:
-                arg = instr.arg
-                if instr.arg > 0xFFFF:
-                    ext = arg >> 16
-                    code.append(chr(ops.EXTENDED_ARG))
-                    code.append(chr(ext & 0xFF))
-                    code.append(chr(ext >> 8))
-                    arg &= 0xFFFF
-                code.append(chr(opcode))
-                code.append(chr(arg & 0xFF))
-                code.append(chr(arg >> 8))
-            else:
-                code.append(chr(opcode))
+            instr.encode(code)
         return ''.join(code)
 
 
diff --git a/pypy/interpreter/astcompiler/test/test_misc.py b/pypy/interpreter/astcompiler/test/test_misc.py
--- a/pypy/interpreter/astcompiler/test/test_misc.py
+++ b/pypy/interpreter/astcompiler/test/test_misc.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.astcompiler.misc import mangle
+from pypy.interpreter.astcompiler.assemble import Instruction, ops
 
 def test_mangle():
     assert mangle("foo", "Bar") == "foo"
@@ -13,6 +14,34 @@
     assert mangle("__foo", "___") == "__foo"
     assert mangle("___foo", "__Bar") == "_Bar___foo"
 
+def test_instruction_size():
+    assert Instruction(ops.POP_TOP).size() == 2
+    assert Instruction(ops.LOAD_FAST, 23).size() == 2
+    assert Instruction(ops.LOAD_FAST, 0xfff0).size() == 4
+    assert Instruction(ops.LOAD_FAST, 0x10000).size() == 6
+    assert Instruction(ops.LOAD_FAST, 0x1000000).size() == 8
+
+def test_instruction_encode():
+    c = []
+    Instruction(ops.POP_TOP).encode(c)
+    assert c == [chr(ops.POP_TOP), '\x00']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 1).encode(c)
+    assert c == [chr(ops.LOAD_FAST), '\x01']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 0x201).encode(c)
+    assert c == [chr(ops.EXTENDED_ARG), '\x02', chr(ops.LOAD_FAST), '\x01']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 0x30201).encode(c)
+    assert c == [chr(ops.EXTENDED_ARG), '\x03', chr(ops.EXTENDED_ARG), '\x02', chr(ops.LOAD_FAST), '\x01']
+
+    c = []
+    Instruction(ops.LOAD_FAST, 0x5030201).encode(c)
+    assert c == [chr(ops.EXTENDED_ARG), '\x05', chr(ops.EXTENDED_ARG), '\x03', chr(ops.EXTENDED_ARG), '\x02', chr(ops.LOAD_FAST), '\x01']
+
 def app_test_warning_to_error_translation():
     import warnings
 

From pypy.commits at gmail.com  Mon May 14 08:55:55 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 05:55:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: how did this ever work?
Message-ID: <5af9875b.1c69fb81.7a2f6.e651@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94578:016bcfb6289b
Date: 2018-05-14 14:54 +0200
http://bitbucket.org/pypy/pypy/changeset/016bcfb6289b/

Log:	how did this ever work?

diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -940,7 +940,7 @@
 
     def test_flufl(self):
         source = "x <> y"
-        raises(SyntaxError, self.get_ast, source)
+        py.test.raises(SyntaxError, self.get_ast, source)
         comp = self.get_first_expr(source,
                                    flags=consts.CO_FUTURE_BARRY_AS_BDFL)
         assert isinstance(comp, ast.Compare)
@@ -1168,7 +1168,7 @@
         s = self.get_first_expr("b'hi' b' implicitly' b' extra'")
         assert isinstance(s, ast.Bytes)
         assert space.eq_w(s.s, space.newbytes("hi implicitly extra"))
-        raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
+        py.test.raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
         sentence = u"Die M&#228;nner &#228;rgen sich!"
         source = u"# coding: utf-7\nstuff = '%s'" % (sentence,)
         info = pyparse.CompileInfo("<test>", "exec")
@@ -1363,8 +1363,8 @@
         assert isinstance(if2, ast.Name)
 
     def test_cpython_issue12983(self):
-        raises(SyntaxError, self.get_ast, r"""b'\x'""")
-        raises(SyntaxError, self.get_ast, r"""b'\x0'""")
+        py.test.raises(SyntaxError, self.get_ast, r"""b'\x'""")
+        py.test.raises(SyntaxError, self.get_ast, r"""b'\x0'""")
 
     def test_matmul(self):
         mod = self.get_ast("a @ b")
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1008,6 +1008,7 @@
         ("C4.__doc__", 'docstring'),
         ("C4.__doc__", 'docstring'),
         ("__doc__", None),])
+
     def test_remove_docstring(self, expr, result):
         source = '"module_docstring"\n' + """if 1:
         def f1():

From pypy.commits at gmail.com  Mon May 14 08:55:57 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 05:55:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: start adapting the interpreter
 to wordcode
Message-ID: <5af9875d.1c69fb81.e0a2.f1d3@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94579:4b61cd94d616
Date: 2018-05-14 14:54 +0200
http://bitbucket.org/pypy/pypy/changeset/4b61cd94d616/

Log:	start adapting the interpreter to wordcode

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -166,16 +166,20 @@
             # Normal case: the call above raises Yield.
             # We reach this point if the iterable is exhausted.
             last_instr = jit.promote(frame.last_instr)
+            assert last_instr & 1 == 0
             assert last_instr >= 0
-            return r_uint(last_instr + 1)
+            return r_uint(last_instr + 2)
 
         if isinstance(w_arg_or_err, SApplicationException):
             return frame.handle_generator_error(w_arg_or_err.operr)
 
         last_instr = jit.promote(frame.last_instr)
         if last_instr != -1:
+            assert last_instr & 1 == 0
             frame.pushvalue(w_arg_or_err)
-        return r_uint(last_instr + 1)
+            return r_uint(last_instr + 2)
+        else:
+            return r_uint(0)
 
     def next_yield_from(self, frame, w_yf, w_inputvalue_or_err):
         """Fetch the next item of the current 'yield from', push it on
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -774,6 +774,7 @@
             block.cleanupstack(self)
 
         self.getorcreatedebug().f_lineno = new_lineno
+        assert new_lasti & 1 == 0
         self.last_instr = new_lasti
 
     def get_last_lineno(self):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -62,6 +62,7 @@
         co_code = pycode.co_code
         try:
             while True:
+                assert next_instr & 1 == 0
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
         except ExitFrame:
             return self.popvalue()
@@ -152,22 +153,17 @@
     @jit.unroll_safe
     def dispatch_bytecode(self, co_code, next_instr, ec):
         while True:
+            assert next_instr & 1 == 0
             self.last_instr = intmask(next_instr)
             if jit.we_are_jitted():
                 ec.bytecode_only_trace(self)
             else:
                 ec.bytecode_trace(self)
             next_instr = r_uint(self.last_instr)
+            assert next_instr & 1 == 0
             opcode = ord(co_code[next_instr])
-            next_instr += 1
-
-            if opcode >= HAVE_ARGUMENT:
-                lo = ord(co_code[next_instr])
-                hi = ord(co_code[next_instr+1])
-                next_instr += 2
-                oparg = (hi * 256) | lo
-            else:
-                oparg = 0
+            oparg = ord(co_code[next_instr + 1])
+            next_instr += 2
 
             # note: the structure of the code here is such that it makes
             # (after translation) a big "if/elif" chain, which is then
@@ -175,12 +171,11 @@
 
             while opcode == opcodedesc.EXTENDED_ARG.index:
                 opcode = ord(co_code[next_instr])
+                arg = ord(co_code[next_instr + 1])
                 if opcode < HAVE_ARGUMENT:
                     raise BytecodeCorruption
-                lo = ord(co_code[next_instr+1])
-                hi = ord(co_code[next_instr+2])
-                next_instr += 3
-                oparg = (oparg * 65536) | (hi * 256) | lo
+                next_instr += 2
+                oparg = (oparg << 8) | arg
 
             if opcode == opcodedesc.RETURN_VALUE.index:
                 w_returnvalue = self.popvalue()

From pypy.commits at gmail.com  Mon May 14 08:55:53 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 05:55:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: dis3 needs adaptations
Message-ID: <5af98759.1c69fb81.e6ae.d791@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94577:fa6ecad4bd8a
Date: 2018-05-14 14:53 +0200
http://bitbucket.org/pypy/pypy/changeset/fa6ecad4bd8a/

Log:	dis3 needs adaptations

diff --git a/pypy/tool/dis3.py b/pypy/tool/dis3.py
--- a/pypy/tool/dis3.py
+++ b/pypy/tool/dis3.py
@@ -59,6 +59,7 @@
 
 def disassemble(co, lasti=-1):
     """Disassemble a code object."""
+    XXX
     code = co.co_code
     labels = findlabels(code)
     linestarts = dict(findlinestarts(co))

From pypy.commits at gmail.com  Mon May 14 08:55:50 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 05:55:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: bump magic,
 disable exec_host_bytecode
Message-ID: <5af98756.1c69fb81.b5177.a2a8@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94576:6327e535f2fb
Date: 2018-05-14 14:53 +0200
http://bitbucket.org/pypy/pypy/changeset/6327e535f2fb/

Log:	bump magic, disable exec_host_bytecode

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -39,7 +39,7 @@
 # time you make pyc files incompatible.  This value ends up in the frozen
 # importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
-pypy_incremental_magic = 128 # bump it by 16
+pypy_incremental_magic = 144 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
@@ -301,11 +301,7 @@
                 w_co.remove_docstrings(space)
 
     def exec_host_bytecode(self, w_globals, w_locals):
-        if sys.version_info < (2, 7):
-            raise Exception("PyPy no longer supports Python 2.6 or lower")
-        frame = self.space.FrameClass(self.space, self, w_globals, None)
-        frame.setdictscope(w_locals)
-        return frame.run()
+        raise Exception("no longer supported after the switch to wordcode!")
 
     def dump(self):
         """NOT_RPYTHON: A dis.dis() dump of the code object."""

From pypy.commits at gmail.com  Mon May 14 09:43:38 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 06:43:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: we can't use _from_code anymore
 either
Message-ID: <5af9928a.1c69fb81.de124.ba6a@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94580:72505716f39c
Date: 2018-05-14 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/72505716f39c/

Log:	we can't use _from_code anymore either

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -216,6 +216,7 @@
         """
         Hack to initialize the code object from a real (CPython) one.
         """
+        XXX
         assert isinstance(code, types.CodeType)
         newconsts_w = [None] * len(code.co_consts)
         num = 0

From pypy.commits at gmail.com  Mon May 14 09:43:41 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 06:43:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: use _from_code less often (it's
 still in the apptest support, but I'll ignore that for now)
Message-ID: <5af9928d.1c69fb81.f4c2f.a7f3@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94581:a890ba3fdb85
Date: 2018-05-14 15:42 +0200
http://bitbucket.org/pypy/pypy/changeset/a890ba3fdb85/

Log:	use _from_code less often (it's still in the apptest support, but
	I'll ignore that for now)

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -72,9 +72,8 @@
     def check(self, w_dict, evalexpr, expected):
         # for now, we compile evalexpr with CPython's compiler but run
         # it with our own interpreter to extract the data from w_dict
-        co_expr = compile(evalexpr, '<evalexpr>', 'eval')
         space = self.space
-        pyco_expr = PyCode._from_code(space, co_expr)
+        pyco_expr = space.createcompiler().compile(evalexpr, '<evalexpr>', 'eval')
         w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict)
         res = space.str_w(space.repr(w_res))
         expected_repr = self.get_py3_repr(expected)
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -641,11 +641,20 @@
 
 
 class TestMethod:
-    def setup_method(self, method):
-        def c(self, bar):
-            return bar
-        code = PyCode._from_code(self.space, c.__code__)
-        self.fn = Function(self.space, code, self.space.newdict())
+    @classmethod
+    def compile(cls, src):
+        assert src.strip().startswith("def ")
+        compiler = cls.space.createcompiler()
+        code = compiler.compile(src, '<hello>', 'exec', 0).co_consts_w[0]
+        return Function(cls.space, code, cls.space.newdict())
+
+    def setup_class(cls):
+        src = """
+def c(self, bar):
+    return bar
+        """
+        cls.fn = cls.compile(src)
+
 
     def test_get(self):
         space = self.space
@@ -670,9 +679,7 @@
     def test_method_get(self):
         space = self.space
         # Create some function for this test only
-        def m(self): return self
-        func = Function(space, PyCode._from_code(self.space, m.__code__),
-                        space.newdict())
+        func = self.compile("def m(self): return self")
         # Some shorthands
         obj1 = space.wrap(23)
         obj2 = space.wrap(42)
@@ -694,6 +701,11 @@
         assert meth3 is func
 
 class TestShortcuts(object):
+    def compile(self, src):
+        assert src.strip().startswith("def ")
+        compiler = self.space.createcompiler()
+        code = compiler.compile(src, '<hello>', 'exec', 0).co_consts_w[0]
+        return Function(self.space, code, self.space.newdict())
 
     def test_call_function(self):
         space = self.space
@@ -701,14 +713,15 @@
         d = {}
         for i in range(10):
             args = "(" + ''.join(["a%d," % a for a in range(i)]) + ")"
-            exec """
+            src = """
 def f%s:
     return %s
-""" % (args, args) in d
+""" % (args, args)
+            exec src in d
             f = d['f']
             res = f(*range(i))
-            code = PyCode._from_code(self.space, f.__code__)
-            fn = Function(self.space, code, self.space.newdict())
+            fn = self.compile(src)
+            code = fn.code
 
             assert fn.code.fast_natural_arity == i|PyCode.FLATPYCALL
             if i < 5:
@@ -727,18 +740,18 @@
     def test_flatcall(self):
         space = self.space
 
-        def f(a):
-            return a
-        code = PyCode._from_code(self.space, f.__code__)
-        fn = Function(self.space, code, self.space.newdict())
+        src = """
+def f(a):
+    return a"""
+        fn = self.compile(src)
 
         assert fn.code.fast_natural_arity == 1|PyCode.FLATPYCALL
 
         def bomb(*args):
             assert False, "shortcutting should have avoided this"
 
-        code.funcrun = bomb
-        code.funcrun_obj = bomb
+        fn.code.funcrun = bomb
+        fn.code.funcrun_obj = bomb
 
         w_3 = space.newint(3)
         w_res = space.call_function(fn, w_3)
@@ -754,18 +767,19 @@
     def test_flatcall_method(self):
         space = self.space
 
-        def f(self, a):
-            return a
-        code = PyCode._from_code(self.space, f.__code__)
-        fn = Function(self.space, code, self.space.newdict())
+        src = """
+def f(self, a):
+    return a
+"""
+        fn = self.compile(src)
 
         assert fn.code.fast_natural_arity == 2|PyCode.FLATPYCALL
 
         def bomb(*args):
             assert False, "shortcutting should have avoided this"
 
-        code.funcrun = bomb
-        code.funcrun_obj = bomb
+        fn.code.funcrun = bomb
+        fn.code.funcrun_obj = bomb
 
         w_3 = space.newint(3)
         w_res = space.appexec([fn, w_3], """(f, x):
@@ -782,9 +796,11 @@
     def test_flatcall_default_arg(self):
         space = self.space
 
-        def f(a, b):
-            return a+b
-        code = PyCode._from_code(self.space, f.__code__)
+        src = """
+def f(a, b):
+    return a+b
+"""
+        code = self.compile(src).code
         fn = Function(self.space, code, self.space.newdict(),
                       defs_w=[space.newint(1)])
 
@@ -811,9 +827,11 @@
     def test_flatcall_default_arg_method(self):
         space = self.space
 
-        def f(self, a, b):
-            return a+b
-        code = PyCode._from_code(self.space, f.__code__)
+        src = """
+def f(self, a, b):
+    return a+b
+        """
+        code = self.compile(src).code
         fn = Function(self.space, code, self.space.newdict(),
                       defs_w=[space.newint(1)])
 
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -788,7 +788,7 @@
     f.write(_getlong(mtime))
     if co:
         # marshal the code object with the PyPy marshal impl
-        pyco = PyCode._from_code(space, co)
+        pyco = space.createcompiler().compile(co, '?', 'exec', 0)
         w_marshal = space.getbuiltinmodule('marshal')
         w_marshaled_code = space.call_method(w_marshal, 'dumps', pyco)
         marshaled_code = space.bytes_w(w_marshaled_code)
@@ -809,7 +809,7 @@
     def test_read_compiled_module(self):
         space = self.space
         mtime = 12345
-        co = compile('x = 42', '?', 'exec')
+        co = 'x = 42'
         cpathname = _testfile(space, importing.get_pyc_magic(space), mtime, co)
         stream = streamio.open_file_as_stream(cpathname, "rb")
         try:
@@ -829,7 +829,7 @@
     def test_load_compiled_module(self):
         space = self.space
         mtime = 12345
-        co = compile('x = 42', '?', 'exec')
+        co = 'x = 42'
         cpathname = _testfile(space, importing.get_pyc_magic(space), mtime, co)
         w_modulename = space.wrap('somemodule')
         stream = streamio.open_file_as_stream(cpathname, "rb")
@@ -854,7 +854,7 @@
     def test_load_compiled_module_nopathname(self):
         space = self.space
         mtime = 12345
-        co = compile('x = 42', '?', 'exec')
+        co = 'x = 42'
         cpathname = _testfile(space, importing.get_pyc_magic(space), mtime, co)
         w_modulename = space.wrap('somemodule')
         stream = streamio.open_file_as_stream(cpathname, "rb")
@@ -931,7 +931,7 @@
                     continue
                 pathname = "whatever"
                 mtime = 12345
-                co = compile('x = 42', '?', 'exec')
+                co = 'x = 42'
                 cpathname = _testfile(space1, importing.get_pyc_magic(space1),
                                       mtime, co)
                 w_modulename = space2.wrap('somemodule')

From pypy.commits at gmail.com  Mon May 14 09:49:13 2018
From: pypy.commits at gmail.com (arigo)
Date: Mon, 14 May 2018 06:49:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Empty tuples
Message-ID: <5af993d9.1c69fb81.787ee.b9c5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.6
Changeset: r94582:3131adbe1e58
Date: 2018-05-14 15:48 +0200
http://bitbucket.org/pypy/pypy/changeset/3131adbe1e58/

Log:	Empty tuples

diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py
--- a/pypy/interpreter/astcompiler/asthelpers.py
+++ b/pypy/interpreter/astcompiler/asthelpers.py
@@ -83,10 +83,7 @@
         if self.elts:
             for elt in self.elts:
                 elt.set_context(space, ctx)
-            self.ctx = ctx
-        else:
-            # Assignment to () raises an error.
-            ast.expr.set_context(self, space, ctx)
+        self.ctx = ctx
 
 class __extend__(ast.Lambda):
 
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -287,6 +287,9 @@
 
     def visit_Tuple(self, tup):
         """Try to turn tuple building into a constant."""
+        if tup.ctx != ast.Load:
+            return tup   # Don't do the rest for assignment or delete targets.
+                         # It would replace Tuple([]) with Constant('()')!
         if tup.elts:
             consts_w = [None]*len(tup.elts)
             for i in range(len(tup.elts)):
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -810,7 +810,6 @@
             ("{x : x for x in z}", "dict comprehension"),
             ("'str'", "literal"),
             ("b'bytes'", "literal"),
-            ("()", "()"),
             ("23", "literal"),
             ("{}", "literal"),
             ("{1, 2, 3}", "literal"),
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -129,7 +129,7 @@
         self.simple_test(stmt, "type(x)", int)
 
     def test_tuple_assign(self):
-        yield self.error_test, "() = 1", SyntaxError
+        yield self.simple_test, "() = []", "1", 1
         yield self.simple_test, "x,= 1,", "x", 1
         yield self.simple_test, "x,y = 1,2", "x,y", (1, 2)
         yield self.simple_test, "x,y,z = 1,2,3", "x,y,z", (1, 2, 3)
@@ -1482,3 +1482,12 @@
             return f'ab{x}cd'
         """
         code, blocks = generate_function_code(source, self.space)
+
+    def test_empty_tuple_target(self):
+        source = """def f():
+            () = ()
+            del ()
+            [] = []
+            del []
+        """
+        generate_function_code(source, self.space)
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -783,21 +783,6 @@
         assert isinstance(code, PyCode)
         assert code.co_filename == '<filename2>'
 
-    def test_with_empty_tuple(self):
-        source = py.code.Source("""
-        from __future__ import with_statement
-
-        with x as ():
-            pass
-        """)
-        try:
-            self.compiler.compile(str(source), '<filename>', 'exec', 0)
-        except OperationError as e:
-            if not e.match(self.space, self.space.w_SyntaxError):
-                raise
-        else:
-            py.test.fail("Did not raise")
-
     def test_assign_to_yield(self):
         code = 'def f(): (yield bar) += y'
         try:
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -752,6 +752,33 @@
         exc = raises(SyntaxError, compile, code, 'foo', 'exec')
         assert exc.value.offset in (19, 20) # pypy, cpython
 
+    def test_empty_tuple_target(self): """
+        def f(n):
+            () = ()
+            ((), ()) = [[], []]
+            del ()
+            del ((), ())
+            [] = {}
+            ([], ()) = [[], ()]
+            [[], ()] = ((), [])
+            del []
+            del [[], ()]
+            for () in [(), []]: pass
+            for [] in ([],): pass
+            class Zen:
+                def __enter__(self): return ()
+                def __exit__(self, *args): pass
+            with Zen() as (): pass
+            () = [2, 3] * n
+        f(0)
+        try:
+            f(5)
+        except ValueError:
+            pass
+        else:
+            raise AssertionError("should have raised")
+        """
+
 
 if __name__ == '__main__':
     # only to check on top of CPython (you need 2.4)

From pypy.commits at gmail.com  Mon May 14 10:36:32 2018
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 14 May 2018 07:36:32 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8: hg merge default
Message-ID: <5af99ef0.1c69fb81.d4cfe.c513@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: unicode-utf8
Changeset: r94583:916081855b81
Date: 2018-05-14 15:35 +0100
http://bitbucket.org/pypy/pypy/changeset/916081855b81/

Log:	hg merge default

diff too long, truncating to 2000 out of 3209 lines

diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -73,3 +73,63 @@
 This division between bytecode evaluator and object space gives a lot of
 flexibility.  One can plug in different :doc:`object spaces <objspace>` to get
 different or enriched behaviours of the Python objects.
+
+Layers
+------
+
+RPython
+~~~~~~~
+:ref:`RPython <rpython:language>` is the language in which we write interpreters.
+Not the entire PyPy project is written in RPython, only the parts that are
+compiled in the translation process. The interesting point is that RPython
+has no parser, it's compiled from the live python objects, which makes it
+possible to do all kinds of metaprogramming during import time. In short,
+Python is a meta programming language for RPython.
+
+The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+Consult `Getting Started with RPython`_ for further reading
+
+Translation
+~~~~~~~~~~~
+The translation toolchain - this is the part that takes care of translating
+RPython to flow graphs and then to C. There is more in the
+:doc:`architecture <architecture>` document written about it.
+
+It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+and ``rtyper``.
+
+PyPy Interpreter
+~~~~~~~~~~~~~~~~
+This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
+interpreter for Python written in RPython.  The fact that it is
+RPython is not apparent at first.  Built-in modules are written in
+``pypy/module/*``.  Some modules that CPython implements in C are
+simply written in pure Python; they are in the top-level ``lib_pypy``
+directory.  The standard library of Python (with a few changes to
+accomodate PyPy) is in ``lib-python``.
+
+JIT Compiler
+~~~~~~~~~~~~
+:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
+interpreter written in RPython, rather than the user program that it
+interprets.  As a result it applies to any interpreter, i.e. any
+language.  But getting it to work correctly is not trivial: it
+requires a small number of precise "hints" and possibly some small
+refactorings of the interpreter.  The JIT itself also has several
+almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+that turns it into machine code.  Writing a new backend is a
+traditional way to get into the project.
+
+Garbage Collectors
+~~~~~~~~~~~~~~~~~~
+Garbage Collectors (GC): as you may notice if you are used to CPython's
+C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+:ref:`rpython:garbage-collection` is inserted
+during translation.  Moreover, this is not reference counting; it is a real
+GC written as more RPython code.  The best one we have so far is in
+``rpython/memory/gc/incminimark.py``.
+
+.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -267,14 +267,14 @@
 * PyPy 2.5.1 or earlier: normal users would see permission errors.
   Installers need to run ``pypy -c "import gdbm"`` and other similar
   commands at install time; the exact list is in
-  :source:`pypy/tool/release/package.py <package.py>`.  Users
+  :source:`pypy/tool/release/package.py`.  Users
   seeing a broken installation of PyPy can fix it after-the-fact if they
   have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
 
 * PyPy 2.6 and later: anyone would get ``ImportError: no module named
   _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
   ``lib_pypy`` directory during the installation process (plus others;
-  see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+  see the exact list in :source:`pypy/tool/release/package.py`).
   Users seeing a broken
   installation of PyPy can fix it after-the-fact, by running ``pypy
   /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -539,7 +539,7 @@
 
     hg help branch
 
-.. _official wiki: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: https://www.mercurial-scm.org/wiki/
 
 
 .. _using-development-tracker:
@@ -547,15 +547,7 @@
 Using the development bug/feature tracker
 -----------------------------------------
 
-We have a `development tracker`_, based on Richard Jones'
-`roundup`_ application.  You can file bugs,
-feature requests or see what's going on
-for the next milestone, both from an E-Mail and from a
-web interface.
-
-.. _development tracker: https://bugs.pypy.org/
-.. _roundup: http://roundup.sourceforge.net/
-
+We use bitbucket for :source:`issues` tracking and :source:`pull-requests`.
 
 .. _testing:
 
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -8,3 +8,4 @@
    :maxdepth: 1
 
    man/pypy.1.rst
+   man/pypy3.1.rst
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -66,9 +66,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '5.8'
+version = '6.0'
 # The full version, including alpha/beta/rc tags.
-release = '5.8.0'
+release = '6.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/config/objspace.usemodules._cppyy.txt b/pypy/doc/config/objspace.usemodules._cppyy.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules._cppyy.txt
@@ -0,0 +1,1 @@
+The internal backend for cppyy
diff --git a/pypy/doc/config/objspace.usemodules._rawffi.txt b/pypy/doc/config/objspace.usemodules._rawffi.txt
--- a/pypy/doc/config/objspace.usemodules._rawffi.txt
+++ b/pypy/doc/config/objspace.usemodules._rawffi.txt
@@ -1,3 +1,3 @@
-An experimental module providing very low-level interface to
+A module providing very low-level interface to
 C-level libraries, for use when implementing ctypes, not
-intended for a direct use at all.
\ No newline at end of file
+intended for a direct use at all.
diff --git a/pypy/doc/config/objspace.usemodules.cpyext.txt b/pypy/doc/config/objspace.usemodules.cpyext.txt
--- a/pypy/doc/config/objspace.usemodules.cpyext.txt
+++ b/pypy/doc/config/objspace.usemodules.cpyext.txt
@@ -1,1 +1,1 @@
-Use (experimental) cpyext module, that tries to load and run CPython extension modules
+Use cpyext module to load and run CPython extension modules
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/contributing.rst
@@ -0,0 +1,472 @@
+Contributing Guidelines
+===========================
+
+.. contents::
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy's
+  approach is great if you do TDD, and not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below.
+
+The first thing to remember is that PyPy project is very different than most
+projects out there. It's also different from a classic compiler project,
+so academic courses about compilers often don't apply or lead in the wrong
+direction. However, if you want to understand how designing & building a runtime
+works in the real world then this is a great project!
+
+Getting involved
+^^^^^^^^^^^^^^^^
+
+PyPy employs a relatively standard open-source development process. You are
+encouraged as a first step to join our `pypy-dev mailing list`_ and IRC channel,
+details of which can be found in our :ref:`contact <contact>` section. The folks
+there are very friendly, and can point you in the right direction.
+
+We give out commit rights usually fairly liberally, so if you want to do something
+with PyPy, you can become a committer. We also run frequent coding sprints which
+are separately announced and often happen around Python conferences such as
+EuroPython or PyCon. Upcoming events are usually announced on `the blog`_.
+
+Further Reading: :ref:`Contact <contact>`
+
+.. _the blog: http://morepypy.blogspot.com
+.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
+
+
+Your first contribution
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The first and most important rule how **not** to contribute to PyPy is
+"just hacking a feature". This won't work, and you'll find your PR will typically
+require a lot of re-work. There are a few reasons why not:
+
+* build times are large
+* PyPy has very thick layer separation
+* context of the cPython runtime is often required
+
+Instead, reach out on the dev mailing list or the IRC channel, and we're more
+than happy to help! :)
+
+Some ideas for first contributions are:
+
+* Documentation - this will give you an understanding of the pypy architecture
+* Test failures - find a failing test in the `nightly builds`_, and fix it
+* Missing language features - these are listed in our `issue tracker`_
+
+.. _nightly builds: http://buildbot.pypy.org/nightly/
+.. _issue tracker: https://bitbucket.org/pypy/pypy/issues
+
+Source Control
+--------------
+
+PyPy development is based a typical fork/pull request based workflow, centered
+around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
+before, a good introduction can be found here:
+
+    https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow
+
+The cycle for a new PyPy contributor goes typically like this:
+
+Fork & Clone
+------------
+
+* Make an account on bitbucket_.
+
+* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
+  icons).  You get a fork of the repository, e.g. in
+  `https://bitbucket.org/yourname/pypy/`.
+
+* Clone your new repo (i.e. the fork) to your local machine with the command
+  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
+  operation but only ever needs to be done once.  See also
+  http://pypy.org/download.html#building-from-source .
+  If you already cloned
+  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
+  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
+  your clone to contain the line ``default =
+  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
+  up``.  If you already have such a clone but don't want to change it,
+  you can clone that copy with ``hg clone /path/to/other/copy``, and
+  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
+
+* Now you have a complete copy of the PyPy repo.  Make a branch
+  with a command like ``hg branch name_of_your_branch``.
+
+Edit
+----
+
+* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
+  to make Mercurial aware of new files you added, e.g. new test files.
+  Use ``hg status`` to see if there are such files.  Write and run tests!
+  (See the rest of this page.)
+
+* Commit regularly with ``hg commit``.  A one-line commit message is
+  fine.  We love to have tons of commits; make one as soon as you have
+  some progress, even if it is only some new test that doesn't pass yet,
+  or fixing things even if not all tests pass.  Step by step, you are
+  building the history of your changes, which is the point of a version
+  control system.  (There are commands like ``hg log`` and ``hg up``
+  that you should read about later, to learn how to navigate this
+  history.)
+
+* The commits stay on your machine until you do ``hg push`` to "push"
+  them back to the repo named in the file ``.hg/hgrc``.  Repos are
+  basically just collections of commits (a commit is also called a
+  changeset): there is one repo per url, plus one for each local copy on
+  each local machine.  The commands ``hg push`` and ``hg pull`` copy
+  commits around, with the goal that all repos in question end up with
+  the exact same set of commits.  By opposition, ``hg up`` only updates
+  the "working copy" by reading the local repository, i.e. it makes the
+  files that you see correspond to the latest (or any other) commit
+  locally present.
+
+* You should push often; there is no real reason not to.  Remember that
+  even if they are pushed, with the setup above, the commits are (1)
+  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
+  named.  Yes, they are publicly visible, but don't worry about someone
+  walking around the thousands of repos on bitbucket saying "hah, look
+  at the bad coding style of that guy".  Try to get into the mindset
+  that your work is not secret and it's fine that way.  We might not
+  accept it as is for PyPy, asking you instead to improve some things,
+  but we are not going to judge you.
+
+Pull Request
+------------
+
+* The final step is to open a pull request, so that we know that you'd
+  like to merge that branch back to the original ``pypy/pypy`` repo.
+  This can also be done several times if you have interesting
+  intermediate states, but if you get there, then we're likely to
+  proceed to the next stage, which is...
+
+* Get a regular account for pushing directly to
+  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
+  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
+  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
+  then be pushed directly to the official repo, but (if you follow these
+  rules) they are still on a branch, and we can still review the
+  branches you want to merge.
+
+* If you get closer to the regular day-to-day development, you'll notice
+  that we generally push small changes as one or a few commits directly
+  to the branch ``default``.  Also, we often collaborate even if we are
+  on other branches, which do not really "belong" to anyone.  At this
+  point you'll need ``hg merge`` and learn how to resolve conflicts that
+  sometimes occur when two people try to push different commits in
+  parallel on the same branch.  But it is likely an issue for later ``:-)``
+
+.. _bitbucket: https://bitbucket.org/
+
+
+Architecture
+^^^^^^^^^^^^
+
+PyPy has layers. Just like ogres or onions. Those layers help us keep the
+respective parts separated enough to be worked on independently and make the
+complexity manageable. This is, again, just a sanity requirement for such
+a complex project. For example writing a new optimization for the JIT usually
+does **not** involve touching a Python interpreter at all or the JIT assembler
+backend or the garbage collector. Instead it requires writing small tests in
+``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
+After that, you can just compile PyPy and things should just work.
+
+Further Reading: :doc:`architecture <architecture>`
+
+Where to start?
+---------------
+
+PyPy is made from parts that are relatively independent of each other.
+You should start looking at the part that attracts you most (all paths are
+relative to the PyPy top level directory).  You may look at our
+:doc:`directory reference <dir-reference>` or start off at one of the following
+points:
+
+*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
+   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
+   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
+   function objects and argument passing in :source:`pypy/interpreter/function.py`
+   and :source:`pypy/interpreter/argument.py`, the object space interface
+   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
+   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
+   Core types supporting the bytecode interpreter are defined in
+   :source:`pypy/interpreter/typedef.py`.
+
+*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
+   and grammar files that allow it to parse the syntax of various Python
+   versions. Once the grammar has been processed, the parser can be
+   translated by the above machinery into efficient code.
+
+*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
+   contains a modified version of the compiler package from CPython
+   that fixes some bugs and is translatable.
+
+*  :source:`pypy/objspace/std` contains the
+   :ref:`Standard object space <standard-object-space>`.  The main file
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)
+
+Building
+^^^^^^^^
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Further Reading: :doc:`Build <build>`
+
+Coding Guide
+------------
+
+As well as the usual pep8 and formatting standards, there are a number of
+naming conventions and coding styles that are important to understand before
+browsing the source.
+
+Further Reading: :doc:`Coding Guide <coding-guide>`
+
+Testing
+^^^^^^^
+
+Test driven development
+-----------------------
+
+Instead, we practice a lot of test driven development. This is partly because
+of very high quality requirements for compilers and partly because there is
+simply no other way to get around such complex project, that will keep you sane.
+There are probably people out there who are smart enough not to need it, we're
+not one of those. You may consider familiarizing yourself with `pytest`_,
+since this is a tool we use for tests.
+This leads to the next issue:
+
+.. _pytest: http://pytest.org/
+
+py.test and the py lib
+----------------------
+
+The `py.test testing tool`_ drives all our testing needs.
+
+We use the `py library`_ for filesystem path manipulations, terminal
+writing, logging and some other support  functionality.
+
+You don't necessarily need to install these two libraries because
+we also ship them inlined in the PyPy source tree.
+
+.. _py library: http://pylib.readthedocs.org/
+
+Running PyPy's unit tests
+-------------------------
+
+PyPy development always was and is still thoroughly test-driven.
+We use the flexible `py.test testing tool`_ which you can `install independently
+<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
+
+The PyPy source tree comes with an inlined version of ``py.test``
+which you can invoke by typing::
+
+    python pytest.py -h
+
+This is usually equivalent to using an installed version::
+
+    py.test -h
+
+If you encounter problems with the installed version
+make sure you have the correct version installed which
+you can find out with the ``--version`` switch.
+
+You will need the `build requirements`_ to run tests successfully, since many of
+them compile little pieces of PyPy and then run the tests inside that minimal
+interpreter. The `cpyext` tests also require `pycparser`, and many tests build
+cases with `hypothesis`.
+
+Now on to running some tests.  PyPy has many different test directories
+and you can use shell completion to point at directories or files::
+
+    py.test pypy/interpreter/test/test_pyframe.py
+
+    # or for running tests of a whole subdirectory
+    py.test pypy/interpreter/
+
+See `py.test usage and invocations`_ for some more generic info
+on how you can run tests.
+
+Beware trying to run "all" pypy tests by pointing to the root
+directory or even the top level subdirectory ``pypy``.  It takes
+hours and uses huge amounts of RAM and is not recommended.
+
+To run CPython regression tests you can point to the ``lib-python``
+directory::
+
+    py.test lib-python/2.7/test/test_datetime.py
+
+This will usually take a long time because this will run
+the PyPy Python interpreter on top of CPython.  On the plus
+side, it's usually still faster than doing a full translation
+and running the regression test with the translated PyPy Python
+interpreter.
+
+.. _py.test testing tool: http://pytest.org
+.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
+.. _`build requirements`: build.html#install-build-time-dependencies
+
+Testing After Translation
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While the usual invocation of `pytest` translates a piece of RPython code and
+runs it, we have a test extension to run tests without translation, directly
+on the host python. This is very convenient for modules such as `cpyext`, to
+compare and contrast test results between CPython and PyPy. Untranslated tests
+are invoked by using the `-A` or `--runappdirect` option to `pytest`::
+
+    python2 pytest.py -A pypy/module/cpyext/test
+
+where `python2` can be either `python2` or `pypy2`. On the `py3` branch, the
+collection phase must be run with `python2` so untranslated tests are run
+with::
+
+    cpython2 pytest.py -A pypy/module/cpyext/test --python=path/to/pypy3
+
+
+Tooling & Utilities
+^^^^^^^^^^^^^^^^^^^
+
+If you are interested in the inner workings of the PyPy Python interpreter,
+there are some features of the untranslated Python interpreter that allow you
+to introspect its internals.
+
+
+Interpreter-level console
+-------------------------
+
+To start interpreting Python with PyPy, install a C compiler that is
+supported by distutils and use Python 2.7 or greater to run PyPy::
+
+    cd pypy
+    python bin/pyinteractive.py
+
+After a few seconds (remember: this is running on top of CPython), you should
+be at the PyPy prompt, which is the same as the Python prompt, but with an
+extra ">".
+
+If you press
+<Ctrl-C> on the console you enter the interpreter-level console, a
+usual CPython console.  You can then access internal objects of PyPy
+(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
+prompt with the prefix ``w_``::
+
+    >>>> a = 123
+    >>>> <Ctrl-C>
+    *** Entering interpreter-level console ***
+    >>> w_a
+    W_IntObject(123)
+
+The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
+
+    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
+    >>> <Ctrl-D>
+    *** Leaving interpreter-level console ***
+
+    KeyboardInterrupt
+    >>>> l
+    [1, 'abc']
+
+Note that the prompt of the interpreter-level console is only '>>>' since
+it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
+Linux) or <Ctrl-Z>, <Enter> (under Windows).
+
+Also note that not all modules are available by default in this mode (for
+example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
+``--withmod-...`` command line options.
+
+You may be interested in reading more about the distinction between
+:ref:`interpreter-level and app-level <interpreter-level>`.
+
+pyinteractive.py options
+------------------------
+
+To list the PyPy interpreter command line options, type::
+
+    cd pypy
+    python bin/pyinteractive.py --help
+
+pyinteractive.py supports most of the options that CPython supports too (in addition to a
+large amount of options that can be used to customize pyinteractive.py).
+As an example of using PyPy from the command line, you could type::
+
+    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
+
+Alternatively, as with regular Python, you can simply give a
+script name on the command line::
+
+    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
+
+See our :doc:`configuration sections <config/index>` for details about what all the commandline
+options do.
+
+
+.. _trace example:
+
+Tracing bytecode and operations on objects
+------------------------------------------
+
+You can use a simple tracing mode to monitor the interpretation of
+bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
+PyPy console::
+
+    >>>> __pytrace__ = 1
+    Tracing enabled
+    >>>> x = 5
+            <module>:           LOAD_CONST    0 (5)
+            <module>:           STORE_NAME    0 (x)
+            <module>:           LOAD_CONST    1 (None)
+            <module>:           RETURN_VALUE    0
+    >>>> x
+            <module>:           LOAD_NAME    0 (x)
+            <module>:           PRINT_EXPR    0
+    5
+            <module>:           LOAD_CONST    0 (None)
+            <module>:           RETURN_VALUE    0
+    >>>>
+
+
+Demos
+^^^^^
+
+The `example-interpreter`_ repository contains an example interpreter
+written using the RPython translation toolchain.
+
+.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
+
+
+graphviz & pygame for flow graph viewing (highly recommended)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+graphviz and pygame are both necessary if you want to look at generated flow
+graphs:
+
+    graphviz: http://www.graphviz.org/Download.php
+
+    pygame: http://www.pygame.org/download.shtml
+
diff --git a/pypy/doc/discussion/ctypes-implementation.rst b/pypy/doc/discussion/ctypes-implementation.rst
--- a/pypy/doc/discussion/ctypes-implementation.rst
+++ b/pypy/doc/discussion/ctypes-implementation.rst
@@ -141,28 +141,3 @@
 
 .. _pyglet: http://pyglet.org/
 
-
-ctypes configure
------------------
-
-We also released ``ctypes-configure``, which is an experimental package
-trying to approach the portability issues of ctypes-based code.
-
-idea
-~~~~
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes c
-compiler (via distutils) for various platform-dependent details like
-exact sizes of types (for example size_t), ``#defines``, exact outline of
-structures etc. It replaces in this regard code generator (h2py).
-
-installation
-~~~~~~~~~~~~
-
-``easy_install ctypes_configure``
-
-usage
-~~~~~
-
-:source:`ctypes_configure/doc/sample.py` explains in details how to use it.
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -1,5 +1,5 @@
-Embedding PyPy
-==============
+Embedding PyPy (DEPRECATED)
+===========================
 
 PyPy has a very minimal and a very strange embedding interface, based on
 the usage of `cffi`_ and the philosophy that Python is a better language than
diff --git a/pypy/doc/eventhistory.rst b/pypy/doc/eventhistory.rst
--- a/pypy/doc/eventhistory.rst
+++ b/pypy/doc/eventhistory.rst
@@ -40,11 +40,9 @@
 Main focus of the sprint will be on the goals of the upcoming June 0.9
 release.
 
-Read more in `the sprint announcement`__, see who is  planning to attend
-on the `people page`_.
+Read more about `the sprint`__
 
-__ https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/announce.html
-.. _people page: https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/people.txt
+__ https://bitbucket.org/pypy/extradoc/src/extradoc/sprintinfo/ddorf2006/
 
 
 PyPy sprint at Akihabara (Tokyo, Japan)
diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst
--- a/pypy/doc/extradoc.rst
+++ b/pypy/doc/extradoc.rst
@@ -75,12 +75,12 @@
 .. _A Way Forward in Parallelising Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2014/position-paper.pdf
 .. _Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf
 .. _Allocation Removal by Partial Evaluation in a Tracing JIT: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/pepm2011/bolz-allocation-removal.pdf
-.. _Towards a Jitting VM for Prolog Execution: http://www.stups.uni-duesseldorf.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
+.. _Towards a Jitting VM for Prolog Execution: http://stups.hhu.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
 .. _High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages: http://buildbot.pypy.org/misc/antocuni-thesis.pdf
 .. _How to *not* write Virtual Machines for Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dyla2007/dyla.pdf
 .. _`Tracing the Meta-Level: PyPy's Tracing JIT Compiler`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009/bolz-tracing-jit.pdf
 .. _`Faster than C#: Efficient Implementation of Dynamic Languages on .NET`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009-dotnet/cli-jit.pdf
-.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: http://stups.hhu.de/mediawiki/images/b/b9/Master_bolz.pdf
+.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: https://www.researchgate.net/profile/Davide_Ancona/publication/252023163_Automatic_generation_of_JIT_compilers_for_dynamic_languages_in_NET/links/53f2098e0cf2bc0c40e70023/Automatic-generation-of-JIT-compilers-for-dynamic-languages-in-NET.pdf
 .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/DynamicLanguages_abstracts.html#AACM-DLS07
 .. _EU Reports: index-report.html
 .. _Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution: http://sabi.net/nriley/pubs/dls6-riley.pdf
@@ -368,6 +368,6 @@
 .. _LLVM: http://llvm.org/
 .. _IronPython: http://ironpython.codeplex.com/
 .. _Dynamic Native Optimization of Native Interpreters: http://people.csail.mit.edu/gregs/dynamorio.html
-.. _JikesRVM: http://jikesrvm.org/
+.. _JikesRVM: http://www.jikesrvm.org/
 .. _Tunes: http://tunes.org
 .. _old Tunes Wiki: http://buildbot.pypy.org/misc/cliki.tunes.org/
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -67,7 +67,7 @@
 you may need to run the command with `sudo` for a global installation.
 The other commands of ``setup.py`` are available too, like ``build``.
 
-.. _PyPI: https://pypi.python.org/pypi
+.. _PyPI: https://pypi.org
 .. _`use virtualenv (as documented here)`: install.html#installing-using-virtualenv
 
 
@@ -360,7 +360,7 @@
 (produced during a sprint).  On the `PyPy bitbucket page`_ there is also a
 Scheme and an Io implementation; both of these are unfinished at the moment.
 
-.. _Topaz: http://topazruby.com/
+.. _Topaz: http://docs.topazruby.com/en/latest/
 .. _Hippy: http://morepypy.blogspot.ch/2012/07/hello-everyone.html
 .. _JavaScript interpreter: https://bitbucket.org/pypy/lang-js/
 .. _Prolog interpreter: https://bitbucket.org/cfbolz/pyrolog/
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
deleted file mode 100644
--- a/pypy/doc/getting-started-dev.rst
+++ /dev/null
@@ -1,345 +0,0 @@
-Getting Started Developing With PyPy
-====================================
-
-.. contents::
-
-
-Using Mercurial
----------------
-
-PyPy development is based on Mercurial (hg).  If you are not used to
-version control, the cycle for a new PyPy contributor goes typically
-like this:
-
-* Make an account on bitbucket_.
-
-* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
-  icons).  You get a fork of the repository, e.g. in
-  https://bitbucket.org/yourname/pypy/.
-
-* Clone this new repo (i.e. the fork) to your local machine with the command 
-  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
-  operation but only ever needs to be done once.  See also 
-  http://pypy.org/download.html#building-from-source .
-  If you already cloned
-  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
-  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
-  your clone to contain the line ``default =
-  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
-  up``.  If you already have such a clone but don't want to change it,
-  you can clone that copy with ``hg clone /path/to/other/copy``, and
-  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
-
-* Now you have a complete copy of the PyPy repo.  Make a branch
-  with a command like ``hg branch name_of_your_branch``.
-
-* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
-  to make Mercurial aware of new files you added, e.g. new test files.
-  Use ``hg status`` to see if there are such files.  Write and run tests!
-  (See the rest of this page.)
-
-* Commit regularly with ``hg commit``.  A one-line commit message is
-  fine.  We love to have tons of commits; make one as soon as you have
-  some progress, even if it is only some new test that doesn't pass yet,
-  or fixing things even if not all tests pass.  Step by step, you are
-  building the history of your changes, which is the point of a version
-  control system.  (There are commands like ``hg log`` and ``hg up``
-  that you should read about later, to learn how to navigate this
-  history.)
-
-* The commits stay on your machine until you do ``hg push`` to "push"
-  them back to the repo named in the file ``.hg/hgrc``.  Repos are
-  basically just collections of commits (a commit is also called a
-  changeset): there is one repo per url, plus one for each local copy on
-  each local machine.  The commands ``hg push`` and ``hg pull`` copy
-  commits around, with the goal that all repos in question end up with
-  the exact same set of commits.  By opposition, ``hg up`` only updates
-  the "working copy" by reading the local repository, i.e. it makes the
-  files that you see correspond to the latest (or any other) commit
-  locally present.
-
-* You should push often; there is no real reason not to.  Remember that
-  even if they are pushed, with the setup above, the commits are (1)
-  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
-  named.  Yes, they are publicly visible, but don't worry about someone
-  walking around the thousands of repos on bitbucket saying "hah, look
-  at the bad coding style of that guy".  Try to get into the mindset
-  that your work is not secret and it's fine that way.  We might not
-  accept it as is for PyPy, asking you instead to improve some things,
-  but we are not going to judge you.
-
-* The final step is to open a pull request, so that we know that you'd
-  like to merge that branch back to the original ``pypy/pypy`` repo.
-  This can also be done several times if you have interesting
-  intermediate states, but if you get there, then we're likely to
-  proceed to the next stage, which is...
-
-* Get a regular account for pushing directly to
-  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
-  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
-  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
-  then be pushed directly to the official repo, but (if you follow these
-  rules) they are still on a branch, and we can still review the
-  branches you want to merge.
-
-* If you get closer to the regular day-to-day development, you'll notice
-  that we generally push small changes as one or a few commits directly
-  to the branch ``default``.  Also, we often collaborate even if we are
-  on other branches, which do not really "belong" to anyone.  At this
-  point you'll need ``hg merge`` and learn how to resolve conflicts that
-  sometimes occur when two people try to push different commits in
-  parallel on the same branch.  But it is likely an issue for later ``:-)``
-
-.. _bitbucket: https://bitbucket.org/
-
-
-Running PyPy's unit tests
--------------------------
-
-PyPy development always was and is still thoroughly test-driven.
-We use the flexible `py.test testing tool`_ which you can `install independently
-<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
-
-The PyPy source tree comes with an inlined version of ``py.test``
-which you can invoke by typing::
-
-    python pytest.py -h
-
-This is usually equivalent to using an installed version::
-
-    py.test -h
-
-If you encounter problems with the installed version
-make sure you have the correct version installed which
-you can find out with the ``--version`` switch.
-
-You will need the `build requirements`_ to run tests successfully, since many of
-them compile little pieces of PyPy and then run the tests inside that minimal
-interpreter
-
-Now on to running some tests.  PyPy has many different test directories
-and you can use shell completion to point at directories or files::
-
-    py.test pypy/interpreter/test/test_pyframe.py
-
-    # or for running tests of a whole subdirectory
-    py.test pypy/interpreter/
-
-See `py.test usage and invocations`_ for some more generic info
-on how you can run tests.
-
-Beware trying to run "all" pypy tests by pointing to the root
-directory or even the top level subdirectory ``pypy``.  It takes
-hours and uses huge amounts of RAM and is not recommended.
-
-To run CPython regression tests you can point to the ``lib-python``
-directory::
-
-    py.test lib-python/2.7/test/test_datetime.py
-
-This will usually take a long time because this will run
-the PyPy Python interpreter on top of CPython.  On the plus
-side, it's usually still faster than doing a full translation
-and running the regression test with the translated PyPy Python
-interpreter.
-
-.. _py.test testing tool: http://pytest.org
-.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
-.. _`build requirements`: build.html#install-build-time-dependencies
-
-Special Introspection Features of the Untranslated Python Interpreter
----------------------------------------------------------------------
-
-If you are interested in the inner workings of the PyPy Python interpreter,
-there are some features of the untranslated Python interpreter that allow you
-to introspect its internals.
-
-
-Interpreter-level console
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To start interpreting Python with PyPy, install a C compiler that is
-supported by distutils and use Python 2.7 or greater to run PyPy::
-
-    cd pypy
-    python bin/pyinteractive.py
-
-After a few seconds (remember: this is running on top of CPython), you should
-be at the PyPy prompt, which is the same as the Python prompt, but with an
-extra ">".
-
-If you press
-<Ctrl-C> on the console you enter the interpreter-level console, a
-usual CPython console.  You can then access internal objects of PyPy
-(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
-prompt with the prefix ``w_``::
-
-    >>>> a = 123
-    >>>> <Ctrl-C>
-    *** Entering interpreter-level console ***
-    >>> w_a
-    W_IntObject(123)
-
-The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
-
-    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
-    >>> <Ctrl-D>
-    *** Leaving interpreter-level console ***
-
-    KeyboardInterrupt
-    >>>> l
-    [1, 'abc']
-
-Note that the prompt of the interpreter-level console is only '>>>' since
-it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
-Linux) or <Ctrl-Z>, <Enter> (under Windows).
-
-Also note that not all modules are available by default in this mode (for
-example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
-``--withmod-...`` command line options.
-
-You may be interested in reading more about the distinction between
-:ref:`interpreter-level and app-level <interpreter-level>`.
-
-pyinteractive.py options
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-To list the PyPy interpreter command line options, type::
-
-    cd pypy
-    python bin/pyinteractive.py --help
-
-pyinteractive.py supports most of the options that CPython supports too (in addition to a
-large amount of options that can be used to customize pyinteractive.py).
-As an example of using PyPy from the command line, you could type::
-
-    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
-
-Alternatively, as with regular Python, you can simply give a
-script name on the command line::
-
-    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
-
-The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
-default almost none of them are, because initializing them takes time.
-If you want anyway to enable all built-in modules, you can use
-``--allworkingmodules``.
-
-See our :doc:`configuration sections <config/index>` for details about what all the commandline
-options do.
-
-
-.. _trace example:
-
-Tracing bytecode and operations on objects
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can use a simple tracing mode to monitor the interpretation of
-bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
-PyPy console::
-
-    >>>> __pytrace__ = 1
-    Tracing enabled
-    >>>> x = 5
-            <module>:           LOAD_CONST    0 (5)
-            <module>:           STORE_NAME    0 (x)
-            <module>:           LOAD_CONST    1 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>> x
-            <module>:           LOAD_NAME    0 (x)
-            <module>:           PRINT_EXPR    0 
-    5
-            <module>:           LOAD_CONST    0 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>>
-
-
-Demos
------
-
-The `example-interpreter`_ repository contains an example interpreter
-written using the RPython translation toolchain.
-
-.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
-
-
-Additional Tools for running (and hacking) PyPy
------------------------------------------------
-
-We use some optional tools for developing PyPy. They are not required to run
-the basic tests or to get an interactive PyPy prompt but they help to
-understand  and debug PyPy especially for the translation process.
-
-
-graphviz & pygame for flow graph viewing (highly recommended)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-graphviz and pygame are both necessary if you
-want to look at generated flow graphs:
-
-	graphviz: http://www.graphviz.org/Download.php
-
-	pygame: http://www.pygame.org/download.shtml
-
-
-py.test and the py lib
-~~~~~~~~~~~~~~~~~~~~~~
-
-The `py.test testing tool`_ drives all our testing needs.
-
-We use the `py library`_ for filesystem path manipulations, terminal
-writing, logging and some other support  functionality.
-
-You don't necessarily need to install these two libraries because
-we also ship them inlined in the PyPy source tree.
-
-.. _py library: http://pylib.readthedocs.org/
-
-
-Getting involved
-----------------
-
-PyPy employs an open development process.  You are invited to join our
-`pypy-dev mailing list`_ or look at the other :ref:`contact
-possibilities <contact>`.  Usually we give out commit rights fairly liberally, so if you
-want to do something with PyPy, you can become a committer. We also run frequent
-coding sprints which are separately announced and often happen around Python
-conferences such as EuroPython or PyCon. Upcoming events are usually announced
-on `the blog`_.
-
-.. _the blog: http://morepypy.blogspot.com
-.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
-
-
-.. _start-reading-sources:
-
-Where to start reading the sources
-----------------------------------
-
-PyPy is made from parts that are relatively independent of each other.
-You should start looking at the part that attracts you most (all paths are
-relative to the PyPy top level directory).  You may look at our :doc:`directory reference <dir-reference>`
-or start off at one of the following points:
-
-*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
-   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
-   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
-   function objects and argument passing in :source:`pypy/interpreter/function.py`
-   and :source:`pypy/interpreter/argument.py`, the object space interface
-   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
-   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
-   Core types supporting the bytecode interpreter are defined in :source:`pypy/interpreter/typedef.py`.
-
-*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
-   and grammar files that allow it to parse the syntax of various Python
-   versions. Once the grammar has been processed, the parser can be
-   translated by the above machinery into efficient code.
-
-*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
-   contains a modified version of the compiler package from CPython
-   that fixes some bugs and is translatable.
-
-*  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
-   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
-   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
-   as a first approximation.  (Some types have multiple implementations.)
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
deleted file mode 100644
--- a/pypy/doc/how-to-contribute.rst
+++ /dev/null
@@ -1,93 +0,0 @@
-How to contribute to PyPy
-=========================
-
-This page describes how to contribute to the PyPy project. The first thing
-to remember is that PyPy project is very different than most projects out there.
-It's also different from a classic compiler project, so academic courses
-about compilers often don't apply or lead in the wrong direction.
-
-
-Don't just hack
----------------
-
-The first and most important rule how not to contribute to PyPy is
-"just hacking". This won't work. There are two major reasons why not
--- build times are large and PyPy has very thick layer separation which
-make it harder to "just hack a feature".
-
-
-Test driven development
------------------------
-
-Instead, we practice a lot of test driven development. This is partly because
-of very high quality requirements for compilers and partly because there is
-simply no other way to get around such complex project, that will keep you sane.
-There are probably people out there who are smart enough not to need it, we're
-not one of those. You may consider familiarizing yourself with `pytest`_,
-since this is a tool we use for tests.
-This leads to the next issue:
-
-.. _pytest: http://pytest.org/
-
-
-Layers
-------
-
-PyPy has layers. Just like Ogres or onions.
-Those layers help us keep the respective parts separated enough
-to be worked on independently and make the complexity manageable. This is,
-again, just a sanity requirement for such a complex project. For example writing
-a new optimization for the JIT usually does **not** involve touching a Python
-interpreter at all or the JIT assembler backend or the garbage collector.
-Instead it requires writing small tests in
-``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
-After that, you can just compile PyPy and things should just work.
-
-The short list of layers for further reading. For each of those layers, a good
-entry point is a test subdirectory in respective directories. It usually
-describes (better or worse) the interfaces between the submodules. For the
-``pypy`` subdirectory, most tests are small snippets of python programs that
-check for correctness (calls ``AppTestXxx``) that will call the appropriate
-part of the interpreter. For the ``rpython`` directory, most tests are small
-RPython interpreters that perform certain tasks. To see how they translate
-to low-level graphs, run them with ``--view``. To see small interpreters
-with a JIT compiler, use ``--viewloops`` option.
-
-* **python interpreter** - it's the part implemented in the ``pypy/`` directory.
-  It's implemented in RPython, which is a high level static language with
-  classes, garbage collection, just-in-time compiler generation and the ability
-  to call C. A cool part about it is that it can be run untranslated, so all
-  the tests are runnable without translating PyPy.
-
-  **interpreter** contains the interpreter core
-
-  **objspace** contains implementations of various objects exported to
-  the Python layer
-
-  **module** directory contains extension modules written in RPython
-
-* **rpython compiler** that resides in ``rpython/annotator`` and
-  ``rpython/rtyper`` directories. Consult `Getting Started with RPython`_
-  for further reading
-
-* **JIT generator** lives in ``rpython/jit`` directory. optimizations live
-  in ``rpython/jit/metainterp/optimizeopt``, the main JIT in
-  ``rpython/jit/metainterp`` (runtime part) and
-  ``rpython/jit/codewriter`` (translation-time part). Backends live in
-  ``rpython/jit/backend``.
-
-* **garbage collection** lives in ``rpython/memory``
-
-The rest of directories serve specific niche goal and are unlikely a good
-entry point.
-
-
-More documentation
-------------------
-
-* `Getting Started Developing With PyPy`_
-
-* `Getting Started with RPython`_
-
-.. _`Getting Started Developing With PyPy`: getting-started-dev.html
-.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -34,6 +34,7 @@
    whatsnew-2.0.0-beta1.rst
    whatsnew-1.9.rst
 
+
 CPython 3.5 compatible versions
 -------------------------------
 
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -9,7 +9,7 @@
 
 * If you're interested in trying PyPy out, check out the :doc:`installation instructions <install>`.
 
-* If you want to help develop PyPy, please have a look at :doc:`how to contribute <how-to-contribute>`
+* If you want to help develop PyPy, please have a look at :doc:`contributing <contributing>`
   and get in touch (:ref:`contact`)!
 
 All of the documentation and source code is available under the MIT license,
@@ -31,6 +31,7 @@
   introduction
   install
   build
+  windows
   faq
 
 
@@ -40,43 +41,30 @@
 ----------
 
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
   cpython_differences
   extending
-  embedding
   gc_info
   jit-hooks
   stackless
   __pypy__-module
-  objspace-proxies
   sandbox
   stm
-  windows
 
-
-.. _developing-pypy:
-
-Development documentation
--------------------------
+Development
+-----------
 
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
-  getting-started-dev
-  how-to-contribute
-  you-want-to-help
+  contributing
   architecture
   configuration
   project-ideas
   project-documentation
   how-to-release
 
-.. TODO: audit ^^
-
-
-.. TODO: Fill this in
-
 
 Further resources
 -----------------
@@ -84,13 +72,10 @@
 .. toctree::
   :maxdepth: 1
 
-  extradoc
-  eventhistory
-  discussions
   index-of-release-notes
   index-of-whatsnew
   contributor
-
+  glossary
 
 .. _contact:
 
@@ -118,7 +103,7 @@
     the `development mailing list`_.
 
 .. _#pypy on irc.freenode.net: irc://irc.freenode.net/pypy
-.. _here: http://www.tismer.com/pypy/irc-logs/pypy/
+.. _here: https://botbot.me/freenode/pypy/
 .. _Development mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
 .. _Commit mailing list: http://mail.python.org/mailman/listinfo/pypy-commit
 .. _Development bug/feature tracker: https://bitbucket.org/pypy/pypy/issues
diff --git a/pypy/doc/interpreter.rst b/pypy/doc/interpreter.rst
--- a/pypy/doc/interpreter.rst
+++ b/pypy/doc/interpreter.rst
@@ -102,7 +102,7 @@
 program flows with homogeneous name-value assignments on
 function invocations.
 
-.. _how-to guide for descriptors: http://users.rcn.com/python/download/Descriptor.htm
+.. _how-to guide for descriptors: https://docs.python.org/3/howto/descriptor.html
 
 
 Bytecode Interpreter Implementation Classes
diff --git a/pypy/doc/man/pypy3.1.rst b/pypy/doc/man/pypy3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/man/pypy3.1.rst
@@ -0,0 +1,135 @@
+=======
+ pypy3
+=======
+
+.. note: this is turned into a regular man page "pypy3.1" by
+   doing "make man" in pypy/doc/
+
+SYNOPSIS
+========
+
+``pypy3`` [*options*]
+[``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ]
+[*arg*\ ...]
+
+OPTIONS
+=======
+
+-i
+    Inspect interactively after running script.
+
+-O
+    Skip assert statements.
+
+-OO
+    Remove docstrings when importing modules in addition to ``-O``.
+
+-c CMD
+    Program passed in as ``CMD`` (terminates option list).
+
+-S
+    Do not ``import site`` on initialization.
+
+-s
+    Don't add the user site directory to `sys.path`.
+
+-u
+    Unbuffered binary ``stdout`` and ``stderr``.
+
+-h, --help
+    Show a help message and exit.
+
+-m MOD
+    Library module to be run as a script (terminates option list).
+
+-W ARG
+    Warning control (*arg* is *action*:*message*:*category*:*module*:*lineno*).
+
+-E
+    Ignore environment variables (such as ``PYTHONPATH``).
+
+-B
+    Disable writing bytecode (``.pyc``) files.
+
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
+--version
+    Print the PyPy version.
+
+--info
+    Print translation information about this PyPy executable.
+
+--jit ARG
+    Low level JIT parameters.  Mostly internal.  Run ``--jit help``
+    for more information.
+
+ENVIRONMENT
+===========
+
+``PYTHONPATH``
+    Add directories to pypy3's module search path.
+    The format is the same as shell's ``PATH``.
+
+``PYTHONSTARTUP``
+    A script referenced by this variable will be executed before the
+    first prompt is displayed, in interactive mode.
+
+``PYTHONDONTWRITEBYTECODE``
+    If set to a non-empty value, equivalent to the ``-B`` option.
+    Disable writing ``.pyc`` files.
+
+``PYTHONINSPECT``
+    If set to a non-empty value, equivalent to the ``-i`` option.
+    Inspect interactively after running the specified script.
+
+``PYTHONIOENCODING``
+    If this is set, it overrides the encoding used for
+    *stdin*/*stdout*/*stderr*.
+    The syntax is *encodingname*:*errorhandler*
+    The *errorhandler* part is optional and has the same meaning as in
+    `str.encode`.
+
+``PYTHONNOUSERSITE``
+    If set to a non-empty value, equivalent to the ``-s`` option.
+    Don't add the user site directory to `sys.path`.
+
+``PYTHONWARNINGS``
+    If set, equivalent to the ``-W`` option (warning control).
+    The value should be a comma-separated list of ``-W`` parameters.
+
+``PYPYLOG``
+    If set to a non-empty value, enable logging, the format is:
+
+    *fname* or *+fname*
+        logging for profiling: includes all
+        ``debug_start``/``debug_stop`` but not any nested
+        ``debug_print``.
+        *fname* can be ``-`` to log to *stderr*.
+        The *+fname* form can be used if there is a *:* in fname
+
+    ``:``\ *fname*
+        Full logging, including ``debug_print``.
+
+    *prefix*\ ``:``\ *fname*
+        Conditional logging.
+        Multiple prefixes can be specified, comma-separated.
+        Only sections whose name match the prefix will be logged.
+
+    ``PYPYLOG=jit-log-opt,jit-backend:logfile`` will
+    generate a log suitable for *jitviewer*, a tool for debugging
+    performance issues under PyPy.
+
+``PYPY_IRC_TOPIC``
+    If set to a non-empty value, print a random #pypy IRC
+    topic at startup of interactive mode.
+
+
+.. include:: ../gc_info.rst
+   :start-line: 7
+
+SEE ALSO
+========
+
+**python3**\ (1)
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -1,28 +1,7 @@
-What PyPy can do for your objects
-=================================
-
-.. contents::
-
-
-Thanks to the :doc:`Object Space <objspace>` architecture, any feature that is
-based on proxying, extending, changing or otherwise controlling the
-behavior of objects in a running program is easy to implement on top of PyPy.
-
-Here is what we have implemented so far, in historical order:
-
-* *Dump Object Space*: dumps all operations performed on all the objects
-  into a large log file.  For debugging your applications.
-
-* *Transparent Proxies Extension*: adds new proxy objects to
-  the Standard Object Space that enable applications to
-  control operations on application and builtin objects,
-  e.g lists, dictionaries, tracebacks.
-
-
 .. _tproxy:
 
-Transparent Proxies
--------------------
+Transparent Proxies (DEPRECATED)
+--------------------------------
 
 .. warning::
 
@@ -194,7 +173,7 @@
 application-level code.
 
 Transparent proxies are implemented on top of the :ref:`standard object
-space <standard-object-space>`, in :source:`pypy/objspace/std/proxy_helpers.py`,
+space <standard-object-space>`, in :source:`pypy/objspace/std/proxyobject.py`,
 :source:`pypy/objspace/std/proxyobject.py` and :source:`pypy/objspace/std/transparent.py`.
 To use them you will need to pass a `--objspace-std-withtproxy`_ option to ``pypy``
 or ``translate.py``.  This registers implementations named :py:class:`W_TransparentXxx`
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -474,8 +474,8 @@
 :source:`pypy/objspace/std/bytesobject.py` defines ``W_AbstractBytesObject``,
 which contains everything needed to build the ``str`` app-level type;
 and there are subclasses ``W_BytesObject`` (the usual string) and
-``W_StringBufferObject`` (a special implementation tweaked for repeated
-additions, in :source:`pypy/objspace/std/strbufobject.py`).  For mutable data
+``W_Buffer`` (a special implementation tweaked for repeated
+additions, in :source:`pypy/objspace/std/bufferobject.py`).  For mutable data
 types like lists and dictionaries, we have a single class
 ``W_ListObject`` or ``W_DictMultiObject`` which has an indirection to
 the real data and a strategy; the strategy can change as the content of
diff --git a/pypy/doc/project-documentation.rst b/pypy/doc/project-documentation.rst
--- a/pypy/doc/project-documentation.rst
+++ b/pypy/doc/project-documentation.rst
@@ -32,10 +32,13 @@
    coding-guide
    sprint-reports
    extradoc
+   eventhistory
    video-index
    index-report
+   discussions
    dev_method
-   glossary
+   embedding
+   objspace-proxies
 
 
 Source Code Documentation
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,8 +5,19 @@
 .. this is a revision shortly after release-pypy-6.0.0
 .. startrev: e50e11af23f1
 
+.. branch: cppyy-packaging
 
+Upgrade to backend 0.6.0, support exception handling from wrapped functions,
+update enum handling, const correctness for data members and associated tests,
+support anonymous enums, support for function pointer arguments
 
+.. branch: socket_default_timeout_blockingness
+
+Make sure 'blocking-ness' of socket is set along with default timeout
+
+.. branch: crypt_h
+
+Include crypt.h for crypt() on Linux
 
 .. branch: unicode-utf8-re
 .. branch: utf8-io
diff --git a/pypy/doc/whatsnew-pypy2-5.10.0.rst b/pypy/doc/whatsnew-pypy2-5.10.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.10.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst
@@ -32,6 +32,7 @@
 
 .. branch: fix-vmprof-stacklet-switch
 .. branch: fix-vmprof-stacklet-switch-2
+
 Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...)
 
 .. branch: win32-vcvars
@@ -39,8 +40,3 @@
 .. branch: rdict-fast-hash
 
 Make it possible to declare that the hash function of an r_dict is fast in RPython.
-
-.. branch: unicode-utf8-re
-.. branch: utf8-io
-Utf8 handling for unicode
-
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -25,7 +25,7 @@
 
 This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has
 made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
-was checked in Nov 2016). Note that the compiler suite may be installed in
+was checked in May 2018). Note that the compiler suite may be installed in
 ``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``
 or in
 ``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``.
@@ -37,7 +37,7 @@
 by running the mt.exe command by hand from a DOS window (that is how the author
 discovered the problem).
 
-.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
+.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/EN-US/DOWNLOAD/DETAILS.ASPX?ID=44266
 
 Installing "Build Tools for Visual Studio 2017" (for Python 3)
 --------------------------------------------------------------
@@ -150,7 +150,7 @@
 build dependencies for windows. As part of the `rpython` setup stage, environment
 variables will be set to use these dependencies. The repository has a README
 file on how to replicate, and a branch for each supported platform. You may run
- the `get_externals.py` utility to checkout the proper branch for your platform
+the `get_externals.py` utility to checkout the proper branch for your platform
 and PyPy version.
 
 .. _repository:  https://bitbucket.org/pypy/external
diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
deleted file mode 100644
--- a/pypy/doc/you-want-to-help.rst
+++ /dev/null
@@ -1,81 +0,0 @@
-You want to help with PyPy, now what?
-=====================================
-
-PyPy is a very large project that has a reputation of being hard to dive into.
-Some of this fame is warranted, some of it is purely accidental. There are three
-important lessons that everyone willing to contribute should learn:
-
-* PyPy has layers. There are many pieces of architecture that are very well
-  separated from each other. More about this below, but often the manifestation
-  of this is that things are at a different layer than you would expect them
-  to be. For example if you are looking for the JIT implementation, you will
-  not find it in the implementation of the Python programming language.
-
-* Because of the above, we are very serious about Test Driven Development.
-  It's not only what we believe in, but also that PyPy's architecture is
-  working very well with TDD in mind and not so well without it. Often
-  development means progressing in an unrelated corner, one unittest
-  at a time; and then flipping a giant switch, bringing it all together.
-  (It generally works out of the box.  If it doesn't, then we didn't
-  write enough unit tests.)  It's worth repeating - PyPy's
-  approach is great if you do TDD, and not so great otherwise.
-
-* PyPy uses an entirely different set of tools - most of them included
-  in the PyPy repository. There is no Makefile, nor autoconf. More below.
-
-
-Architecture
-------------
-
-PyPy has layers. The 100 miles view:
-
-* :ref:`RPython <rpython:language>` is the language in which we write interpreters. Not the entire
-  PyPy project is written in RPython, only the parts that are compiled in
-  the translation process. The interesting point is that RPython has no parser,
-  it's compiled from the live python objects, which makes it possible to do
-  all kinds of metaprogramming during import time. In short, Python is a meta
-  programming language for RPython.
-
-  The RPython standard library is to be found in the ``rlib`` subdirectory.
-
-* The translation toolchain - this is the part that takes care of translating
-  RPython to flow graphs and then to C. There is more in the :doc:`architecture <architecture>`
-  document written about it.
-
-  It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
-  and ``rtyper``.
-
-* Python Interpreter and modules
-
-  This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
-  interpreter for Python written in RPython.  The fact that it is
-  RPython is not apparent at first.  Built-in modules are written in
-  ``pypy/module/*``.  Some modules that CPython implements in C are
-  simply written in pure Python; they are in the top-level ``lib_pypy``
-  directory.  The standard library of Python (with a few changes to
-  accomodate PyPy) is in ``lib-python``.
-
-* :ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
-  interpreter written in RPython, rather than the user program that it
-  interprets.  As a result it applies to any interpreter, i.e. any
-  language.  But getting it to work correctly is not trivial: it
-  requires a small number of precise "hints" and possibly some small
-  refactorings of the interpreter.  The JIT itself also has several
-  almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
-  optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
-  residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
-  that turns it into machine code.  Writing a new backend is a
-  traditional way to get into the project.
-
-* Garbage Collectors (GC): as you may notice if you are used to CPython's
-  C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
-  :ref:`rpython:garbage-collection` is inserted
-  during translation.  Moreover, this is not reference counting; it is a real
-  GC written as more RPython code.  The best one we have so far is in
-  ``rpython/memory/gc/incminimark.py``.
-
-
-Toolset
--------
-
-xxx
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -8,7 +8,7 @@
 from rpython.tool.sourcetools import func_with_new_name
 
 
-class W_StringBuilder(W_Root):
+class W_BytesBuilder(W_Root):
     def __init__(self, space, size):
         if size < 0:
             self.builder = StringBuilder()
@@ -17,7 +17,7 @@
 
     @unwrap_spec(size=int)
     def descr__new__(space, w_subtype, size=-1):
-        return W_StringBuilder(space, size)
+        return W_BytesBuilder(space, size)
 
     @unwrap_spec(s='bytes')
     def descr_append(self, space, s):
@@ -41,16 +41,17 @@
             raise oefmt(space.w_ValueError, "no length of built builder")
         return space.newint(self.builder.getlength())
 
-W_StringBuilder.typedef = TypeDef("StringBuilder",
+W_BytesBuilder.typedef = TypeDef("StringBuilder",
     __new__ = interp2app(func_with_new_name(
-                                W_StringBuilder.descr__new__.im_func,
-                                'StringBuilder_new')),
-    append = interp2app(W_StringBuilder.descr_append),
-    append_slice = interp2app(W_StringBuilder.descr_append_slice),
-    build = interp2app(W_StringBuilder.descr_build),
-    __len__ = interp2app(W_StringBuilder.descr_len),
+                                W_BytesBuilder.descr__new__.im_func,
+                                'BytesBuilder_new')),
+    append = interp2app(W_BytesBuilder.descr_append),
+    append_slice = interp2app(W_BytesBuilder.descr_append_slice),
+    build = interp2app(W_BytesBuilder.descr_build),
+    __len__ = interp2app(W_BytesBuilder.descr_len),
 )
-W_StringBuilder.typedef.acceptable_as_base_class = False
+W_BytesBuilder.typedef.acceptable_as_base_class = False
+W_StringBuilder = W_BytesBuilder
 
 class W_UnicodeBuilder(W_Root):
     def __init__(self, space, size):
diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py
--- a/pypy/module/_cppyy/__init__.py
+++ b/pypy/module/_cppyy/__init__.py
@@ -7,6 +7,7 @@
     interpleveldefs = {
         '_resolve_name'          : 'interp_cppyy.resolve_name',
         '_scope_byname'          : 'interp_cppyy.scope_byname',
+        '_is_static_data'        : 'interp_cppyy.is_static_data',
         '_is_template'           : 'interp_cppyy.is_template',
         '_std_string_name'       : 'interp_cppyy.std_string_name',
         '_set_class_generator'   : 'interp_cppyy.set_class_generator',
@@ -21,7 +22,7 @@
     }
 
     appleveldefs = {
-        '_init_pythonify'        : 'pythonify._init_pythonify',
+        '_post_import_startup'   : 'pythonify._post_import_startup',
         'add_pythonization'      : 'pythonify.add_pythonization',
         'Template'               : 'pythonify.CPPTemplate',
     }
@@ -34,9 +35,3 @@
         # code generation is not, so give it a chance to run now
         from pypy.module._cppyy import capi
         capi.register_pythonizations(space)
-
-    def startup(self, space):
-        from pypy.module._cppyy import capi
-        capi.verify_backend(space)      # may raise ImportError
-
-        space.call_method(self, '_init_pythonify')
diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -308,7 +308,7 @@
         c_call = state.capi_calls[name]
     except KeyError:
         if state.backend is None:
-            load_backend(space)
+            verify_backend(space)
         iface = state.capi_call_ifaces[name]
         cfunc = W_RCTypeFunc(space, iface[0], iface[1], False)
         c_call = state.backend.load_function(cfunc, 'cppyy_'+name)
@@ -421,7 +421,7 @@
         _cdata_to_ptr(space, call_capi(space, 'function_address_from_index', args)))
 def c_function_address_from_method(space, cppmethod):
     return rffi.cast(C_FUNC_PTR,
-        _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', _ArgH(cppmethod))))
+        _cdata_to_ptr(space, call_capi(space, 'function_address_from_method', [_ArgH(cppmethod)])))
 
 # handling of function argument buffer ---------------------------------------
 def c_allocate_function_args(space, size):
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -686,6 +686,34 @@
         decref(space, rffi.cast(PyObject, rffi.cast(rffi.VOIDPP, arg)[0]))
 
 
+class FunctionPointerConverter(TypeConverter):
+    _immutable_fields_ = ['signature']
+
+    def __init__(self, space, signature):
+        self.signature = signature
+
+    def convert_argument(self, space, w_obj, address, call_local):
+        # TODO: atm, does not actually get an overload, but a staticmethod
+        from pypy.module._cppyy.interp_cppyy import W_CPPOverload
+        cppol = space.interp_w(W_CPPOverload, w_obj)
+
+        # find the function with matching signature
+        for i in range(len(cppol.functions)):
+            m = cppol.functions[i]
+            if m.signature(False) == self.signature:
+                x = rffi.cast(rffi.VOIDPP, address)
+                x[0] = rffi.cast(rffi.VOIDP,
+                    capi.c_function_address_from_method(space, m.cppmethod))
+                address = rffi.cast(capi.C_OBJECT, address)
+                ba = rffi.cast(rffi.CCHARP, address)
+                ba[capi.c_function_arg_typeoffset(space)] = 'p'
+                return
+
+        # lookup failed
+        raise oefmt(space.w_TypeError,
+                    "no overload found matching %s", self.signature)
+
+
 class MacroConverter(TypeConverter):
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         # TODO: get the actual type info from somewhere ...
@@ -749,6 +777,14 @@
             return InstancePtrPtrConverter(space, clsdecl)
         elif compound == "":
             return InstanceConverter(space, clsdecl)
+    elif "(anonymous)" in name:
+        # special case: enum w/o a type name
+        return _converters["internal_enum_type_t"](space, default)
+    elif "(*)" in name or "::*)" in name:
+        # function pointer
+        pos = name.find("*)")
+        if pos > 0:
+            return FunctionPointerConverter(space, name[pos+2:])
 
     #   5) void* or void converter (which fails on use)
     if 0 <= compound.find('*'):
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -289,6 +289,9 @@
             return InstancePtrExecutor(space, cppclass)
         elif compound == '**' or compound == '*&':
             return InstancePtrPtrExecutor(space, cppclass)
+    elif "(anonymous)" in name:
+        # special case: enum w/o a type name
+        return _executors["internal_enum_type_t"](space, None)
 
     # 4) additional special cases
     if compound == '*':
diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -83,8 +83,8 @@
             if len(value) != 1:
                 raise oefmt(space.w_ValueError,
                         "char expected, got string of size %d", len(value))
-            value = rffi.cast(rffi.CHAR, value[0])
 
+        value = rffi.cast(rffi.CHAR, value[0])
         return value     # turn it into a "char" to the annotator
 
     def cffi_type(self, space):
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -128,7 +128,7 @@
 
 def register_class(space, w_pycppclass):
     w_cppclass = space.findattr(w_pycppclass, space.newtext("__cppdecl__"))
-    cppclass = space.interp_w(W_CPPClassDecl, w_cppclass, can_be_None=False)
+    cppclass = space.interp_w(W_CPPClassDecl, w_cppclass)
     # add back-end specific method pythonizations (doing this on the wrapped
     # class allows simple aliasing of methods)
     capi.pythonize(space, cppclass.name, w_pycppclass)
@@ -149,6 +149,24 @@
 W_CPPLibrary.typedef.acceptable_as_base_class = True
 
 
+#-----
+# Classes involved with methods and functions:
+#
+#  CPPMethod:         base class wrapping a single function or method
+#  CPPConstructor:    specialization for allocating a new object
+#  CPPFunction:       specialization for free and static functions
+#  CPPSetItem:        specialization for Python's __setitem__
+#  CPPTemplatedCall:  trampoline to instantiate and bind templated functions
+#  W_CPPOverload, W_CPPConstructorOverload, W_CPPTemplateOverload:
+#     user-facing, app-level, collection of overloads, with specializations
+#     for constructors and templates
+#  W_CPPBoundMethod:  instantiated template method
+#
+# All methods/functions derive from CPPMethod and are collected as overload
+# candidates in user-facing overload classes. Templated methods are a two-step
+# process, where first the template is instantiated (or selected if already
+# available), which returns a callable object that is the actual bound method.
+
 class CPPMethod(object):
     """Dispatcher of methods. Checks the arguments, find the corresponding FFI
     function if available, makes the call, and returns the wrapped result. It
@@ -177,7 +195,7 @@
 
     @staticmethod
     def unpack_cppthis(space, w_cppinstance, declaring_scope):
-        cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False)
+        cppinstance = space.interp_w(W_CPPInstance, w_cppinstance)
         cppinstance._nullcheck()
         return cppinstance.get_cppthis(declaring_scope)
 
@@ -424,7 +442,7 @@
 
 
 class CPPFunction(CPPMethod):
-    """Global (namespaced) function dispatcher."""
+    """Global (namespaced) / static function dispatcher."""
 
     _immutable_ = True
 
@@ -688,6 +706,18 @@
 )
 
 
+#-----
+# Classes for data members:
+#
+#  W_CPPDataMember:        instance data members
+#  W_CPPConstDataMember:   specialization for const data members
+#  W_CPPStaticData:        class-level and global/static data
+#  W_CPPConstStaticData:   specialization for const global/static data
+#
+# Data is represented by an offset which is either a global pointer (static data)
+# or an offset from the start of an instance (data members). The "const"
+# specializations raise when attempting to set their value.
+
 class W_CPPDataMember(W_Root):
     _attrs_ = ['space', 'scope', 'converter', 'offset']
     _immutable_fields = ['scope', 'converter', 'offset']
@@ -698,9 +728,6 @@
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
 
-    def is_static(self):
-        return self.space.w_False
-
     def _get_offset(self, cppinstance):
         if cppinstance:
             assert lltype.typeOf(cppinstance.clsdecl.handle) == lltype.typeOf(self.scope.handle)
@@ -728,16 +755,25 @@
 
 W_CPPDataMember.typedef = TypeDef(
     'CPPDataMember',
-    is_static = interp2app(W_CPPDataMember.is_static),
     __get__ = interp2app(W_CPPDataMember.get),
     __set__ = interp2app(W_CPPDataMember.set),
 )
 W_CPPDataMember.typedef.acceptable_as_base_class = False
 
+
+class W_CPPConstDataMember(W_CPPDataMember):
+    def set(self, w_cppinstance, w_value):
+        raise oefmt(self.space.w_TypeError, "assignment to const data not allowed")
+
+W_CPPConstDataMember.typedef = TypeDef(
+    'CPPConstDataMember',
+    __get__ = interp2app(W_CPPDataMember.get),
+    __set__ = interp2app(W_CPPConstDataMember.set),
+)
+W_CPPConstDataMember.typedef.acceptable_as_base_class = False
+
+
 class W_CPPStaticData(W_CPPDataMember):
-    def is_static(self):
-        return self.space.w_True
-
     @jit.elidable_promote()
     def _get_offset(self, cppinstance):
         return self.offset
@@ -751,19 +787,34 @@
 
 W_CPPStaticData.typedef = TypeDef(
     'CPPStaticData',
-    is_static = interp2app(W_CPPStaticData.is_static),
     __get__ = interp2app(W_CPPStaticData.get),
     __set__ = interp2app(W_CPPStaticData.set),
 )
 W_CPPStaticData.typedef.acceptable_as_base_class = False
 
-def is_static(space, w_obj):
+
+class W_CPPConstStaticData(W_CPPStaticData):
+    def set(self, w_cppinstance, w_value):
+        raise oefmt(self.space.w_TypeError, "assignment to const data not allowed")
+
+W_CPPConstStaticData.typedef = TypeDef(
+    'CPPConstStaticData',
+    __get__ = interp2app(W_CPPConstStaticData.get),
+    __set__ = interp2app(W_CPPConstStaticData.set),
+)
+W_CPPConstStaticData.typedef.acceptable_as_base_class = False
+
+
+def is_static_data(space, w_obj):
     try:

From pypy.commits at gmail.com  Mon May 14 16:50:41 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 13:50:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge py3.5
Message-ID: <5af9f6a1.1c69fb81.79ea9.4be6@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94584:f2eac1569fb6
Date: 2018-05-14 22:49 +0200
http://bitbucket.org/pypy/pypy/changeset/f2eac1569fb6/

Log:	merge py3.5

diff too long, truncating to 2000 out of 5543 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -51,3 +51,5 @@
 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0
 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0
 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1
+ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0
+fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -247,6 +247,7 @@
   Lukas Vacek
   Omer Katz
   Jacek Generowicz
+  Tomasz Dziopa
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -307,6 +308,7 @@
   Yury V. Zaytsev
   florinpapa
   Anders Sigfridsson
+  Matt Jackson
   Nikolay Zinov
   rafalgalczynski at gmail.com
   Joshua Gilbert
diff --git a/dotviewer/font/NOTICE b/dotviewer/font/COPYING.txt
rename from dotviewer/font/NOTICE
rename to dotviewer/font/COPYING.txt
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -7,6 +7,9 @@
 import time as _time
 import math as _math
 
+# for cpyext, use these as base classes
+from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop
+
 def _cmp(x, y):
     return 0 if x == y else 1 if x > y else -1
 
@@ -332,8 +335,7 @@
 
     return q
 
-
-class timedelta:
+class timedelta(deltainterop):
     """Represent the difference between two datetime objects.
 
     Supported operators:
@@ -446,7 +448,7 @@
         if abs(d) > 999999999:
             raise OverflowError("timedelta # of days is too large: %d" % d)
 
-        self = object.__new__(cls)
+        self = deltainterop.__new__(cls)
         self._days = d
         self._seconds = s
         self._microseconds = us
@@ -655,7 +657,7 @@
                           microseconds=999999)
 timedelta.resolution = timedelta(microseconds=1)
 
-class date:
+class date(dateinterop):
     """Concrete date type.
 
     Constructors:
@@ -695,12 +697,12 @@
         if month is None and isinstance(year, bytes) and len(year) == 4 and \
                 1 <= year[2] <= 12:
             # Pickle support
-            self = object.__new__(cls)
+            self = dateinterop.__new__(cls)
             self.__setstate(year)
             self._hashcode = -1
             return self
         year, month, day = _check_date_fields(year, month, day)
-        self = object.__new__(cls)
+        self = dateinterop.__new__(cls)
         self._year = year
         self._month = month
         self._day = day
@@ -1026,7 +1028,7 @@
 
 _tzinfo_class = tzinfo
 
-class time:
+class time(timeinterop):
     """Time with time zone.
 
     Constructors:
@@ -1063,14 +1065,14 @@
         """
         if isinstance(hour, bytes) and len(hour) == 6 and hour[0]&0x7F < 24:
             # Pickle support
-            self = object.__new__(cls)
+            self = timeinterop.__new__(cls)
             self.__setstate(hour, minute or None)
             self._hashcode = -1
             return self
         hour, minute, second, microsecond, fold = _check_time_fields(
             hour, minute, second, microsecond, fold)
         _check_tzinfo_arg(tzinfo)
-        self = object.__new__(cls)
+        self = timeinterop.__new__(cls)
         self._hour = hour
         self._minute = minute
         self._second = second
@@ -1376,7 +1378,7 @@
                 microsecond=0, tzinfo=None, *, fold=0):
         if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2]&0x7F <= 12:
             # Pickle support
-            self = object.__new__(cls)
+            self = dateinterop.__new__(cls)
             self.__setstate(year, month)
             self._hashcode = -1
             return self
@@ -1384,7 +1386,7 @@
         hour, minute, second, microsecond, fold = _check_time_fields(
             hour, minute, second, microsecond, fold)
         _check_tzinfo_arg(tzinfo)
-        self = object.__new__(cls)
+        self = dateinterop.__new__(cls)
         self._year = year
         self._month = month
         self._day = day
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -63,39 +63,9 @@
 
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
-    so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
-
-    g = {}
-    g['CC'] = "cc -pthread"
-    g['CXX'] = "c++ -pthread"
-    g['OPT'] = "-DNDEBUG -O2"
-    g['CFLAGS'] = "-DNDEBUG -O2"
-    g['CCSHARED'] = "-fPIC"
-    g['LDSHARED'] = "cc -pthread -shared"
-    g['EXT_SUFFIX'] = so_ext
-    g['SHLIB_SUFFIX'] = ".so"
-    g['SO'] = so_ext  # deprecated in Python 3, for backward compatibility
-    g['AR'] = "ar"
-    g['ARFLAGS'] = "rc"
-    g['EXE'] = ""
-    g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['VERSION'] = get_python_version()
-
-    if sys.platform[:6] == "darwin":
-        import platform
-        if platform.machine() == 'i386':
-            if platform.architecture()[0] == '32bit':
-                arch = 'i386'
-            else:
-                arch = 'x86_64'
-        else:
-            # just a guess
-            arch = platform.machine()
-        g['LDSHARED'] += ' -undefined dynamic_lookup'
-        g['CC'] += ' -arch %s' % (arch,)
-
+    from _sysconfigdata import build_time_vars
     global _config_vars
-    _config_vars = g
+    _config_vars = build_time_vars
 
 
 def _init_nt():
@@ -221,4 +191,3 @@
 
 from .sysconfig_cpython import (
     parse_makefile, _variable_rx, expand_makefile_vars)
-
diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -20,30 +20,30 @@
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{installed_base}/lib/python{py_version_short}',
-        'platstdlib': '{platbase}/lib/python{py_version_short}',
-        'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
+        'stdlib': '{installed_base}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{platbase}/lib/{implementation_lower}{py_version_short}/site-packages',
         'include':
-            '{installed_base}/include/python{py_version_short}{abiflags}',
+            '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
         'platinclude':
-            '{installed_platbase}/include/python{py_version_short}{abiflags}',
+            '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'posix_home': {
-        'stdlib': '{installed_base}/lib/python',
-        'platstdlib': '{base}/lib/python',
-        'purelib': '{base}/lib/python',
-        'platlib': '{base}/lib/python',
-        'include': '{installed_base}/include/python',
-        'platinclude': '{installed_base}/include/python',
+        'stdlib': '{installed_base}/lib/{implementation_lower}',
+        'platstdlib': '{base}/lib/{implementation_lower}',
+        'purelib': '{base}/lib/{implementation_lower}',
+        'platlib': '{base}/lib/{implementation_lower}',
+        'include': '{installed_base}/include/{implementation_lower}',
+        'platinclude': '{installed_base}/include/{implementation_lower}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'pypy': {
-        'stdlib': '{installed_base}/lib-python',
-        'platstdlib': '{base}/lib-python',
+        'stdlib': '{installed_base}/lib-{implementation_lower}',
+        'platstdlib': '{base}/lib-{implementation_lower}',
         'purelib': '{base}/site-packages',
         'platlib': '{base}/site-packages',
         'include': '{installed_base}/include',
@@ -62,28 +62,28 @@
         'data': '{base}',
         },
     'nt_user': {
-        'stdlib': '{userbase}/Python{py_version_nodot}',
-        'platstdlib': '{userbase}/Python{py_version_nodot}',
-        'purelib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'include': '{userbase}/Python{py_version_nodot}/Include',
-        'scripts': '{userbase}/Python{py_version_nodot}/Scripts',
+        'stdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'platstdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'purelib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'platlib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'include': '{userbase}/{implementation}{py_version_nodot}/Include',
+        'scripts': '{userbase}/{implementation}{py_version_nodot}/Scripts',
         'data': '{userbase}',
         },
     'posix_user': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
+        'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{userbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
         },
     'osx_framework_user': {
-        'stdlib': '{userbase}/lib/python',
-        'platstdlib': '{userbase}/lib/python',
-        'purelib': '{userbase}/lib/python/site-packages',
-        'platlib': '{userbase}/lib/python/site-packages',
+        'stdlib': '{userbase}/lib/{implementation_lower}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}',
+        'purelib': '{userbase}/lib/{implementation_lower}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}/site-packages',
         'include': '{userbase}/include',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
@@ -106,6 +106,11 @@
 _USER_BASE = None
 
 
+def _get_implementation():
+    if '__pypy__' in sys.builtin_module_names:
+        return 'PyPy'
+    return 'Python'
+
 def _safe_realpath(path):
     try:
         return realpath(path)
@@ -556,6 +561,8 @@
         except AttributeError:
             # sys.abiflags may not be defined on all platforms.
             _CONFIG_VARS['abiflags'] = ''
+        _CONFIG_VARS['implementation'] = _get_implementation()
+        _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
 
         if os.name == 'nt':
             _init_non_posix(_CONFIG_VARS)
@@ -731,6 +738,8 @@
     _print_dict('Paths', get_paths())
     print()
     _print_dict('Variables', get_config_vars())
+    print
+    _print_dict('User', get_paths('%s_user' % os.name))
 
 
 if __name__ == '__main__':
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -4,6 +4,7 @@
 import subprocess
 import shutil
 from copy import copy
+from distutils.spawn import find_executable
 
 from test.support import (run_unittest,
                           import_module, TESTFN, unlink, check_warnings,
@@ -298,6 +299,30 @@
 
         self.assertIn(ldflags, ldshared)
 
+    @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows")
+    def test_cc_values(self):
+        """ CC and CXX should be set for pypy """
+        for var in ["CC", "CXX"]:
+            assert sysconfig.get_config_var(var) is not None
+
+    @unittest.skipIf(not find_executable("gcc"),
+        "Does not apply to machines without gcc installed"
+    )
+    def test_gcc_values(self):
+        """ if gcc is installed on the box, gcc values should be set. """
+        assert "gcc" in sysconfig.get_config_var("CC")
+        assert sysconfig.get_config_var("GNULD") == "yes"
+        assert "gcc" in sysconfig.get_config_var("LDSHARED")
+
+
+    @unittest.skipIf(not find_executable("g++"),
+        "Does not apply to machines without g++ installed"
+    )
+    def test_gplusplus_values(self):
+        """ if g++ is installed on the box, g++ values should be set. """
+        assert "g++" in sysconfig.get_config_var("CXX")
+
+
     @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX")
     def test_platform_in_subprocess(self):
         my_platform = sysconfig.get_platform()
diff --git a/lib-python/3/test/test_sysconfig_pypy.py b/lib-python/3/test/test_sysconfig_pypy.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/test_sysconfig_pypy.py
@@ -0,0 +1,17 @@
+import os
+import sys
+import unittest
+import site
+
+
+class TestSysConfigPypy(unittest.TestCase):
+    def test_install_schemes(self):
+        # User-site etc. paths should have "pypy" and not "python"
+        # inside them.
+        if site.ENABLE_USER_SITE:
+            parts = site.USER_SITE.lower().split(os.path.sep)
+            assert any(x.startswith('pypy') for x in parts[-2:]), parts
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
--- a/lib_pypy/_cffi_ssl/README.md
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -14,6 +14,8 @@
 
 * ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1)
 
+* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates()
+
 
 # Tests?
 
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py
@@ -0,0 +1,84 @@
+#
+# An extra bit of logic for the Win32-only functionality that is missing from the
+# version from cryptography.
+#
+
+import sys
+
+INCLUDES = """
+#include <Wincrypt.h>
+"""
+
+TYPES = """
+typedef ... *HCERTSTORE;
+typedef ... *HCRYPTPROV_LEGACY;
+
+typedef struct {
+    DWORD      dwCertEncodingType;
+    BYTE       *pbCertEncoded;
+    DWORD      cbCertEncoded;
+    ...;
+} CERT_CONTEXT, *PCCERT_CONTEXT;
+
+typedef struct {
+    DWORD      dwCertEncodingType;
+    BYTE       *pbCrlEncoded;
+    DWORD      cbCrlEncoded;
+    ...;
+} CRL_CONTEXT, *PCCRL_CONTEXT;
+
+typedef struct {
+    DWORD cUsageIdentifier;
+    LPSTR *rgpszUsageIdentifier;
+    ...;
+} CERT_ENHKEY_USAGE, *PCERT_ENHKEY_USAGE;
+"""
+
+FUNCTIONS = """
+HCERTSTORE WINAPI CertOpenStore(
+         LPCSTR            lpszStoreProvider,
+         DWORD             dwMsgAndCertEncodingType,
+         HCRYPTPROV_LEGACY hCryptProv,
+         DWORD             dwFlags,
+         const char        *pvPara
+);
+PCCERT_CONTEXT WINAPI CertEnumCertificatesInStore(
+         HCERTSTORE     hCertStore,
+         PCCERT_CONTEXT pPrevCertContext
+);
+BOOL WINAPI CertFreeCertificateContext(
+         PCCERT_CONTEXT pCertContext
+);
+BOOL WINAPI CertFreeCRLContext(
+         PCCRL_CONTEXT pCrlContext
+);
+BOOL WINAPI CertCloseStore(
+         HCERTSTORE hCertStore,
+         DWORD      dwFlags
+);
+BOOL WINAPI CertGetEnhancedKeyUsage(
+         PCCERT_CONTEXT     pCertContext,
+         DWORD              dwFlags,
+         PCERT_ENHKEY_USAGE pUsage,
+         DWORD              *pcbUsage
+);
+PCCRL_CONTEXT WINAPI CertEnumCRLsInStore(
+         HCERTSTORE    hCertStore,
+         PCCRL_CONTEXT pPrevCrlContext
+);
+"""
+
+MACROS = """
+#define CERT_STORE_READONLY_FLAG ...
+#define CERT_SYSTEM_STORE_LOCAL_MACHINE ...
+#define CRYPT_E_NOT_FOUND ...
+#define CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG ...
+#define CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG ...
+#define X509_ASN_ENCODING ...
+#define PKCS_7_ASN_ENCODING ...
+
+static const LPCSTR CERT_STORE_PROV_SYSTEM_A;
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -24,6 +24,7 @@
 from enum import IntEnum as _IntEnum
 
 if sys.platform == 'win32':
+    from _cffi_ssl._stdssl.win32_extra import enum_certificates, enum_crls
     HAVE_POLL = False
 else:
     from select import poll, POLLIN, POLLOUT
diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
@@ -0,0 +1,101 @@
+from _pypy_openssl import lib, ffi
+
+
+def enum_certificates(store_name):
+    """Retrieve certificates from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type, trust) tuples.  The encoding_type flag can be interpreted
+with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
+a set of OIDs or the boolean True.
+    """
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name, "ascii"))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+    
+    result = []
+    pCertCtx = ffi.NULL
+    try:
+        while True:
+            pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
+            if pCertCtx == ffi.NULL:
+                break
+            cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:]
+            enc = certEncodingType(pCertCtx.dwCertEncodingType)
+            keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
+            if keyusage is True:
+                keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
+            result.append((cert, enc, keyusage))
+    finally:
+        if pCertCtx != ffi.NULL:
+            lib.CertFreeCertificateContext(pCertCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def enum_crls(store_name):
+    """Retrieve CRLs from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type) tuples.  The encoding_type flag can be interpreted with
+X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name, "ascii"))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+
+    result = []
+    pCrlCtx = ffi.NULL
+    try:
+        while True:
+            pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
+            if pCrlCtx == ffi.NULL:
+                break
+            crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
+            enc = certEncodingType(pCrlCtx.dwCertEncodingType)
+            result.append((crl, enc))
+    finally:
+        if pCrlCtx != ffi.NULL:
+            lib.CertFreeCRLContext(pCrlCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def certEncodingType(encodingType):
+    if encodingType == lib.X509_ASN_ENCODING:
+        return "x509_asn"
+    if encodingType == lib.PKCS_7_ASN_ENCODING:
+        return "pkcs_7_asn"
+    return encodingType
+
+def parseKeyUsage(pCertCtx, flags):
+    pSize = ffi.new("DWORD *")
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    pUsageMem = ffi.new("char[]", pSize[0])
+    pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    retval = set()
+    for i in range(pUsage.cUsageIdentifier):
+        if pUsage.rgpszUsageIdentifier[i]:
+            oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
+            retval.add(oid)
+    return retval
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -81,8 +81,11 @@
     def _CData_output(self, resarray, base=None, index=-1):
         from _rawffi.alt import types
         # If a char_p or unichar_p is received, skip the string interpretation
-        if base._ffiargtype != types.Pointer(types.char_p) and \
-           base._ffiargtype != types.Pointer(types.unichar_p):
+        try:
+            deref = type(base)._deref_ffiargtype()
+        except AttributeError:
+            deref = None
+        if deref != types.char_p and deref != types.unichar_p:
             # this seems to be a string if we're array of char, surprise!
             from ctypes import c_char, c_wchar
             if self._type_ is c_char:
@@ -127,6 +130,12 @@
                 value = self(*value)
         return _CDataMeta.from_param(self, value)
 
+    def _build_ffiargtype(self):
+        return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+    def _deref_ffiargtype(self):
+        return self._type_.get_ffi_argtype()
+
 def array_get_slice_params(self, index):
     if hasattr(self, '_length_'):
         start, stop, step = index.indices(self._length_)
@@ -254,6 +263,5 @@
             _type_ = base
         )
         cls = ArrayMeta(name, (Array,), tpdict)
-        cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype())
         ARRAY_CACHE[key] = cls
         return cls
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -49,10 +49,13 @@
         else:
             return self.from_param(as_parameter)
 
+    def _build_ffiargtype(self):
+        return _shape_to_ffi_type(self._ffiargshape_)
+
     def get_ffi_argtype(self):
         if self._ffiargtype:
             return self._ffiargtype
-        self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_)
+        self._ffiargtype = self._build_ffiargtype()
         return self._ffiargtype
 
     def _CData_output(self, resbuffer, base=None, index=-1):
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -70,7 +70,12 @@
         self._ffiarray = ffiarray
         self.__init__ = __init__
         self._type_ = TP
-        self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype())
+
+    def _build_ffiargtype(self):
+        return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+    def _deref_ffiargtype(self):
+        return self._type_.get_ffi_argtype()
 
     from_address = cdata_from_address
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -160,6 +160,10 @@
             raise AttributeError("_fields_ is final")
         if self in [f[1] for f in value]:
             raise AttributeError("Structure or union cannot contain itself")
+        if self._ffiargtype is not None:
+            raise NotImplementedError("Too late to set _fields_: we already "
+                        "said to libffi that the structure type %s is opaque"
+                        % (self,))
         names_and_fields(
             self,
             value, self.__bases__[0],
diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py
--- a/lib_pypy/_ssl/__init__.py
+++ b/lib_pypy/_ssl/__init__.py
@@ -16,12 +16,14 @@
     RAND_egd          = builtinify(RAND_egd)
 
 import sys
-if sys.platform == "win32" and 'enum_certificates' not in globals():
-    def enum_certificates(*args, **kwds):
-        import warnings
-        warnings.warn("ssl.enum_certificates() is not implemented")
-        return []
-    def enum_crls(*args, **kwds):
-        import warnings
-        warnings.warn("ssl.enum_crls() is not implemented")
-        return []
+if sys.platform == "win32":
+    if 'enum_certificates' not in globals():
+        def enum_certificates(*args, **kwds):
+            import warnings
+            warnings.warn("ssl.enum_certificates() is not implemented")
+            return []
+    if 'enum_crls' not in globals():
+        def enum_crls(*args, **kwds):
+            import warnings
+            warnings.warn("ssl.enum_crls() is not implemented")
+            return []
diff --git a/lib_pypy/_ssl_build.py b/lib_pypy/_ssl_build.py
--- a/lib_pypy/_ssl_build.py
+++ b/lib_pypy/_ssl_build.py
@@ -5,6 +5,11 @@
 from _cffi_ssl._cffi_src.build_openssl import (build_ffi_for_binding,
         _get_openssl_libraries, extra_link_args, compiler_type)
 
+if sys.platform == "win32":
+    pypy_win32_extra = ["pypy_win32_extra"]
+else:
+    pypy_win32_extra = []
+
 ffi = build_ffi_for_binding(
     module_name="_pypy_openssl",
     module_prefix="_cffi_src.openssl.",
@@ -44,10 +49,10 @@
         "x509_vfy",
         "pkcs7",
         "callbacks",
-    ],
+    ] + pypy_win32_extra,
     libraries=_get_openssl_libraries(sys.platform),
     extra_link_args=extra_link_args(compiler_type()),
 )
 
 if __name__ == '__main__':
-    ffi.compile()
+    ffi.compile(verbose=True)
diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -1,10 +1,47 @@
 import _imp
+import os
+import sys
+from distutils.spawn import find_executable
 
 so_ext = _imp.extension_suffixes()[0]
 
+
 build_time_vars = {
-    "EXT_SUFFIX": so_ext,
-    "SHLIB_SUFFIX": so_ext,
     "SOABI": '-'.join(so_ext.split('.')[1].split('-')[:2]),
-    "SO": so_ext  # deprecated in Python 3, for backward compatibility
+    "SO": so_ext,  # deprecated in Python 3, for backward compatibility
+    'CC': "cc -pthread",
+    'CXX': "c++ -pthread",
+    'OPT': "-DNDEBUG -O2",
+    'CFLAGS': "-DNDEBUG -O2",
+    'CCSHARED': "-fPIC",
+    'LDSHARED': "cc -pthread -shared",
+    'EXT_SUFFIX': so_ext,
+    'SHLIB_SUFFIX': ".so",
+    'AR': "ar",
+    'ARFLAGS': "rc",
+    'EXE': "",
+    'LIBDIR': os.path.join(sys.prefix, 'lib'),
+    'VERSION': sys.version[:3]
 }
+
+if sys.platform[:6] == "darwin":
+    import platform
+    if platform.machine() == 'i386':
+        if platform.architecture()[0] == '32bit':
+            arch = 'i386'
+        else:
+            arch = 'x86_64'
+    else:
+        # just a guess
+        arch = platform.machine()
+    build_time_vars['LDSHARED'] += ' -undefined dynamic_lookup'
+    build_time_vars['CC'] += ' -arch %s' % (arch,)
+
+if find_executable("gcc"):
+    build_time_vars.update({
+        "CC": "gcc -pthread",
+        "GNULD": "yes",
+        "LDSHARED": "gcc -pthread -shared",
+    })
+    if find_executable("g++"):
+        build_time_vars["CXX"] = "g++ -pthread"
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,7 +39,7 @@
     "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
-    "_csv", "_pypyjson", "_posixsubprocess", # "_cppyy", "micronumpy"
+    "_csv", "_pypyjson", "_posixsubprocess", "_cppyy", # "micronumpy",
     "_jitlog",
 ])
 
diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -73,3 +73,63 @@
 This division between bytecode evaluator and object space gives a lot of
 flexibility.  One can plug in different :doc:`object spaces <objspace>` to get
 different or enriched behaviours of the Python objects.
+
+Layers
+------
+
+RPython
+~~~~~~~
+:ref:`RPython <rpython:language>` is the language in which we write interpreters.
+Not the entire PyPy project is written in RPython, only the parts that are
+compiled in the translation process. The interesting point is that RPython
+has no parser, it's compiled from the live python objects, which makes it
+possible to do all kinds of metaprogramming during import time. In short,
+Python is a meta programming language for RPython.
+
+The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+Consult `Getting Started with RPython`_ for further reading
+
+Translation
+~~~~~~~~~~~
+The translation toolchain - this is the part that takes care of translating
+RPython to flow graphs and then to C. There is more in the
+:doc:`architecture <architecture>` document written about it.
+
+It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+and ``rtyper``.
+
+PyPy Interpreter
+~~~~~~~~~~~~~~~~
+This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
+interpreter for Python written in RPython.  The fact that it is
+RPython is not apparent at first.  Built-in modules are written in
+``pypy/module/*``.  Some modules that CPython implements in C are
+simply written in pure Python; they are in the top-level ``lib_pypy``
+directory.  The standard library of Python (with a few changes to
+accomodate PyPy) is in ``lib-python``.
+
+JIT Compiler
+~~~~~~~~~~~~
+:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
+interpreter written in RPython, rather than the user program that it
+interprets.  As a result it applies to any interpreter, i.e. any
+language.  But getting it to work correctly is not trivial: it
+requires a small number of precise "hints" and possibly some small
+refactorings of the interpreter.  The JIT itself also has several
+almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+that turns it into machine code.  Writing a new backend is a
+traditional way to get into the project.
+
+Garbage Collectors
+~~~~~~~~~~~~~~~~~~
+Garbage Collectors (GC): as you may notice if you are used to CPython's
+C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+:ref:`rpython:garbage-collection` is inserted
+during translation.  Moreover, this is not reference counting; it is a real
+GC written as more RPython code.  The best one we have so far is in
+``rpython/memory/gc/incminimark.py``.
+
+.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -267,14 +267,14 @@
 * PyPy 2.5.1 or earlier: normal users would see permission errors.
   Installers need to run ``pypy -c "import gdbm"`` and other similar
   commands at install time; the exact list is in
-  :source:`pypy/tool/release/package.py <package.py>`.  Users
+  :source:`pypy/tool/release/package.py`.  Users
   seeing a broken installation of PyPy can fix it after-the-fact if they
   have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
 
 * PyPy 2.6 and later: anyone would get ``ImportError: no module named
   _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
   ``lib_pypy`` directory during the installation process (plus others;
-  see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+  see the exact list in :source:`pypy/tool/release/package.py`).
   Users seeing a broken
   installation of PyPy can fix it after-the-fact, by running ``pypy
   /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -539,7 +539,7 @@
 
     hg help branch
 
-.. _official wiki: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: https://www.mercurial-scm.org/wiki/
 
 
 .. _using-development-tracker:
@@ -547,15 +547,7 @@
 Using the development bug/feature tracker
 -----------------------------------------
 
-We have a `development tracker`_, based on Richard Jones'
-`roundup`_ application.  You can file bugs,
-feature requests or see what's going on
-for the next milestone, both from an E-Mail and from a
-web interface.
-
-.. _development tracker: https://bugs.pypy.org/
-.. _roundup: http://roundup.sourceforge.net/
-
+We use bitbucket for :source:`issues` tracking and :source:`pull-requests`.
 
 .. _testing:
 
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -8,3 +8,4 @@
    :maxdepth: 1
 
    man/pypy.1.rst
+   man/pypy3.1.rst
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -66,9 +66,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '5.8'
+version = '6.0'
 # The full version, including alpha/beta/rc tags.
-release = '5.8.0'
+release = '6.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/config/objspace.usemodules._cppyy.txt b/pypy/doc/config/objspace.usemodules._cppyy.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules._cppyy.txt
@@ -0,0 +1,1 @@
+The internal backend for cppyy
diff --git a/pypy/doc/config/objspace.usemodules._rawffi.txt b/pypy/doc/config/objspace.usemodules._rawffi.txt
--- a/pypy/doc/config/objspace.usemodules._rawffi.txt
+++ b/pypy/doc/config/objspace.usemodules._rawffi.txt
@@ -1,3 +1,3 @@
-An experimental module providing very low-level interface to
+A module providing very low-level interface to
 C-level libraries, for use when implementing ctypes, not
-intended for a direct use at all.
\ No newline at end of file
+intended for a direct use at all.
diff --git a/pypy/doc/config/objspace.usemodules.cpyext.txt b/pypy/doc/config/objspace.usemodules.cpyext.txt
--- a/pypy/doc/config/objspace.usemodules.cpyext.txt
+++ b/pypy/doc/config/objspace.usemodules.cpyext.txt
@@ -1,1 +1,1 @@
-Use (experimental) cpyext module, that tries to load and run CPython extension modules
+Use cpyext module to load and run CPython extension modules
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/contributing.rst
@@ -0,0 +1,472 @@
+Contributing Guidelines
+===========================
+
+.. contents::
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy's
+  approach is great if you do TDD, and not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below.
+
+The first thing to remember is that PyPy project is very different than most
+projects out there. It's also different from a classic compiler project,
+so academic courses about compilers often don't apply or lead in the wrong
+direction. However, if you want to understand how designing & building a runtime
+works in the real world then this is a great project!
+
+Getting involved
+^^^^^^^^^^^^^^^^
+
+PyPy employs a relatively standard open-source development process. You are
+encouraged as a first step to join our `pypy-dev mailing list`_ and IRC channel,
+details of which can be found in our :ref:`contact <contact>` section. The folks
+there are very friendly, and can point you in the right direction.
+
+We give out commit rights usually fairly liberally, so if you want to do something
+with PyPy, you can become a committer. We also run frequent coding sprints which
+are separately announced and often happen around Python conferences such as
+EuroPython or PyCon. Upcoming events are usually announced on `the blog`_.
+
+Further Reading: :ref:`Contact <contact>`
+
+.. _the blog: http://morepypy.blogspot.com
+.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
+
+
+Your first contribution
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The first and most important rule how **not** to contribute to PyPy is
+"just hacking a feature". This won't work, and you'll find your PR will typically
+require a lot of re-work. There are a few reasons why not:
+
+* build times are large
+* PyPy has very thick layer separation
+* context of the cPython runtime is often required
+
+Instead, reach out on the dev mailing list or the IRC channel, and we're more
+than happy to help! :)
+
+Some ideas for first contributions are:
+
+* Documentation - this will give you an understanding of the pypy architecture
+* Test failures - find a failing test in the `nightly builds`_, and fix it
+* Missing language features - these are listed in our `issue tracker`_
+
+.. _nightly builds: http://buildbot.pypy.org/nightly/
+.. _issue tracker: https://bitbucket.org/pypy/pypy/issues
+
+Source Control
+--------------
+
+PyPy development is based a typical fork/pull request based workflow, centered
+around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
+before, a good introduction can be found here:
+
+    https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow
+
+The cycle for a new PyPy contributor goes typically like this:
+
+Fork & Clone
+------------
+
+* Make an account on bitbucket_.
+
+* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
+  icons).  You get a fork of the repository, e.g. in
+  `https://bitbucket.org/yourname/pypy/`.
+
+* Clone your new repo (i.e. the fork) to your local machine with the command
+  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
+  operation but only ever needs to be done once.  See also
+  http://pypy.org/download.html#building-from-source .
+  If you already cloned
+  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
+  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
+  your clone to contain the line ``default =
+  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
+  up``.  If you already have such a clone but don't want to change it,
+  you can clone that copy with ``hg clone /path/to/other/copy``, and
+  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
+
+* Now you have a complete copy of the PyPy repo.  Make a branch
+  with a command like ``hg branch name_of_your_branch``.
+
+Edit
+----
+
+* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
+  to make Mercurial aware of new files you added, e.g. new test files.
+  Use ``hg status`` to see if there are such files.  Write and run tests!
+  (See the rest of this page.)
+
+* Commit regularly with ``hg commit``.  A one-line commit message is
+  fine.  We love to have tons of commits; make one as soon as you have
+  some progress, even if it is only some new test that doesn't pass yet,
+  or fixing things even if not all tests pass.  Step by step, you are
+  building the history of your changes, which is the point of a version
+  control system.  (There are commands like ``hg log`` and ``hg up``
+  that you should read about later, to learn how to navigate this
+  history.)
+
+* The commits stay on your machine until you do ``hg push`` to "push"
+  them back to the repo named in the file ``.hg/hgrc``.  Repos are
+  basically just collections of commits (a commit is also called a
+  changeset): there is one repo per url, plus one for each local copy on
+  each local machine.  The commands ``hg push`` and ``hg pull`` copy
+  commits around, with the goal that all repos in question end up with
+  the exact same set of commits.  By opposition, ``hg up`` only updates
+  the "working copy" by reading the local repository, i.e. it makes the
+  files that you see correspond to the latest (or any other) commit
+  locally present.
+
+* You should push often; there is no real reason not to.  Remember that
+  even if they are pushed, with the setup above, the commits are (1)
+  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
+  named.  Yes, they are publicly visible, but don't worry about someone
+  walking around the thousands of repos on bitbucket saying "hah, look
+  at the bad coding style of that guy".  Try to get into the mindset
+  that your work is not secret and it's fine that way.  We might not
+  accept it as is for PyPy, asking you instead to improve some things,
+  but we are not going to judge you.
+
+Pull Request
+------------
+
+* The final step is to open a pull request, so that we know that you'd
+  like to merge that branch back to the original ``pypy/pypy`` repo.
+  This can also be done several times if you have interesting
+  intermediate states, but if you get there, then we're likely to
+  proceed to the next stage, which is...
+
+* Get a regular account for pushing directly to
+  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
+  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
+  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
+  then be pushed directly to the official repo, but (if you follow these
+  rules) they are still on a branch, and we can still review the
+  branches you want to merge.
+
+* If you get closer to the regular day-to-day development, you'll notice
+  that we generally push small changes as one or a few commits directly
+  to the branch ``default``.  Also, we often collaborate even if we are
+  on other branches, which do not really "belong" to anyone.  At this
+  point you'll need ``hg merge`` and learn how to resolve conflicts that
+  sometimes occur when two people try to push different commits in
+  parallel on the same branch.  But it is likely an issue for later ``:-)``
+
+.. _bitbucket: https://bitbucket.org/
+
+
+Architecture
+^^^^^^^^^^^^
+
+PyPy has layers. Just like ogres or onions. Those layers help us keep the
+respective parts separated enough to be worked on independently and make the
+complexity manageable. This is, again, just a sanity requirement for such
+a complex project. For example writing a new optimization for the JIT usually
+does **not** involve touching a Python interpreter at all or the JIT assembler
+backend or the garbage collector. Instead it requires writing small tests in
+``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
+After that, you can just compile PyPy and things should just work.
+
+Further Reading: :doc:`architecture <architecture>`
+
+Where to start?
+---------------
+
+PyPy is made from parts that are relatively independent of each other.
+You should start looking at the part that attracts you most (all paths are
+relative to the PyPy top level directory).  You may look at our
+:doc:`directory reference <dir-reference>` or start off at one of the following
+points:
+
+*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
+   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
+   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
+   function objects and argument passing in :source:`pypy/interpreter/function.py`
+   and :source:`pypy/interpreter/argument.py`, the object space interface
+   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
+   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
+   Core types supporting the bytecode interpreter are defined in
+   :source:`pypy/interpreter/typedef.py`.
+
+*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
+   and grammar files that allow it to parse the syntax of various Python
+   versions. Once the grammar has been processed, the parser can be
+   translated by the above machinery into efficient code.
+
+*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
+   contains a modified version of the compiler package from CPython
+   that fixes some bugs and is translatable.
+
+*  :source:`pypy/objspace/std` contains the
+   :ref:`Standard object space <standard-object-space>`.  The main file
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)
+
+Building
+^^^^^^^^
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Further Reading: :doc:`Build <build>`
+
+Coding Guide
+------------
+
+As well as the usual pep8 and formatting standards, there are a number of
+naming conventions and coding styles that are important to understand before
+browsing the source.
+
+Further Reading: :doc:`Coding Guide <coding-guide>`
+
+Testing
+^^^^^^^
+
+Test driven development
+-----------------------
+
+Instead, we practice a lot of test driven development. This is partly because
+of very high quality requirements for compilers and partly because there is
+simply no other way to get around such complex project, that will keep you sane.
+There are probably people out there who are smart enough not to need it, we're
+not one of those. You may consider familiarizing yourself with `pytest`_,
+since this is a tool we use for tests.
+This leads to the next issue:
+
+.. _pytest: http://pytest.org/
+
+py.test and the py lib
+----------------------
+
+The `py.test testing tool`_ drives all our testing needs.
+
+We use the `py library`_ for filesystem path manipulations, terminal
+writing, logging and some other support  functionality.
+
+You don't necessarily need to install these two libraries because
+we also ship them inlined in the PyPy source tree.
+
+.. _py library: http://pylib.readthedocs.org/
+
+Running PyPy's unit tests
+-------------------------
+
+PyPy development always was and is still thoroughly test-driven.
+We use the flexible `py.test testing tool`_ which you can `install independently
+<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
+
+The PyPy source tree comes with an inlined version of ``py.test``
+which you can invoke by typing::
+
+    python pytest.py -h
+
+This is usually equivalent to using an installed version::
+
+    py.test -h
+
+If you encounter problems with the installed version
+make sure you have the correct version installed which
+you can find out with the ``--version`` switch.
+
+You will need the `build requirements`_ to run tests successfully, since many of
+them compile little pieces of PyPy and then run the tests inside that minimal
+interpreter. The `cpyext` tests also require `pycparser`, and many tests build
+cases with `hypothesis`.
+
+Now on to running some tests.  PyPy has many different test directories
+and you can use shell completion to point at directories or files::
+
+    py.test pypy/interpreter/test/test_pyframe.py
+
+    # or for running tests of a whole subdirectory
+    py.test pypy/interpreter/
+
+See `py.test usage and invocations`_ for some more generic info
+on how you can run tests.
+
+Beware trying to run "all" pypy tests by pointing to the root
+directory or even the top level subdirectory ``pypy``.  It takes
+hours and uses huge amounts of RAM and is not recommended.
+
+To run CPython regression tests you can point to the ``lib-python``
+directory::
+
+    py.test lib-python/2.7/test/test_datetime.py
+
+This will usually take a long time because this will run
+the PyPy Python interpreter on top of CPython.  On the plus
+side, it's usually still faster than doing a full translation
+and running the regression test with the translated PyPy Python
+interpreter.
+
+.. _py.test testing tool: http://pytest.org
+.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
+.. _`build requirements`: build.html#install-build-time-dependencies
+
+Testing After Translation
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While the usual invocation of `pytest` translates a piece of RPython code and
+runs it, we have a test extension to run tests without translation, directly
+on the host python. This is very convenient for modules such as `cpyext`, to
+compare and contrast test results between CPython and PyPy. Untranslated tests
+are invoked by using the `-A` or `--runappdirect` option to `pytest`::
+
+    python2 pytest.py -A pypy/module/cpyext/test
+
+where `python2` can be either `python2` or `pypy2`. On the `py3` branch, the
+collection phase must be run with `python2` so untranslated tests are run
+with::
+
+    cpython2 pytest.py -A pypy/module/cpyext/test --python=path/to/pypy3
+
+
+Tooling & Utilities
+^^^^^^^^^^^^^^^^^^^
+
+If you are interested in the inner workings of the PyPy Python interpreter,
+there are some features of the untranslated Python interpreter that allow you
+to introspect its internals.
+
+
+Interpreter-level console
+-------------------------
+
+To start interpreting Python with PyPy, install a C compiler that is
+supported by distutils and use Python 2.7 or greater to run PyPy::
+
+    cd pypy
+    python bin/pyinteractive.py
+
+After a few seconds (remember: this is running on top of CPython), you should
+be at the PyPy prompt, which is the same as the Python prompt, but with an
+extra ">".
+
+If you press
+<Ctrl-C> on the console you enter the interpreter-level console, a
+usual CPython console.  You can then access internal objects of PyPy
+(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
+prompt with the prefix ``w_``::
+
+    >>>> a = 123
+    >>>> <Ctrl-C>
+    *** Entering interpreter-level console ***
+    >>> w_a
+    W_IntObject(123)
+
+The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
+
+    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
+    >>> <Ctrl-D>
+    *** Leaving interpreter-level console ***
+
+    KeyboardInterrupt
+    >>>> l
+    [1, 'abc']
+
+Note that the prompt of the interpreter-level console is only '>>>' since
+it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
+Linux) or <Ctrl-Z>, <Enter> (under Windows).
+
+Also note that not all modules are available by default in this mode (for
+example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
+``--withmod-...`` command line options.
+
+You may be interested in reading more about the distinction between
+:ref:`interpreter-level and app-level <interpreter-level>`.
+
+pyinteractive.py options
+------------------------
+
+To list the PyPy interpreter command line options, type::
+
+    cd pypy
+    python bin/pyinteractive.py --help
+
+pyinteractive.py supports most of the options that CPython supports too (in addition to a
+large amount of options that can be used to customize pyinteractive.py).
+As an example of using PyPy from the command line, you could type::
+
+    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
+
+Alternatively, as with regular Python, you can simply give a
+script name on the command line::
+
+    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
+
+See our :doc:`configuration sections <config/index>` for details about what all the commandline
+options do.
+
+
+.. _trace example:
+
+Tracing bytecode and operations on objects
+------------------------------------------
+
+You can use a simple tracing mode to monitor the interpretation of
+bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
+PyPy console::
+
+    >>>> __pytrace__ = 1
+    Tracing enabled
+    >>>> x = 5
+            <module>:           LOAD_CONST    0 (5)
+            <module>:           STORE_NAME    0 (x)
+            <module>:           LOAD_CONST    1 (None)
+            <module>:           RETURN_VALUE    0
+    >>>> x
+            <module>:           LOAD_NAME    0 (x)
+            <module>:           PRINT_EXPR    0
+    5
+            <module>:           LOAD_CONST    0 (None)
+            <module>:           RETURN_VALUE    0
+    >>>>
+
+
+Demos
+^^^^^
+
+The `example-interpreter`_ repository contains an example interpreter
+written using the RPython translation toolchain.
+
+.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
+
+
+graphviz & pygame for flow graph viewing (highly recommended)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+graphviz and pygame are both necessary if you want to look at generated flow
+graphs:
+
+    graphviz: http://www.graphviz.org/Download.php
+
+    pygame: http://www.pygame.org/download.shtml
+
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -214,6 +214,7 @@
   Lukas Vacek
   Omer Katz
   Jacek Generowicz
+  Tomasz Dziopa
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -274,6 +275,7 @@
   Yury V. Zaytsev
   florinpapa
   Anders Sigfridsson
+  Matt Jackson
   Nikolay Zinov
   rafalgalczynski at gmail.com
   Joshua Gilbert
diff --git a/pypy/doc/discussion/ctypes-implementation.rst b/pypy/doc/discussion/ctypes-implementation.rst
--- a/pypy/doc/discussion/ctypes-implementation.rst
+++ b/pypy/doc/discussion/ctypes-implementation.rst
@@ -141,28 +141,3 @@
 
 .. _pyglet: http://pyglet.org/
 
-
-ctypes configure
------------------
-
-We also released ``ctypes-configure``, which is an experimental package
-trying to approach the portability issues of ctypes-based code.
-
-idea
-~~~~
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes c
-compiler (via distutils) for various platform-dependent details like
-exact sizes of types (for example size_t), ``#defines``, exact outline of
-structures etc. It replaces in this regard code generator (h2py).
-
-installation
-~~~~~~~~~~~~
-
-``easy_install ctypes_configure``
-
-usage
-~~~~~
-
-:source:`ctypes_configure/doc/sample.py` explains in details how to use it.
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -1,5 +1,5 @@
-Embedding PyPy
-==============
+Embedding PyPy (DEPRECATED)
+===========================
 
 PyPy has a very minimal and a very strange embedding interface, based on
 the usage of `cffi`_ and the philosophy that Python is a better language than
diff --git a/pypy/doc/eventhistory.rst b/pypy/doc/eventhistory.rst
--- a/pypy/doc/eventhistory.rst
+++ b/pypy/doc/eventhistory.rst
@@ -40,11 +40,9 @@
 Main focus of the sprint will be on the goals of the upcoming June 0.9
 release.
 
-Read more in `the sprint announcement`__, see who is  planning to attend
-on the `people page`_.
+Read more about `the sprint`__
 
-__ https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/announce.html
-.. _people page: https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/people.txt
+__ https://bitbucket.org/pypy/extradoc/src/extradoc/sprintinfo/ddorf2006/
 
 
 PyPy sprint at Akihabara (Tokyo, Japan)
diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst
--- a/pypy/doc/extradoc.rst
+++ b/pypy/doc/extradoc.rst
@@ -75,12 +75,12 @@
 .. _A Way Forward in Parallelising Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2014/position-paper.pdf
 .. _Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf
 .. _Allocation Removal by Partial Evaluation in a Tracing JIT: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/pepm2011/bolz-allocation-removal.pdf
-.. _Towards a Jitting VM for Prolog Execution: http://www.stups.uni-duesseldorf.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
+.. _Towards a Jitting VM for Prolog Execution: http://stups.hhu.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
 .. _High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages: http://buildbot.pypy.org/misc/antocuni-thesis.pdf
 .. _How to *not* write Virtual Machines for Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dyla2007/dyla.pdf
 .. _`Tracing the Meta-Level: PyPy's Tracing JIT Compiler`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009/bolz-tracing-jit.pdf
 .. _`Faster than C#: Efficient Implementation of Dynamic Languages on .NET`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009-dotnet/cli-jit.pdf
-.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: http://stups.hhu.de/mediawiki/images/b/b9/Master_bolz.pdf
+.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: https://www.researchgate.net/profile/Davide_Ancona/publication/252023163_Automatic_generation_of_JIT_compilers_for_dynamic_languages_in_NET/links/53f2098e0cf2bc0c40e70023/Automatic-generation-of-JIT-compilers-for-dynamic-languages-in-NET.pdf
 .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/DynamicLanguages_abstracts.html#AACM-DLS07
 .. _EU Reports: index-report.html
 .. _Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution: http://sabi.net/nriley/pubs/dls6-riley.pdf
@@ -368,6 +368,6 @@
 .. _LLVM: http://llvm.org/
 .. _IronPython: http://ironpython.codeplex.com/
 .. _Dynamic Native Optimization of Native Interpreters: http://people.csail.mit.edu/gregs/dynamorio.html
-.. _JikesRVM: http://jikesrvm.org/
+.. _JikesRVM: http://www.jikesrvm.org/
 .. _Tunes: http://tunes.org
 .. _old Tunes Wiki: http://buildbot.pypy.org/misc/cliki.tunes.org/
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -67,7 +67,7 @@
 you may need to run the command with `sudo` for a global installation.
 The other commands of ``setup.py`` are available too, like ``build``.
 
-.. _PyPI: https://pypi.python.org/pypi
+.. _PyPI: https://pypi.org
 .. _`use virtualenv (as documented here)`: install.html#installing-using-virtualenv
 
 
@@ -360,7 +360,7 @@
 (produced during a sprint).  On the `PyPy bitbucket page`_ there is also a
 Scheme and an Io implementation; both of these are unfinished at the moment.
 
-.. _Topaz: http://topazruby.com/
+.. _Topaz: http://docs.topazruby.com/en/latest/
 .. _Hippy: http://morepypy.blogspot.ch/2012/07/hello-everyone.html
 .. _JavaScript interpreter: https://bitbucket.org/pypy/lang-js/
 .. _Prolog interpreter: https://bitbucket.org/cfbolz/pyrolog/
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -152,7 +152,7 @@
 to wait until it reaches a point in which the interpreter is in a known state
 and calling user-defined code is harmless.  It might happen that multiple
 events occur before the hook is invoked: in this case, you can inspect the
-value ``stats.count`` to know how many times the event occured since the last
+value ``stats.count`` to know how many times the event occurred since the last
 time the hook was called.  Similarly, ``stats.duration`` contains the
 **total** time spent by the GC for this specific event since the last time the
 hook was called.
@@ -163,7 +163,7 @@
 The attributes for ``GcMinorStats`` are:
 
 ``count``
-    The number of minor collections occured since the last hook call.
+    The number of minor collections occurred since the last hook call.
 
 ``duration``
     The total time spent inside minor collections since the last hook
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
deleted file mode 100644
--- a/pypy/doc/getting-started-dev.rst
+++ /dev/null
@@ -1,345 +0,0 @@
-Getting Started Developing With PyPy
-====================================
-
-.. contents::
-
-
-Using Mercurial
----------------
-
-PyPy development is based on Mercurial (hg).  If you are not used to
-version control, the cycle for a new PyPy contributor goes typically
-like this:
-
-* Make an account on bitbucket_.
-
-* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
-  icons).  You get a fork of the repository, e.g. in
-  https://bitbucket.org/yourname/pypy/.
-
-* Clone this new repo (i.e. the fork) to your local machine with the command 
-  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
-  operation but only ever needs to be done once.  See also 
-  http://pypy.org/download.html#building-from-source .
-  If you already cloned
-  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
-  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
-  your clone to contain the line ``default =
-  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
-  up``.  If you already have such a clone but don't want to change it,
-  you can clone that copy with ``hg clone /path/to/other/copy``, and
-  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
-
-* Now you have a complete copy of the PyPy repo.  Make a branch
-  with a command like ``hg branch name_of_your_branch``.
-
-* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
-  to make Mercurial aware of new files you added, e.g. new test files.
-  Use ``hg status`` to see if there are such files.  Write and run tests!
-  (See the rest of this page.)
-
-* Commit regularly with ``hg commit``.  A one-line commit message is
-  fine.  We love to have tons of commits; make one as soon as you have
-  some progress, even if it is only some new test that doesn't pass yet,
-  or fixing things even if not all tests pass.  Step by step, you are
-  building the history of your changes, which is the point of a version
-  control system.  (There are commands like ``hg log`` and ``hg up``
-  that you should read about later, to learn how to navigate this
-  history.)
-
-* The commits stay on your machine until you do ``hg push`` to "push"
-  them back to the repo named in the file ``.hg/hgrc``.  Repos are
-  basically just collections of commits (a commit is also called a
-  changeset): there is one repo per url, plus one for each local copy on
-  each local machine.  The commands ``hg push`` and ``hg pull`` copy
-  commits around, with the goal that all repos in question end up with
-  the exact same set of commits.  By opposition, ``hg up`` only updates
-  the "working copy" by reading the local repository, i.e. it makes the
-  files that you see correspond to the latest (or any other) commit
-  locally present.
-
-* You should push often; there is no real reason not to.  Remember that
-  even if they are pushed, with the setup above, the commits are (1)
-  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
-  named.  Yes, they are publicly visible, but don't worry about someone
-  walking around the thousands of repos on bitbucket saying "hah, look
-  at the bad coding style of that guy".  Try to get into the mindset
-  that your work is not secret and it's fine that way.  We might not
-  accept it as is for PyPy, asking you instead to improve some things,
-  but we are not going to judge you.
-
-* The final step is to open a pull request, so that we know that you'd
-  like to merge that branch back to the original ``pypy/pypy`` repo.
-  This can also be done several times if you have interesting
-  intermediate states, but if you get there, then we're likely to
-  proceed to the next stage, which is...
-
-* Get a regular account for pushing directly to
-  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
-  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
-  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
-  then be pushed directly to the official repo, but (if you follow these
-  rules) they are still on a branch, and we can still review the
-  branches you want to merge.
-
-* If you get closer to the regular day-to-day development, you'll notice
-  that we generally push small changes as one or a few commits directly
-  to the branch ``default``.  Also, we often collaborate even if we are
-  on other branches, which do not really "belong" to anyone.  At this
-  point you'll need ``hg merge`` and learn how to resolve conflicts that
-  sometimes occur when two people try to push different commits in
-  parallel on the same branch.  But it is likely an issue for later ``:-)``
-
-.. _bitbucket: https://bitbucket.org/
-
-
-Running PyPy's unit tests
--------------------------
-
-PyPy development always was and is still thoroughly test-driven.
-We use the flexible `py.test testing tool`_ which you can `install independently
-<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
-
-The PyPy source tree comes with an inlined version of ``py.test``
-which you can invoke by typing::
-
-    python pytest.py -h
-
-This is usually equivalent to using an installed version::
-
-    py.test -h
-
-If you encounter problems with the installed version
-make sure you have the correct version installed which
-you can find out with the ``--version`` switch.
-
-You will need the `build requirements`_ to run tests successfully, since many of
-them compile little pieces of PyPy and then run the tests inside that minimal
-interpreter
-
-Now on to running some tests.  PyPy has many different test directories
-and you can use shell completion to point at directories or files::
-
-    py.test pypy/interpreter/test/test_pyframe.py
-
-    # or for running tests of a whole subdirectory
-    py.test pypy/interpreter/
-
-See `py.test usage and invocations`_ for some more generic info
-on how you can run tests.
-
-Beware trying to run "all" pypy tests by pointing to the root
-directory or even the top level subdirectory ``pypy``.  It takes
-hours and uses huge amounts of RAM and is not recommended.
-
-To run CPython regression tests you can point to the ``lib-python``
-directory::
-
-    py.test lib-python/2.7/test/test_datetime.py
-
-This will usually take a long time because this will run
-the PyPy Python interpreter on top of CPython.  On the plus
-side, it's usually still faster than doing a full translation
-and running the regression test with the translated PyPy Python
-interpreter.
-
-.. _py.test testing tool: http://pytest.org
-.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
-.. _`build requirements`: build.html#install-build-time-dependencies
-
-Special Introspection Features of the Untranslated Python Interpreter
----------------------------------------------------------------------
-
-If you are interested in the inner workings of the PyPy Python interpreter,
-there are some features of the untranslated Python interpreter that allow you
-to introspect its internals.
-
-
-Interpreter-level console
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To start interpreting Python with PyPy, install a C compiler that is
-supported by distutils and use Python 2.7 or greater to run PyPy::
-
-    cd pypy
-    python bin/pyinteractive.py
-
-After a few seconds (remember: this is running on top of CPython), you should
-be at the PyPy prompt, which is the same as the Python prompt, but with an
-extra ">".
-
-If you press
-<Ctrl-C> on the console you enter the interpreter-level console, a
-usual CPython console.  You can then access internal objects of PyPy
-(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
-prompt with the prefix ``w_``::
-
-    >>>> a = 123
-    >>>> <Ctrl-C>
-    *** Entering interpreter-level console ***
-    >>> w_a
-    W_IntObject(123)
-
-The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
-
-    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
-    >>> <Ctrl-D>
-    *** Leaving interpreter-level console ***
-
-    KeyboardInterrupt
-    >>>> l
-    [1, 'abc']
-
-Note that the prompt of the interpreter-level console is only '>>>' since
-it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
-Linux) or <Ctrl-Z>, <Enter> (under Windows).
-
-Also note that not all modules are available by default in this mode (for
-example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
-``--withmod-...`` command line options.
-
-You may be interested in reading more about the distinction between
-:ref:`interpreter-level and app-level <interpreter-level>`.
-
-pyinteractive.py options
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-To list the PyPy interpreter command line options, type::
-
-    cd pypy
-    python bin/pyinteractive.py --help
-
-pyinteractive.py supports most of the options that CPython supports too (in addition to a
-large amount of options that can be used to customize pyinteractive.py).
-As an example of using PyPy from the command line, you could type::
-
-    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
-
-Alternatively, as with regular Python, you can simply give a
-script name on the command line::
-
-    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
-
-The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
-default almost none of them are, because initializing them takes time.
-If you want anyway to enable all built-in modules, you can use
-``--allworkingmodules``.
-
-See our :doc:`configuration sections <config/index>` for details about what all the commandline
-options do.
-
-
-.. _trace example:
-
-Tracing bytecode and operations on objects
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can use a simple tracing mode to monitor the interpretation of
-bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
-PyPy console::
-
-    >>>> __pytrace__ = 1
-    Tracing enabled
-    >>>> x = 5
-            <module>:           LOAD_CONST    0 (5)
-            <module>:           STORE_NAME    0 (x)
-            <module>:           LOAD_CONST    1 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>> x
-            <module>:           LOAD_NAME    0 (x)
-            <module>:           PRINT_EXPR    0 
-    5
-            <module>:           LOAD_CONST    0 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>>
-
-
-Demos
------
-
-The `example-interpreter`_ repository contains an example interpreter
-written using the RPython translation toolchain.
-
-.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
-
-
-Additional Tools for running (and hacking) PyPy
------------------------------------------------
-
-We use some optional tools for developing PyPy. They are not required to run
-the basic tests or to get an interactive PyPy prompt but they help to
-understand  and debug PyPy especially for the translation process.
-
-
-graphviz & pygame for flow graph viewing (highly recommended)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-graphviz and pygame are both necessary if you
-want to look at generated flow graphs:
-
-	graphviz: http://www.graphviz.org/Download.php
-
-	pygame: http://www.pygame.org/download.shtml
-
-
-py.test and the py lib
-~~~~~~~~~~~~~~~~~~~~~~
-
-The `py.test testing tool`_ drives all our testing needs.
-
-We use the `py library`_ for filesystem path manipulations, terminal
-writing, logging and some other support  functionality.
-
-You don't necessarily need to install these two libraries because
-we also ship them inlined in the PyPy source tree.
-
-.. _py library: http://pylib.readthedocs.org/
-
-
-Getting involved
-----------------
-
-PyPy employs an open development process.  You are invited to join our
-`pypy-dev mailing list`_ or look at the other :ref:`contact
-possibilities <contact>`.  Usually we give out commit rights fairly liberally, so if you
-want to do something with PyPy, you can become a committer. We also run frequent
-coding sprints which are separately announced and often happen around Python
-conferences such as EuroPython or PyCon. Upcoming events are usually announced
-on `the blog`_.
-
-.. _the blog: http://morepypy.blogspot.com
-.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
-
-
-.. _start-reading-sources:
-
-Where to start reading the sources
-----------------------------------
-
-PyPy is made from parts that are relatively independent of each other.
-You should start looking at the part that attracts you most (all paths are
-relative to the PyPy top level directory).  You may look at our :doc:`directory reference <dir-reference>`
-or start off at one of the following points:
-
-*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
-   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
-   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
-   function objects and argument passing in :source:`pypy/interpreter/function.py`
-   and :source:`pypy/interpreter/argument.py`, the object space interface
-   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
-   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
-   Core types supporting the bytecode interpreter are defined in :source:`pypy/interpreter/typedef.py`.
-
-*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
-   and grammar files that allow it to parse the syntax of various Python
-   versions. Once the grammar has been processed, the parser can be
-   translated by the above machinery into efficient code.
-
-*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
-   contains a modified version of the compiler package from CPython
-   that fixes some bugs and is translatable.
-
-*  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
-   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
-   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
-   as a first approximation.  (Some types have multiple implementations.)
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
deleted file mode 100644
--- a/pypy/doc/how-to-contribute.rst
+++ /dev/null
@@ -1,93 +0,0 @@
-How to contribute to PyPy
-=========================
-
-This page describes how to contribute to the PyPy project. The first thing
-to remember is that PyPy project is very different than most projects out there.
-It's also different from a classic compiler project, so academic courses
-about compilers often don't apply or lead in the wrong direction.
-
-
-Don't just hack
----------------
-
-The first and most important rule how not to contribute to PyPy is
-"just hacking". This won't work. There are two major reasons why not
--- build times are large and PyPy has very thick layer separation which
-make it harder to "just hack a feature".
-
-
-Test driven development
------------------------
-
-Instead, we practice a lot of test driven development. This is partly because
-of very high quality requirements for compilers and partly because there is
-simply no other way to get around such complex project, that will keep you sane.
-There are probably people out there who are smart enough not to need it, we're
-not one of those. You may consider familiarizing yourself with `pytest`_,
-since this is a tool we use for tests.
-This leads to the next issue:
-
-.. _pytest: http://pytest.org/
-
-
-Layers
-------
-
-PyPy has layers. Just like Ogres or onions.
-Those layers help us keep the respective parts separated enough
-to be worked on independently and make the complexity manageable. This is,
-again, just a sanity requirement for such a complex project. For example writing
-a new optimization for the JIT usually does **not** involve touching a Python
-interpreter at all or the JIT assembler backend or the garbage collector.
-Instead it requires writing small tests in
-``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
-After that, you can just compile PyPy and things should just work.
-
-The short list of layers for further reading. For each of those layers, a good
-entry point is a test subdirectory in respective directories. It usually
-describes (better or worse) the interfaces between the submodules. For the
-``pypy`` subdirectory, most tests are small snippets of python programs that
-check for correctness (calls ``AppTestXxx``) that will call the appropriate
-part of the interpreter. For the ``rpython`` directory, most tests are small
-RPython interpreters that perform certain tasks. To see how they translate
-to low-level graphs, run them with ``--view``. To see small interpreters
-with a JIT compiler, use ``--viewloops`` option.
-
-* **python interpreter** - it's the part implemented in the ``pypy/`` directory.
-  It's implemented in RPython, which is a high level static language with
-  classes, garbage collection, just-in-time compiler generation and the ability
-  to call C. A cool part about it is that it can be run untranslated, so all
-  the tests are runnable without translating PyPy.
-
-  **interpreter** contains the interpreter core
-
-  **objspace** contains implementations of various objects exported to
-  the Python layer
-
-  **module** directory contains extension modules written in RPython
-

From pypy.commits at gmail.com  Mon May 14 16:50:43 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 13:50:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge
Message-ID: <5af9f6a3.1c69fb81.198f9.1bbf@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94585:ff5d3e68d832
Date: 2018-05-14 22:49 +0200
http://bitbucket.org/pypy/pypy/changeset/ff5d3e68d832/

Log:	merge

diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py
--- a/pypy/interpreter/astcompiler/asthelpers.py
+++ b/pypy/interpreter/astcompiler/asthelpers.py
@@ -83,10 +83,7 @@
         if self.elts:
             for elt in self.elts:
                 elt.set_context(space, ctx)
-            self.ctx = ctx
-        else:
-            # Assignment to () raises an error.
-            ast.expr.set_context(self, space, ctx)
+        self.ctx = ctx
 
 class __extend__(ast.Lambda):
 
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -287,6 +287,9 @@
 
     def visit_Tuple(self, tup):
         """Try to turn tuple building into a constant."""
+        if tup.ctx != ast.Load:
+            return tup   # Don't do the rest for assignment or delete targets.
+                         # It would replace Tuple([]) with Constant('()')!
         if tup.elts:
             consts_w = [None]*len(tup.elts)
             for i in range(len(tup.elts)):
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -810,7 +810,6 @@
             ("{x : x for x in z}", "dict comprehension"),
             ("'str'", "literal"),
             ("b'bytes'", "literal"),
-            ("()", "()"),
             ("23", "literal"),
             ("{}", "literal"),
             ("{1, 2, 3}", "literal"),
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -129,7 +129,7 @@
         self.simple_test(stmt, "type(x)", int)
 
     def test_tuple_assign(self):
-        yield self.error_test, "() = 1", SyntaxError
+        yield self.simple_test, "() = []", "1", 1
         yield self.simple_test, "x,= 1,", "x", 1
         yield self.simple_test, "x,y = 1,2", "x,y", (1, 2)
         yield self.simple_test, "x,y,z = 1,2,3", "x,y,z", (1, 2, 3)
@@ -1482,3 +1482,12 @@
             return f'ab{x}cd'
         """
         code, blocks = generate_function_code(source, self.space)
+
+    def test_empty_tuple_target(self):
+        source = """def f():
+            () = ()
+            del ()
+            [] = []
+            del []
+        """
+        generate_function_code(source, self.space)
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -783,21 +783,6 @@
         assert isinstance(code, PyCode)
         assert code.co_filename == '<filename2>'
 
-    def test_with_empty_tuple(self):
-        source = py.code.Source("""
-        from __future__ import with_statement
-
-        with x as ():
-            pass
-        """)
-        try:
-            self.compiler.compile(str(source), '<filename>', 'exec', 0)
-        except OperationError as e:
-            if not e.match(self.space, self.space.w_SyntaxError):
-                raise
-        else:
-            py.test.fail("Did not raise")
-
     def test_assign_to_yield(self):
         code = 'def f(): (yield bar) += y'
         try:
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -752,6 +752,33 @@
         exc = raises(SyntaxError, compile, code, 'foo', 'exec')
         assert exc.value.offset in (19, 20) # pypy, cpython
 
+    def test_empty_tuple_target(self): """
+        def f(n):
+            () = ()
+            ((), ()) = [[], []]
+            del ()
+            del ((), ())
+            [] = {}
+            ([], ()) = [[], ()]
+            [[], ()] = ((), [])
+            del []
+            del [[], ()]
+            for () in [(), []]: pass
+            for [] in ([],): pass
+            class Zen:
+                def __enter__(self): return ()
+                def __exit__(self, *args): pass
+            with Zen() as (): pass
+            () = [2, 3] * n
+        f(0)
+        try:
+            f(5)
+        except ValueError:
+            pass
+        else:
+            raise AssertionError("should have raised")
+        """
+
 
 if __name__ == '__main__':
     # only to check on top of CPython (you need 2.4)

From pypy.commits at gmail.com  Mon May 14 17:25:51 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 14:25:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix skips
Message-ID: <5af9fedf.1c69fb81.a3095.456e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94586:e5f1a7f119f2
Date: 2018-05-14 23:25 +0200
http://bitbucket.org/pypy/pypy/changeset/e5f1a7f119f2/

Log:	fix skips

diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -375,7 +375,7 @@
         child.expect('>>>')
 
     def test_atexit(self):
-        skip("Python3 atexit is a builtin module")
+        py.test.skip("Python3 atexit is a builtin module")
         child = self.spawn([])
         child.expect('>>> ')
         child.sendline('def f(): print("foobye")')
@@ -525,9 +525,9 @@
 
     def test_options_i_m(self, monkeypatch):
         if sys.platform == "win32":
-            skip("close_fds is not supported on Windows platforms")
+            py.test.skip("close_fds is not supported on Windows platforms")
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'mymodule.py')
         p = os.path.abspath(p)
         monkeypatch.chdir(os.path.dirname(app_main))
@@ -557,7 +557,7 @@
 
     def test_options_u_i(self):
         if sys.platform == "win32":
-            skip("close_fds is not supported on Windows platforms")
+            py.test.skip("close_fds is not supported on Windows platforms")
         import subprocess, select, os
         pipe = subprocess.Popen([get_python3(), app_main, "-u", "-i"],
                                 stdout=subprocess.PIPE,
@@ -614,7 +614,7 @@
             del os.environ['PYTHONINSPECT_']
 
     def test_stdout_flushes_before_stdin_blocks(self):
-        skip("Python3 does not implement this behavior")
+        py.test.skip("Python3 does not implement this behavior")
         # This doesn't really test app_main.py, but a behavior that
         # can only be checked on top of py.py with pexpect.
         path = getscript("""
@@ -632,7 +632,7 @@
 
     def test_no_space_before_argument(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         child = self.spawn(['-cprint("hel" + "lo")'])
         child.expect('hello')
 
@@ -753,7 +753,7 @@
 
     def test_option_m(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'mymodule.py')
         p = os.path.abspath(p)
         monkeypatch.chdir(os.path.dirname(app_main))
@@ -767,7 +767,7 @@
 
     def test_option_m_package(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)),
                          'mypackage', '__main__.py')
         p = os.path.abspath(p)

From pypy.commits at gmail.com  Mon May 14 17:26:12 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 14:26:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: fix code.dump()
Message-ID: <5af9fef4.1c69fb81.f65a9.f748@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94588:f0d5731c5251
Date: 2018-05-14 23:25 +0200
http://bitbucket.org/pypy/pypy/changeset/f0d5731c5251/

Log:	fix code.dump()

diff --git a/pypy/interpreter/test/test_pycode.py b/pypy/interpreter/test/test_pycode.py
--- a/pypy/interpreter/test/test_pycode.py
+++ b/pypy/interpreter/test/test_pycode.py
@@ -16,4 +16,4 @@
     print '>>>\n' + output + '\n<<<'
     assert ' 1 (7)' in output
     assert ' 4 (None)' in output
-    assert ' 19 RETURN_VALUE ' in output
+    assert ' 16 RETURN_VALUE' in output
diff --git a/pypy/tool/dis3.py b/pypy/tool/dis3.py
--- a/pypy/tool/dis3.py
+++ b/pypy/tool/dis3.py
@@ -1,20 +1,40 @@
-"""Disassembler of Python byte code into mnemonics.
-Python 3 dis.py partly backported to Python 2"""
+"""Disassembler of Python byte code into mnemonics."""
+
+from __future__ import print_function
 
 import sys
 import types
+import collections
+import io
 
 from opcode3 import *
 from opcode3 import __all__ as _opcodes_all
 
-__all__ = ["dis", "disassemble", "distb", "disco",
-           "findlinestarts", "findlabels"] + _opcodes_all
+__all__ = ["code_info", "dis", "disassemble", "distb", "disco",
+           "findlinestarts", "findlabels", "show_code",
+           "get_instructions", "Instruction", "Bytecode"] + _opcodes_all
 del _opcodes_all
 
-_have_code = (types.MethodType, types.FunctionType, types.CodeType, type)
+_have_code = (types.MethodType, types.FunctionType, types.CodeType,
+              classmethod, staticmethod, type)
+
+FORMAT_VALUE = opmap['FORMAT_VALUE']
+
+def _try_compile(source, name):
+    """Attempts to compile the given source, first as an expression and
+       then as a statement if the first approach fails.
+
+       Utility function to accept strings in functions that otherwise
+       expect code objects
+    """
+    try:
+        c = compile(source, name, 'eval')
+    except SyntaxError:
+        c = compile(source, name, 'exec')
+    return c
 
 def dis(x=None):
-    """Disassemble classes, methods, functions, or code.
+    """Disassemble classes, methods, functions, generators, or code.
 
     With no argument, disassemble the last traceback.
 
@@ -22,30 +42,31 @@
     if x is None:
         distb()
         return
-    if isinstance(x, types.InstanceType):
-        x = x.__class__
-    if hasattr(x, 'im_func'):
-        x = x.im_func
-    if hasattr(x, 'func_code'):
-        x = x.func_code
-    if hasattr(x, 'co_code'): # PyCode needs co_code before __dict__
+    if hasattr(x, '__func__'):  # Method
+        x = x.__func__
+    if hasattr(x, '__code__'):  # Function
+        x = x.__code__
+    if hasattr(x, 'gi_code'):  # Generator
+        x = x.gi_code
+    if hasattr(x, 'co_code'): # Code object
         disassemble(x)
-    elif hasattr(x, '__dict__'):
-        items = x.__dict__.items()
-        items.sort()
+    elif hasattr(x, '__dict__'):  # Class or module
+        items = sorted(x.__dict__.items())
         for name, x1 in items:
             if isinstance(x1, _have_code):
-                print "Disassembly of %s:" % name
+                print("Disassembly of %s:" % name)
                 try:
                     dis(x1)
                 except TypeError as msg:
-                    print "Sorry:", msg
-                print
-    elif isinstance(x, str):
-        disassemble_string(x)
+                    print("Sorry:", msg)
+                print()
+    elif isinstance(x, (bytes, bytearray)): # Raw bytecode
+        _disassemble_bytes(x)
+    elif isinstance(x, str):    # Source code
+        _disassemble_str(x)
     else:
-        raise TypeError("don't know how to disassemble %s objects" % \
-              type(x).__name__)
+        raise TypeError("don't know how to disassemble %s objects" %
+                        type(x).__name__)
 
 def distb(tb=None):
     """Disassemble a traceback (default: last traceback)."""
@@ -57,104 +78,290 @@
         while tb.tb_next: tb = tb.tb_next
     disassemble(tb.tb_frame.f_code, tb.tb_lasti)
 
+# The inspect module interrogates this dictionary to build its
+# list of CO_* constants. It is also used by pretty_flags to
+# turn the co_flags field into a human readable list.
+COMPILER_FLAG_NAMES = {
+     1: "OPTIMIZED",
+     2: "NEWLOCALS",
+     4: "VARARGS",
+     8: "VARKEYWORDS",
+    16: "NESTED",
+    32: "GENERATOR",
+    64: "NOFREE",
+   128: "COROUTINE",
+   256: "ITERABLE_COROUTINE",
+   512: "ASYNC_GENERATOR",
+}
+
+def pretty_flags(flags):
+    """Return pretty representation of code flags."""
+    names = []
+    for i in range(32):
+        flag = 1<<i
+        if flags & flag:
+            names.append(COMPILER_FLAG_NAMES.get(flag, hex(flag)))
+            flags ^= flag
+            if not flags:
+                break
+    else:
+        names.append(hex(flags))
+    return ", ".join(names)
+
+def _get_code_object(x):
+    """Helper to handle methods, functions, generators, strings and raw code objects"""
+    if hasattr(x, '__func__'): # Method
+        x = x.__func__
+    if hasattr(x, '__code__'): # Function
+        x = x.__code__
+    if hasattr(x, 'gi_code'):  # Generator
+        x = x.gi_code
+    if isinstance(x, str):     # Source code
+        x = _try_compile(x, "<disassembly>")
+    if hasattr(x, 'co_code'):  # Code object
+        return x
+    raise TypeError("don't know how to disassemble %s objects" %
+                    type(x).__name__)
+
+def code_info(x):
+    """Formatted details of methods, functions, or code."""
+    return _format_code_info(_get_code_object(x))
+
+def _format_code_info(co):
+    lines = []
+    lines.append("Name:              %s" % co.co_name)
+    lines.append("Filename:          %s" % co.co_filename)
+    lines.append("Argument count:    %s" % co.co_argcount)
+    lines.append("Kw-only arguments: %s" % co.co_kwonlyargcount)
+    lines.append("Number of locals:  %s" % co.co_nlocals)
+    lines.append("Stack size:        %s" % co.co_stacksize)
+    lines.append("Flags:             %s" % pretty_flags(co.co_flags))
+    if co.co_consts:
+        lines.append("Constants:")
+        for i_c in enumerate(co.co_consts):
+            lines.append("%4d: %r" % i_c)
+    if co.co_names:
+        lines.append("Names:")
+        for i_n in enumerate(co.co_names):
+            lines.append("%4d: %s" % i_n)
+    if co.co_varnames:
+        lines.append("Variable names:")
+        for i_n in enumerate(co.co_varnames):
+            lines.append("%4d: %s" % i_n)
+    if co.co_freevars:
+        lines.append("Free variables:")
+        for i_n in enumerate(co.co_freevars):
+            lines.append("%4d: %s" % i_n)
+    if co.co_cellvars:
+        lines.append("Cell variables:")
+        for i_n in enumerate(co.co_cellvars):
+            lines.append("%4d: %s" % i_n)
+    return "\n".join(lines)
+
+def show_code(co, file=None):
+    """Print details of methods, functions, or code to *file*.
+
+    If *file* is not provided, the output is printed on stdout.
+    """
+    print(code_info(co))
+
+_Instruction = collections.namedtuple("_Instruction",
+     "opname opcode arg argval argrepr offset starts_line is_jump_target")
+
+class Instruction(_Instruction):
+    """Details for a bytecode operation
+
+       Defined fields:
+         opname - human readable name for operation
+         opcode - numeric code for operation
+         arg - numeric argument to operation (if any), otherwise None
+         argval - resolved arg value (if known), otherwise same as arg
+         argrepr - human readable description of operation argument
+         offset - start index of operation within bytecode sequence
+         starts_line - line started by this opcode (if any), otherwise None
+         is_jump_target - True if other code jumps to here, otherwise False
+    """
+
+    def _disassemble(self, lineno_width=3, mark_as_current=False):
+        """Format instruction details for inclusion in disassembly output
+
+        *lineno_width* sets the width of the line number field (0 omits it)
+        *mark_as_current* inserts a '-->' marker arrow as part of the line
+        """
+        fields = []
+        # Column: Source code line number
+        if lineno_width:
+            if self.starts_line is not None:
+                lineno_fmt = "%%%dd" % lineno_width
+                fields.append(lineno_fmt % self.starts_line)
+            else:
+                fields.append(' ' * lineno_width)
+        # Column: Current instruction indicator
+        if mark_as_current:
+            fields.append('-->')
+        else:
+            fields.append('   ')
+        # Column: Jump target marker
+        if self.is_jump_target:
+            fields.append('>>')
+        else:
+            fields.append('  ')
+        # Column: Instruction offset from start of code sequence
+        fields.append(repr(self.offset).rjust(4))
+        # Column: Opcode name
+        fields.append(self.opname.ljust(20))
+        # Column: Opcode argument
+        if self.arg is not None:
+            fields.append(repr(self.arg).rjust(5))
+            # Column: Opcode argument details
+            if self.argrepr:
+                fields.append('(' + self.argrepr + ')')
+        return ' '.join(fields).rstrip()
+
+
+def get_instructions(x, first_line=None):
+    """Iterator for the opcodes in methods, functions or code
+
+    Generates a series of Instruction named tuples giving the details of
+    each operations in the supplied code.
+
+    If *first_line* is not None, it indicates the line number that should
+    be reported for the first source line in the disassembled code.
+    Otherwise, the source line information (if any) is taken directly from
+    the disassembled code object.
+    """
+    co = _get_code_object(x)
+    cell_names = co.co_cellvars + co.co_freevars
+    linestarts = dict(findlinestarts(co))
+    if first_line is not None:
+        line_offset = first_line - co.co_firstlineno
+    else:
+        line_offset = 0
+    return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
+                                   co.co_consts, cell_names, linestarts,
+                                   line_offset)
+
+def _get_const_info(const_index, const_list):
+    """Helper to get optional details about const references
+
+       Returns the dereferenced constant and its repr if the constant
+       list is defined.
+       Otherwise returns the constant index and its repr().
+    """
+    argval = const_index
+    if const_list is not None:
+        argval = const_list[const_index]
+    return argval, repr(argval)
+
+def _get_name_info(name_index, name_list):
+    """Helper to get optional details about named references
+
+       Returns the dereferenced name as both value and repr if the name
+       list is defined.
+       Otherwise returns the name index and its repr().
+    """
+    argval = name_index
+    if name_list is not None:
+        argval = name_list[name_index]
+        argrepr = argval
+    else:
+        argrepr = repr(argval)
+    return argval, argrepr
+
+
+def _get_instructions_bytes(code, varnames=None, names=None, constants=None,
+                      cells=None, linestarts=None, line_offset=0):
+    """Iterate over the instructions in a bytecode string.
+
+    Generates a sequence of Instruction namedtuples giving the details of each
+    opcode.  Additional information about the code's runtime environment
+    (e.g. variable names, constants) can be specified using optional
+    arguments.
+
+    """
+    labels = findlabels(code)
+    starts_line = None
+    for offset, op, arg in _unpack_opargs(code):
+        if linestarts is not None:
+            starts_line = linestarts.get(offset, None)
+            if starts_line is not None:
+                starts_line += line_offset
+        is_jump_target = offset in labels
+        argval = None
+        argrepr = ''
+        if arg is not None:
+            #  Set argval to the dereferenced value of the argument when
+            #  available, and argrepr to the string representation of argval.
+            #    _disassemble_bytes needs the string repr of the
+            #    raw name index for LOAD_GLOBAL, LOAD_CONST, etc.
+            argval = arg
+            if op in hasconst:
+                argval, argrepr = _get_const_info(arg, constants)
+            elif op in hasname:
+                argval, argrepr = _get_name_info(arg, names)
+            elif op in hasjrel:
+                argval = offset + 2 + arg
+                argrepr = "to " + repr(argval)
+            elif op in haslocal:
+                argval, argrepr = _get_name_info(arg, varnames)
+            elif op in hascompare:
+                argval = cmp_op[arg]
+                argrepr = argval
+            elif op in hasfree:
+                argval, argrepr = _get_name_info(arg, cells)
+            elif op == FORMAT_VALUE:
+                argval = ((None, str, repr, ascii)[arg & 0x3], bool(arg & 0x4))
+                argrepr = ('', 'str', 'repr', 'ascii')[arg & 0x3]
+                if argval[1]:
+                    if argrepr:
+                        argrepr += ', '
+                    argrepr += 'with format'
+        yield Instruction(opname[op], op,
+                          arg, argval, argrepr,
+                          offset, starts_line, is_jump_target)
+
 def disassemble(co, lasti=-1):
     """Disassemble a code object."""
-    XXX
-    code = co.co_code
-    labels = findlabels(code)
+    cell_names = co.co_cellvars + co.co_freevars
     linestarts = dict(findlinestarts(co))
-    n = len(code)
-    i = 0
-    extended_arg = 0
-    free = None
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        if i in linestarts:
-            if i > 0:
-                print
-            print "%3d" % linestarts[i],
-        else:
-            print '   ',
+    _disassemble_bytes(co.co_code, lasti, co.co_varnames, co.co_names,
+                       co.co_consts, cell_names, linestarts)
 
-        if i == lasti: print '-->',
-        else: print '   ',
-        if i in labels: print '>>',
-        else: print '  ',
-        print repr(i).rjust(4),
-        print opname[op].ljust(20),
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256 + extended_arg
-            extended_arg = 0
-            i = i+2
-            if op == EXTENDED_ARG:
-                extended_arg = oparg*65536L
-            print repr(oparg).rjust(5),
-            if op in hasconst:
-                print '(' + repr(co.co_consts[oparg]) + ')',
-            elif op in hasname:
-                print '(' + co.co_names[oparg] + ')',
-            elif op in hasjrel:
-                print '(to ' + repr(i + oparg) + ')',
-            elif op in haslocal:
-                print '(' + co.co_varnames[oparg] + ')',
-            elif op in hascompare:
-                print '(' + cmp_op[oparg] + ')',
-            elif op in hasfree:
-                if free is None:
-                    free = co.co_cellvars + co.co_freevars
-                print '(' + free[oparg] + ')',
-            elif op in hasnargs:
-                print '(%d positional, %d keyword pair)' % \
-                      (ord(code[i-2]), ord(code[i-1])),
-        print
+def _disassemble_bytes(code, lasti=-1, varnames=None, names=None,
+                       constants=None, cells=None, linestarts=None,
+                       line_offset=0):
+    # Omit the line number column entirely if we have no line number info
+    show_lineno = linestarts is not None
+    # TODO?: Adjust width upwards if max(linestarts.values()) >= 1000?
+    lineno_width = 3 if show_lineno else 0
+    for instr in _get_instructions_bytes(code, varnames, names,
+                                         constants, cells, linestarts,
+                                         line_offset=line_offset):
+        new_source_line = (show_lineno and
+                           instr.starts_line is not None and
+                           instr.offset > 0)
+        if new_source_line:
+            print()
+        is_current_instr = instr.offset == lasti
+        print(instr._disassemble(lineno_width, is_current_instr))
 
-def disassemble_string(code, lasti=-1, varnames=None, names=None,
-                       constants=None):
-    labels = findlabels(code)
-    n = len(code)
-    i = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        if i == lasti: print '-->',
-        else: print '   ',
-        if i in labels: print '>>',
-        else: print '  ',
-        print repr(i).rjust(4),
-        print opname[op].ljust(15),
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256
-            i = i+2
-            print repr(oparg).rjust(5),
-            if op in hasconst:
-                if constants:
-                    print '(' + repr(constants[oparg]) + ')',
-                else:
-                    print '(%d)'%oparg,
-            elif op in hasname:
-                if names is not None:
-                    print '(' + names[oparg] + ')',
-                else:
-                    print '(%d)'%oparg,
-            elif op in hasjrel:
-                print '(to ' + repr(i + oparg) + ')',
-            elif op in haslocal:
-                if varnames:
-                    print '(' + varnames[oparg] + ')',
-                else:
-                    print '(%d)' % oparg,
-            elif op in hascompare:
-                print '(' + cmp_op[oparg] + ')',
-            elif op in hasnargs:
-                print '(%d positional, %d keyword pair)' % \
-                      (ord(code[i-2]), ord(code[i-1])),
-        print
+def _disassemble_str(source):
+    """Compile the source string, then disassemble the code object."""
+    disassemble(_try_compile(source, '<dis>'))
 
 disco = disassemble                     # XXX For backwards compatibility
 
+def _unpack_opargs(code):
+    extended_arg = 0
+    for i in range(0, len(code), 2):
+        op = ord(code[i])
+        if op >= HAVE_ARGUMENT:
+            arg = ord(code[i+1]) | extended_arg
+            extended_arg = (arg << 8) if op == EXTENDED_ARG else 0
+        else:
+            arg = None
+        yield (i, op, arg)
+
 def findlabels(code):
     """Detect all offsets in a byte code which are jump targets.
 
@@ -162,23 +369,16 @@
 
     """
     labels = []
-    n = len(code)
-    i = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256
-            i = i+2
-            label = -1
+    for offset, op, arg in _unpack_opargs(code):
+        if arg is not None:
             if op in hasjrel:
-                label = i+oparg
+                label = offset + 2 + arg
             elif op in hasjabs:
-                label = oparg
-            if label >= 0:
-                if label not in labels:
-                    labels.append(label)
+                label = arg
+            else:
+                continue
+            if label not in labels:
+                labels.append(label)
     return labels
 
 def findlinestarts(code):
@@ -187,8 +387,8 @@
     Generate pairs (offset, lineno) as described in Python/compile.c.
 
     """
-    byte_increments = [ord(c) for c in code.co_lnotab[0::2]]
-    line_increments = [ord(c) for c in code.co_lnotab[1::2]]
+    byte_increments = code.co_lnotab[0::2]
+    line_increments = code.co_lnotab[1::2]
 
     lastlineno = None
     lineno = code.co_firstlineno
@@ -206,27 +406,77 @@
     if lineno != lastlineno:
         yield (addr, lineno)
 
+class Bytecode:
+    """The bytecode operations of a piece of code
+
+    Instantiate this with a function, method, string of code, or a code object
+    (as returned by compile()).
+
+    Iterating over this yields the bytecode operations as Instruction instances.
+    """
+    def __init__(self, x, first_line=None, current_offset=None):
+        self.codeobj = co = _get_code_object(x)
+        if first_line is None:
+            self.first_line = co.co_firstlineno
+            self._line_offset = 0
+        else:
+            self.first_line = first_line
+            self._line_offset = first_line - co.co_firstlineno
+        self._cell_names = co.co_cellvars + co.co_freevars
+        self._linestarts = dict(findlinestarts(co))
+        self._original_object = x
+        self.current_offset = current_offset
+
+    def __iter__(self):
+        co = self.codeobj
+        return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
+                                       co.co_consts, self._cell_names,
+                                       self._linestarts,
+                                       line_offset=self._line_offset)
+
+    def __repr__(self):
+        return "{}({!r})".format(self.__class__.__name__,
+                                 self._original_object)
+
+    @classmethod
+    def from_traceback(cls, tb):
+        """ Construct a Bytecode from the given traceback """
+        while tb.tb_next:
+            tb = tb.tb_next
+        return cls(tb.tb_frame.f_code, current_offset=tb.tb_lasti)
+
+    def info(self):
+        """Return formatted information about the code object."""
+        return _format_code_info(self.codeobj)
+
+    def dis(self):
+        """Return a formatted view of the bytecode operations."""
+        co = self.codeobj
+        if self.current_offset is not None:
+            offset = self.current_offset
+        else:
+            offset = -1
+        with io.StringIO() as output:
+            _disassemble_bytes(co.co_code, varnames=co.co_varnames,
+                               names=co.co_names, constants=co.co_consts,
+                               cells=self._cell_names,
+                               linestarts=self._linestarts,
+                               line_offset=self._line_offset,
+                               file=output,
+                               lasti=offset)
+            return output.getvalue()
+
+
 def _test():
     """Simple test program to disassemble a file."""
-    if sys.argv[1:]:
-        if sys.argv[2:]:
-            sys.stderr.write("usage: python dis.py [-|file]\n")
-            sys.exit(2)
-        fn = sys.argv[1]
-        if not fn or fn == "-":
-            fn = None
-    else:
-        fn = None
-    if fn is None:
-        f = sys.stdin
-    else:
-        f = open(fn)
-    source = f.read()
-    if fn is not None:
-        f.close()
-    else:
-        fn = "<stdin>"
-    code = compile(source, fn, "exec")
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('infile', type=argparse.FileType(), nargs='?', default='-')
+    args = parser.parse_args()
+    with args.infile as infile:
+        source = infile.read()
+    code = compile(source, args.infile.name, "exec")
     dis(code)
 
 if __name__ == "__main__":
diff --git a/pypy/tool/opcode3.py b/pypy/tool/opcode3.py
--- a/pypy/tool/opcode3.py
+++ b/pypy/tool/opcode3.py
@@ -2,10 +2,8 @@
 """
 opcode module - potentially shared between dis and other modules which
 operate on bytecodes (e.g. peephole optimizers).
-"Backported" from Python 3 to Python 2 land - an excact copy of lib-python/3/opcode.py
 """
 
-
 __all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs",
            "haslocal", "hascompare", "hasfree", "opname", "opmap",
            "HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"]
@@ -33,10 +31,12 @@
 haslocal = []
 hascompare = []
 hasfree = []
-hasnargs = [] # unused
+hasnargs = []
 
 opmap = {}
-opname = ['<%r>' % (op,) for op in range(256)]
+opname = [''] * 256
+for op in range(256): opname[op] = '<%r>' % (op,)
+del op
 
 def def_op(name, op):
     opname[op] = name
@@ -174,9 +174,11 @@
 name_op('STORE_ANNOTATION', 127) # Index in name list
 
 def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
-def_op('CALL_FUNCTION', 131)    # #args
-def_op('MAKE_FUNCTION', 132)    # Flags
+def_op('CALL_FUNCTION', 131)    # #args + (#kwargs << 8)
+hasnargs.append(131)
+def_op('MAKE_FUNCTION', 132)    # Number of args with default values
 def_op('BUILD_SLICE', 133)      # Number of items
+def_op('MAKE_CLOSURE', 134)
 def_op('LOAD_CLOSURE', 135)
 hasfree.append(135)
 def_op('LOAD_DEREF', 136)
@@ -186,8 +188,12 @@
 def_op('DELETE_DEREF', 138)
 hasfree.append(138)
 
-def_op('CALL_FUNCTION_KW', 141)  # #args + #kwargs
-def_op('CALL_FUNCTION_EX', 142)  # Flags
+def_op('CALL_FUNCTION_VAR', 140)     # #args + (#kwargs << 8)
+hasnargs.append(140)
+def_op('CALL_FUNCTION_KW', 141)      # #args + (#kwargs << 8)
+hasnargs.append(141)
+def_op('CALL_FUNCTION_VAR_KW', 142)  # #args + (#kwargs << 8)
+hasnargs.append(142)
 
 jrel_op('SETUP_WITH', 143)
 
@@ -198,6 +204,8 @@
 def_op('LOAD_CLASSDEREF', 148)
 hasfree.append(148)
 
+jrel_op('SETUP_ASYNC_WITH', 154)
+
 def_op('EXTENDED_ARG', 144)
 EXTENDED_ARG = 144
 
@@ -207,12 +215,8 @@
 def_op('BUILD_TUPLE_UNPACK', 152)
 def_op('BUILD_SET_UNPACK', 153)
 
-jrel_op('SETUP_ASYNC_WITH', 154)
-
-def_op('FORMAT_VALUE', 155)
-def_op('BUILD_CONST_KEY_MAP', 156)
-def_op('BUILD_STRING', 157)
-def_op('BUILD_TUPLE_UNPACK_WITH_CALL', 158)
+def_op('FORMAT_VALUE', 155)   # in CPython 3.6, but available in PyPy from 3.5
+def_op('BUILD_STRING', 157)   # in CPython 3.6, but available in PyPy from 3.5
 
 # pypy modification, experimental bytecode
 def_op('LOOKUP_METHOD', 201)          # Index in name list

From pypy.commits at gmail.com  Mon May 14 17:26:10 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 14:26:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: fix test_compiler
Message-ID: <5af9fef2.1c69fb81.f357f.127c@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94587:4b345ab7bd57
Date: 2018-05-14 15:54 +0200
http://bitbucket.org/pypy/pypy/changeset/4b345ab7bd57/

Log:	fix test_compiler

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -73,8 +73,8 @@
         # for now, we compile evalexpr with CPython's compiler but run
         # it with our own interpreter to extract the data from w_dict
         space = self.space
-        pyco_expr = space.createcompiler().compile(evalexpr, '<evalexpr>', 'eval')
-        w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict)
+        pyco_expr = space.createcompiler().compile(evalexpr, '<evalexpr>', 'eval', 0)
+        w_res = space.exec_(pyco_expr, w_dict, w_dict)
         res = space.str_w(space.repr(w_res))
         expected_repr = self.get_py3_repr(expected)
         if isinstance(expected, float):
@@ -1207,12 +1207,12 @@
         yield self.st, """z=f'{f"{0}"*3}'""", 'z', '000'
 
     def test_fstring_error(self):
-        raises(SyntaxError, self.run, "f'{}'")
-        raises(SyntaxError, self.run, "f'{   \t   }'")
-        raises(SyntaxError, self.run, "f'{5#}'")
-        raises(SyntaxError, self.run, "f'{5)#}'")
-        raises(SyntaxError, self.run, "f'''{5)\n#}'''")
-        raises(SyntaxError, self.run, "f'\\x'")
+        py.test.raises(SyntaxError, self.run, "f'{}'")
+        py.test.raises(SyntaxError, self.run, "f'{   \t   }'")
+        py.test.raises(SyntaxError, self.run, "f'{5#}'")
+        py.test.raises(SyntaxError, self.run, "f'{5)#}'")
+        py.test.raises(SyntaxError, self.run, "f'''{5)\n#}'''")
+        py.test.raises(SyntaxError, self.run, "f'\\x'")
 
     def test_fstring_encoding(self):
         src = """# -*- coding: latin-1 -*-\nz=ord(f'{"\xd8"}')\n"""

From pypy.commits at gmail.com  Mon May 14 17:29:43 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 14 May 2018 14:29:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix raises
Message-ID: <5af9ffc7.1c69fb81.db018.b883@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94589:5cf998ecbe9b
Date: 2018-05-14 23:29 +0200
http://bitbucket.org/pypy/pypy/changeset/5cf998ecbe9b/

Log:	fix raises

diff --git a/pypy/interpreter/astcompiler/test/test_validate.py b/pypy/interpreter/astcompiler/test/test_validate.py
--- a/pypy/interpreter/astcompiler/test/test_validate.py
+++ b/pypy/interpreter/astcompiler/test/test_validate.py
@@ -1,4 +1,5 @@
 import os
+from pytest import raises
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.astcompiler import ast

From pypy.commits at gmail.com  Mon May 14 17:49:37 2018
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 14 May 2018 14:49:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: The py3k_ variants have been replaced by
 unicodehandler functions
Message-ID: <5afa0471.1c69fb81.e08a2.b011@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r94592:2c3be745bc08
Date: 2018-05-14 22:47 +0100
http://bitbucket.org/pypy/pypy/changeset/2c3be745bc08/

Log:	The py3k_ variants have been replaced by unicodehandler functions

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -621,8 +621,6 @@
     try:
         func = getattr(unicodehelper, impl_name)
     except AttributeError:
-        if hasattr(runicode, 'py3k_' + impl_name):
-            impl_name = 'py3k_' + impl_name
         func = getattr(runicode, impl_name)
     return func
 

From pypy.commits at gmail.com  Mon May 14 17:49:32 2018
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 14 May 2018 14:49:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add utf_16 functions to
 unicodehandler.py
Message-ID: <5afa046c.1c69fb81.9d65c.9e75@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r94590:7b8b1a6cec71
Date: 2018-05-14 22:16 +0100
http://bitbucket.org/pypy/pypy/changeset/7b8b1a6cec71/

Log:	Add utf_16 functions to unicodehandler.py

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -78,6 +78,216 @@
         allow_surrogates=True)
 
 # ____________________________________________________________
+# utf-16
+
+def str_decode_utf_16(s, size, errors, final=True,
+                      errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "native")
+    return result, length
+
+def str_decode_utf_16_be(s, size, errors, final=True,
+                         errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "big")
+    return result, length
+
+def str_decode_utf_16_le(s, size, errors, final=True,
+                         errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "little")
+    return result, length
+
+def str_decode_utf_16_helper(s, size, errors, final=True,
+                             errorhandler=None,
+                             byteorder="native",
+                             public_encoding_name='utf16'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+    bo = 0
+
+    if BYTEORDER == 'little':
+        ihi = 1
+        ilo = 0
+    else:
+        ihi = 0
+        ilo = 1
+
+    #  Check for BOM marks (U+FEFF) in the input and adjust current
+    #  byte order setting accordingly. In native mode, the leading BOM
+    #  mark is skipped, in all other modes, it is copied to the output
+    #  stream as-is (giving a ZWNBSP character).
+    pos = 0
+    if byteorder == 'native':
+        if size >= 2:
+            bom = (ord(s[ihi]) << 8) | ord(s[ilo])
+            if BYTEORDER == 'little':
+                if bom == 0xFEFF:
+                    pos += 2
+                    bo = -1
+                elif bom == 0xFFFE:
+                    pos += 2
+                    bo = 1
+            else:
+                if bom == 0xFEFF:
+                    pos += 2
+                    bo = 1
+                elif bom == 0xFFFE:
+                    pos += 2
+                    bo = -1
+    elif byteorder == 'little':
+        bo = -1
+    else:
+        bo = 1
+    if size == 0:
+        return u'', 0, bo
+    if bo == -1:
+        # force little endian
+        ihi = 1
+        ilo = 0
+
+    elif bo == 1:
+        # force big endian
+        ihi = 0
+        ilo = 1
+
+    result = UnicodeBuilder(size // 2)
+
+    #XXX I think the errors are not correctly handled here
+    while pos < size:
+        # remaining bytes at the end? (size should be even)
+        if len(s) - pos < 2:
+            if not final:
+                break
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "truncated data",
+                                  s, pos, len(s))
+            result.append(r)
+            if len(s) - pos < 2:
+                break
+        ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo])
+        pos += 2
+        if ch < 0xD800 or ch > 0xDFFF:
+            result.append(unichr(ch))
+            continue
+        # UTF-16 code pair:
+        if len(s) - pos < 2:
+            pos -= 2
+            if not final:
+                break
+            errmsg = "unexpected end of data"
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  errmsg, s, pos, len(s))
+            result.append(r)
+            if len(s) - pos < 2:
+                break
+        elif 0xD800 <= ch <= 0xDBFF:
+            ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo])
+            pos += 2
+            if 0xDC00 <= ch2 <= 0xDFFF:
+                if MAXUNICODE < 65536:
+                    result.append(unichr(ch))
+                    result.append(unichr(ch2))
+                else:
+                    result.append(UNICHR((((ch & 0x3FF)<<10) |
+                                           (ch2 & 0x3FF)) + 0x10000))
+                continue
+            else:
+                r, pos = errorhandler(errors, public_encoding_name,
+                                      "illegal UTF-16 surrogate",
+                                      s, pos - 4, pos - 2)
+                result.append(r)
+        else:
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "illegal encoding",
+                                  s, pos - 2, pos)
+            result.append(r)
+    return result.build(), pos, bo
+
+def _STORECHAR(result, CH, byteorder):
+    hi = chr(((CH) >> 8) & 0xff)
+    lo = chr((CH) & 0xff)
+    if byteorder == 'little':
+        result.append(lo)
+        result.append(hi)
+    else:
+        result.append(hi)
+        result.append(lo)
+
+def unicode_encode_utf_16_helper(s, size, errors,
+                                 errorhandler=None,
+                                 allow_surrogates=True,
+                                 byteorder='little',
+                                 public_encoding_name='utf16'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_encode
+    if size == 0:
+        if byteorder == 'native':
+            result = StringBuilder(2)
+            _STORECHAR(result, 0xFEFF, BYTEORDER)
+            return result.build()
+        return ""
+
+    result = StringBuilder(size * 2 + 2)
+    if byteorder == 'native':
+        _STORECHAR(result, 0xFEFF, BYTEORDER)
+        byteorder = BYTEORDER
+
+    pos = 0
+    while pos < size:
+        ch = ord(s[pos])
+        pos += 1
+
+        if ch < 0xD800:
+            _STORECHAR(result, ch, byteorder)
+        elif ch >= 0x10000:
+            _STORECHAR(result, 0xD800 | ((ch-0x10000) >> 10), byteorder)
+            _STORECHAR(result, 0xDC00 | ((ch-0x10000) & 0x3FF), byteorder)
+        elif ch >= 0xE000 or allow_surrogates:
+            _STORECHAR(result, ch, byteorder)
+        else:
+            ru, rs, pos = errorhandler(errors, public_encoding_name,
+                                       'surrogates not allowed',
+                                       s, pos-1, pos)
+            if rs is not None:
+                # py3k only
+                if len(rs) % 2 != 0:
+                    errorhandler('strict', public_encoding_name,
+                                 'surrogates not allowed',
+                                 s, pos-1, pos)
+                result.append(rs)
+                continue
+            for ch in ru:
+                if ord(ch) < 0xD800:
+                    _STORECHAR(result, ord(ch), byteorder)
+                else:
+                    errorhandler('strict', public_encoding_name,
+                                 'surrogates not allowed',
+                                 s, pos-1, pos)
+            continue
+
+    return result.build()
+
+def unicode_encode_utf_16(s, size, errors,
+                          errorhandler=None,
+                          allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "native")
+
+def unicode_encode_utf_16_be(s, size, errors,
+                             errorhandler=None,
+                             allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "big")
+
+def unicode_encode_utf_16_le(s, size, errors,
+                             errorhandler=None,
+                             allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "little")
+
+
+# ____________________________________________________________
 # utf-32
 
 def str_decode_utf_32(s, size, errors, final=True,
@@ -98,24 +308,6 @@
         s, size, errors, final, errorhandler, "little")
     return result, length
 
-def py3k_str_decode_utf_32(s, size, errors, final=True,
-                           errorhandler=None):
-    result, length, byteorder = str_decode_utf_32_helper(
-        s, size, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2)
-    return result, length
-
-def py3k_str_decode_utf_32_be(s, size, errors, final=True,
-                              errorhandler=None):
-    result, length, byteorder = str_decode_utf_32_helper(
-        s, size, errors, final, errorhandler, "big", 'utf-32-be')
-    return result, length
-
-def py3k_str_decode_utf_32_le(s, size, errors, final=True,
-                              errorhandler=None):
-    result, length, byteorder = str_decode_utf_32_helper(
-        s, size, errors, final, errorhandler, "little", 'utf-32-le')
-    return result, length
-
 BOM32_DIRECT = intmask(0x0000FEFF)
 BOM32_REVERSE = intmask(0xFFFE0000)
 
@@ -284,21 +476,3 @@
                              errorhandler=None, allow_surrogates=True):
     return unicode_encode_utf_32_helper(s, size, errors, errorhandler,
                                         allow_surrogates, "little")
-
-def py3k_unicode_encode_utf_32(s, size, errors,
-                               errorhandler=None, allow_surrogates=True):
-    return unicode_encode_utf_32_helper(s, size, errors, errorhandler,
-                                        allow_surrogates, "native",
-                                        'utf-32-' + BYTEORDER2)
-
-def py3k_unicode_encode_utf_32_be(s, size, errors,
-                                  errorhandler=None, allow_surrogates=True):
-    return unicode_encode_utf_32_helper(s, size, errors, errorhandler,
-                                        allow_surrogates, "big",
-                                        'utf-32-be')
-
-def py3k_unicode_encode_utf_32_le(s, size, errors,
-                                  errorhandler=None, allow_surrogates=True):
-    return unicode_encode_utf_32_helper(s, size, errors, errorhandler,
-                                        allow_surrogates, "little",
-                                        'utf-32-le')

From pypy.commits at gmail.com  Mon May 14 17:49:39 2018
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 14 May 2018 14:49:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <5afa0473.1c69fb81.e08a2.b01d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r94593:104fba53d2eb
Date: 2018-05-14 22:48 +0100
http://bitbucket.org/pypy/pypy/changeset/104fba53d2eb/

Log:	merge heads

diff --git a/pypy/interpreter/astcompiler/test/test_validate.py b/pypy/interpreter/astcompiler/test/test_validate.py
--- a/pypy/interpreter/astcompiler/test/test_validate.py
+++ b/pypy/interpreter/astcompiler/test/test_validate.py
@@ -1,4 +1,5 @@
 import os
+from pytest import raises
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.astcompiler import ast
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -375,7 +375,7 @@
         child.expect('>>>')
 
     def test_atexit(self):
-        skip("Python3 atexit is a builtin module")
+        py.test.skip("Python3 atexit is a builtin module")
         child = self.spawn([])
         child.expect('>>> ')
         child.sendline('def f(): print("foobye")')
@@ -525,9 +525,9 @@
 
     def test_options_i_m(self, monkeypatch):
         if sys.platform == "win32":
-            skip("close_fds is not supported on Windows platforms")
+            py.test.skip("close_fds is not supported on Windows platforms")
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'mymodule.py')
         p = os.path.abspath(p)
         monkeypatch.chdir(os.path.dirname(app_main))
@@ -557,7 +557,7 @@
 
     def test_options_u_i(self):
         if sys.platform == "win32":
-            skip("close_fds is not supported on Windows platforms")
+            py.test.skip("close_fds is not supported on Windows platforms")
         import subprocess, select, os
         pipe = subprocess.Popen([get_python3(), app_main, "-u", "-i"],
                                 stdout=subprocess.PIPE,
@@ -614,7 +614,7 @@
             del os.environ['PYTHONINSPECT_']
 
     def test_stdout_flushes_before_stdin_blocks(self):
-        skip("Python3 does not implement this behavior")
+        py.test.skip("Python3 does not implement this behavior")
         # This doesn't really test app_main.py, but a behavior that
         # can only be checked on top of py.py with pexpect.
         path = getscript("""
@@ -632,7 +632,7 @@
 
     def test_no_space_before_argument(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         child = self.spawn(['-cprint("hel" + "lo")'])
         child.expect('hello')
 
@@ -753,7 +753,7 @@
 
     def test_option_m(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'mymodule.py')
         p = os.path.abspath(p)
         monkeypatch.chdir(os.path.dirname(app_main))
@@ -767,7 +767,7 @@
 
     def test_option_m_package(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)),
                          'mypackage', '__main__.py')
         p = os.path.abspath(p)

From pypy.commits at gmail.com  Mon May 14 17:49:35 2018
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 14 May 2018 14:49:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5afa046f.1c69fb81.6098c.6b0d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r94591:2396fb397495
Date: 2018-05-14 22:34 +0100
http://bitbucket.org/pypy/pypy/changeset/2396fb397495/

Log:	hg merge default

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -168,6 +168,222 @@
     return decode_utf8(space, string, allow_surrogates=True)
 
 # ____________________________________________________________
+# utf-16
+
+def str_decode_utf_16(s, size, errors, final=True,
+                           errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "native",
+                                                         'utf-16-' + BYTEORDER2)
+    return result, length
+
+def str_decode_utf_16_be(s, size, errors, final=True,
+                              errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "big",
+                                                         'utf-16-be')
+    return result, length
+
+def str_decode_utf_16_le(s, size, errors, final=True,
+                              errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "little",
+                                                         'utf-16-le')
+    return result, length
+
+def str_decode_utf_16_helper(s, size, errors, final=True,
+                             errorhandler=None,
+                             byteorder="native",
+                             public_encoding_name='utf16'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+    bo = 0
+
+    if BYTEORDER == 'little':
+        ihi = 1
+        ilo = 0
+    else:
+        ihi = 0
+        ilo = 1
+
+    #  Check for BOM marks (U+FEFF) in the input and adjust current
+    #  byte order setting accordingly. In native mode, the leading BOM
+    #  mark is skipped, in all other modes, it is copied to the output
+    #  stream as-is (giving a ZWNBSP character).
+    pos = 0
+    if byteorder == 'native':
+        if size >= 2:
+            bom = (ord(s[ihi]) << 8) | ord(s[ilo])
+            if BYTEORDER == 'little':
+                if bom == 0xFEFF:
+                    pos += 2
+                    bo = -1
+                elif bom == 0xFFFE:
+                    pos += 2
+                    bo = 1
+            else:
+                if bom == 0xFEFF:
+                    pos += 2
+                    bo = 1
+                elif bom == 0xFFFE:
+                    pos += 2
+                    bo = -1
+    elif byteorder == 'little':
+        bo = -1
+    else:
+        bo = 1
+    if size == 0:
+        return u'', 0, bo
+    if bo == -1:
+        # force little endian
+        ihi = 1
+        ilo = 0
+
+    elif bo == 1:
+        # force big endian
+        ihi = 0
+        ilo = 1
+
+    result = UnicodeBuilder(size // 2)
+
+    #XXX I think the errors are not correctly handled here
+    while pos < size:
+        # remaining bytes at the end? (size should be even)
+        if len(s) - pos < 2:
+            if not final:
+                break
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "truncated data",
+                                  s, pos, len(s))
+            result.append(r)
+            if len(s) - pos < 2:
+                break
+        ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo])
+        pos += 2
+        if ch < 0xD800 or ch > 0xDFFF:
+            result.append(unichr(ch))
+            continue
+        # UTF-16 code pair:
+        if len(s) - pos < 2:
+            pos -= 2
+            if not final:
+                break
+            errmsg = "unexpected end of data"
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  errmsg, s, pos, len(s))
+            result.append(r)
+            if len(s) - pos < 2:
+                break
+        elif 0xD800 <= ch <= 0xDBFF:
+            ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo])
+            pos += 2
+            if 0xDC00 <= ch2 <= 0xDFFF:
+                if MAXUNICODE < 65536:
+                    result.append(unichr(ch))
+                    result.append(unichr(ch2))
+                else:
+                    result.append(UNICHR((((ch & 0x3FF)<<10) |
+                                           (ch2 & 0x3FF)) + 0x10000))
+                continue
+            else:
+                r, pos = errorhandler(errors, public_encoding_name,
+                                      "illegal UTF-16 surrogate",
+                                      s, pos - 4, pos - 2)
+                result.append(r)
+        else:
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "illegal encoding",
+                                  s, pos - 2, pos)
+            result.append(r)
+    return result.build(), pos, bo
+
+def _STORECHAR(result, CH, byteorder):
+    hi = chr(((CH) >> 8) & 0xff)
+    lo = chr((CH) & 0xff)
+    if byteorder == 'little':
+        result.append(lo)
+        result.append(hi)
+    else:
+        result.append(hi)
+        result.append(lo)
+
+def unicode_encode_utf_16_helper(s, size, errors,
+                                 errorhandler=None,
+                                 allow_surrogates=True,
+                                 byteorder='little',
+                                 public_encoding_name='utf16'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_encode
+    if size == 0:
+        if byteorder == 'native':
+            result = StringBuilder(2)
+            _STORECHAR(result, 0xFEFF, BYTEORDER)
+            return result.build()
+        return ""
+
+    result = StringBuilder(size * 2 + 2)
+    if byteorder == 'native':
+        _STORECHAR(result, 0xFEFF, BYTEORDER)
+        byteorder = BYTEORDER
+
+    pos = 0
+    while pos < size:
+        ch = ord(s[pos])
+        pos += 1
+
+        if ch < 0xD800:
+            _STORECHAR(result, ch, byteorder)
+        elif ch >= 0x10000:
+            _STORECHAR(result, 0xD800 | ((ch-0x10000) >> 10), byteorder)
+            _STORECHAR(result, 0xDC00 | ((ch-0x10000) & 0x3FF), byteorder)
+        elif ch >= 0xE000 or allow_surrogates:
+            _STORECHAR(result, ch, byteorder)
+        else:
+            ru, rs, pos = errorhandler(errors, public_encoding_name,
+                                       'surrogates not allowed',
+                                       s, pos-1, pos)
+            if rs is not None:
+                # py3k only
+                if len(rs) % 2 != 0:
+                    errorhandler('strict', public_encoding_name,
+                                 'surrogates not allowed',
+                                 s, pos-1, pos)
+                result.append(rs)
+                continue
+            for ch in ru:
+                if ord(ch) < 0xD800:
+                    _STORECHAR(result, ord(ch), byteorder)
+                else:
+                    errorhandler('strict', public_encoding_name,
+                                 'surrogates not allowed',
+                                 s, pos-1, pos)
+            continue
+
+    return result.build()
+
+def unicode_encode_utf_16(s, size, errors,
+                          errorhandler=None,
+                          allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "native",
+                                        'utf-16-' + BYTEORDER2)
+
+def unicode_encode_utf_16_be(s, size, errors,
+                             errorhandler=None,
+                             allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "big",
+                                        'utf-16-be')
+
+def unicode_encode_utf_16_le(s, size, errors,
+                             errorhandler=None,
+                             allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "little",
+                                        'utf-16-le')
+
+
+# ____________________________________________________________
 # utf-32
 
 def str_decode_utf_32(s, size, errors, final=True,

From pypy.commits at gmail.com  Tue May 15 20:06:20 2018
From: pypy.commits at gmail.com (alex_gaynor)
Date: Tue, 15 May 2018 17:06:20 -0700 (PDT)
Subject: [pypy-commit] pypy
 alex_gaynor/remove-an-unneeded-call-into-openssl-th-1526429141011: Remove
 an unneeded call into OpenSSL
Message-ID: <5afb75fc.1c69fb81.c1edb.e2c5@mx.google.com>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: alex_gaynor/remove-an-unneeded-call-into-openssl-th-1526429141011
Changeset: r94594:7a78467a6e72
Date: 2018-05-16 00:05 +0000
http://bitbucket.org/pypy/pypy/changeset/7a78467a6e72/

Log:	Remove an unneeded call into OpenSSL

	This call doesn't do anything, on every version of OpenSSL going
	back to forever the call to ERR_clear_error() calls ERR_get_state(),
	so it's always been a no-op.

	Same change for CPython: https://github.com/python/cpython/pull/6887

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -224,7 +224,6 @@
         if server_hostname:
             self.server_hostname = server_hostname.decode('idna', 'strict')
 
-        lib.ERR_get_state()
         lib.ERR_clear_error()
         self.ssl = ssl = ffi.gc(lib.SSL_new(ctx), lib.SSL_free)
 

From pypy.commits at gmail.com  Wed May 16 00:17:48 2018
From: pypy.commits at gmail.com (wlav)
Date: Tue, 15 May 2018 21:17:48 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: add helper to extract outer
 namespace from a C++ name
Message-ID: <5afbb0ec.1c69fb81.e841.6597@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94595:ba933501a318
Date: 2018-05-07 22:29 -0700
http://bitbucket.org/pypy/pypy/changeset/ba933501a318/

Log:	add helper to extract outer namespace from a C++ name

diff --git a/pypy/module/_cppyy/helper.py b/pypy/module/_cppyy/helper.py
--- a/pypy/module/_cppyy/helper.py
+++ b/pypy/module/_cppyy/helper.py
@@ -59,6 +59,26 @@
         name = name[:_find_qualifier_index(name)]
     return name.strip(' ')
 
+def extract_namespace(name):
+    # find the namespace the named class lives in, take care of templates
+    tpl_open = 0
+    for pos in xrange(len(name)-1, 1, -1):
+        c = name[pos]
+
+        # count '<' and '>' to be able to skip template contents
+        if c == '>':
+            tpl_open += 1
+        elif c == '<':
+            tpl_open -= 1
+
+        # collect name up to "::"
+        elif tpl_open == 0 and c == ':' and name[pos-1] == ':':
+            # found the extend of the scope ... done
+            return name[0:pos-1]
+
+    # no namespace; assume outer scope
+    return ""
+
 
 #- operator mappings --------------------------------------------------------
 _operator_mappings = {}
diff --git a/pypy/module/_cppyy/test/test_helper.py b/pypy/module/_cppyy/test/test_helper.py
--- a/pypy/module/_cppyy/test/test_helper.py
+++ b/pypy/module/_cppyy/test/test_helper.py
@@ -50,3 +50,14 @@
 
     assert helper.map_operator_name(None, "func", 0, "")        == "func"
     assert helper.map_operator_name(None, "some_method", 0, "") == "some_method"
+
+
+def test_namespace_extraction():
+    assert helper.extract_namespace("vector")                        == ""
+    assert helper.extract_namespace("std::vector")                   == "std"
+    assert helper.extract_namespace("std::vector<double>")           == "std"
+    assert helper.extract_namespace("std::vector<std::vector>")      == "std"
+    assert helper.extract_namespace("vector<double>")                == ""
+    assert helper.extract_namespace("vector<std::vector>")           == ""
+    assert helper.extract_namespace("aap::noot::mies::zus")          == "aap::noot::mies"
+

From pypy.commits at gmail.com  Wed May 16 00:17:50 2018
From: pypy.commits at gmail.com (wlav)
Date: Tue, 15 May 2018 21:17:50 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: pythonization improvements
Message-ID: <5afbb0ee.1c69fb81.26992.830e@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94596:10c6393b2cd5
Date: 2018-05-15 20:59 -0700
http://bitbucket.org/pypy/pypy/changeset/10c6393b2cd5/

Log:	pythonization improvements

diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py
--- a/pypy/module/_cppyy/__init__.py
+++ b/pypy/module/_cppyy/__init__.py
@@ -19,12 +19,14 @@
         '_bind_object'           : 'interp_cppyy._bind_object',
         'bind_object'            : 'interp_cppyy.bind_object',
         'move'                   : 'interp_cppyy.move',
+        '_pin_type'              : 'interp_cppyy._pin_type',
     }
 
     appleveldefs = {
         '_post_import_startup'   : 'pythonify._post_import_startup',
+        'Template'               : 'pythonify.CPPTemplate',
         'add_pythonization'      : 'pythonify.add_pythonization',
-        'Template'               : 'pythonify.CPPTemplate',
+        'remove_pythonization'   : 'pythonify.remove_pythonization',
     }
 
     def __init__(self, space, *args):
diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py
--- a/pypy/module/_cppyy/capi/loadable_capi.py
+++ b/pypy/module/_cppyy/capi/loadable_capi.py
@@ -676,7 +676,7 @@
     space.setattr(w_pycppclass, space.newtext(m1),
                   space.getattr(w_pycppclass, space.newtext(m2)))
 
-def pythonize(space, name, w_pycppclass):
+def pythonize(space, w_pycppclass, name):
     if name == "string":
         space.setattr(w_pycppclass, space.newtext("c_str"), _pythonizations["stdstring_c_str"])
         _method_alias(space, w_pycppclass, "_cppyy_as_builtin", "c_str")
diff --git a/pypy/module/_cppyy/helper.py b/pypy/module/_cppyy/helper.py
--- a/pypy/module/_cppyy/helper.py
+++ b/pypy/module/_cppyy/helper.py
@@ -59,26 +59,6 @@
         name = name[:_find_qualifier_index(name)]
     return name.strip(' ')
 
-def extract_namespace(name):
-    # find the namespace the named class lives in, take care of templates
-    tpl_open = 0
-    for pos in xrange(len(name)-1, 1, -1):
-        c = name[pos]
-
-        # count '<' and '>' to be able to skip template contents
-        if c == '>':
-            tpl_open += 1
-        elif c == '<':
-            tpl_open -= 1
-
-        # collect name up to "::"
-        elif tpl_open == 0 and c == ':' and name[pos-1] == ':':
-            # found the extend of the scope ... done
-            return name[0:pos-1]
-
-    # no namespace; assume outer scope
-    return ""
-
 
 #- operator mappings --------------------------------------------------------
 _operator_mappings = {}
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -14,6 +14,7 @@
 from pypy.module._cffi_backend import ctypefunc
 from pypy.module._cppyy import converter, executor, ffitypes, helper
 
+CLASS_FLAGS_IS_PINNED      = 0x0001
 
 INSTANCE_FLAGS_PYTHON_OWNS = 0x0001
 INSTANCE_FLAGS_IS_REF      = 0x0002
@@ -131,7 +132,7 @@
     cppclass = space.interp_w(W_CPPClassDecl, w_cppclass)
     # add back-end specific method pythonizations (doing this on the wrapped
     # class allows simple aliasing of methods)
-    capi.pythonize(space, cppclass.name, w_pycppclass)
+    capi.pythonize(space, w_pycppclass, cppclass.name)
     state = space.fromcache(State)
     state.cppclass_registry[rffi.cast(rffi.LONG, cppclass.handle)] = w_pycppclass
 
@@ -816,14 +817,15 @@
 
 
 class W_CPPScopeDecl(W_Root):
-    _attrs_ = ['space', 'handle', 'name', 'methods', 'datamembers']
+    _attrs_ = ['space', 'handle', 'flags', 'name', 'methods', 'datamembers']
     _immutable_fields_ = ['handle', 'name']
 
     def __init__(self, space, opaque_handle, final_scoped_name):
         self.space = space
-        self.name = final_scoped_name
         assert lltype.typeOf(opaque_handle) == capi.C_SCOPE
         self.handle = opaque_handle
+        self.flags = 0
+        self.name = final_scoped_name
         self.methods = {}
         # Do not call "self._build_methods()" here, so that a distinction can
         #  be made between testing for existence (i.e. existence in the cache
@@ -1316,7 +1318,7 @@
 
     # cast to actual if requested and possible
     w_pycppclass = None
-    if do_cast and rawobject:
+    if do_cast and rawobject and not (clsdecl.flags & CLASS_FLAGS_IS_PINNED):
         actual = capi.c_actual_class(space, clsdecl, rawobject)
         if actual != clsdecl.handle:
             try:
@@ -1390,3 +1392,13 @@
     if obj:
         obj.flags |= INSTANCE_FLAGS_IS_R_VALUE
     return w_obj
+
+
+# pythonization interface ---------------------------------------------------
+
+# do not auto-cast to given type
+ at unwrap_spec(w_pycppclass=W_Root)
+def _pin_type(space, w_pycppclass):
+    w_clsdecl = space.findattr(w_pycppclass, space.newtext("__cppdecl__"))
+    decl = space.interp_w(W_CPPClassDecl, w_clsdecl)
+    decl.flags |= CLASS_FLAGS_IS_PINNED
diff --git a/pypy/module/_cppyy/pythonify.py b/pypy/module/_cppyy/pythonify.py
--- a/pypy/module/_cppyy/pythonify.py
+++ b/pypy/module/_cppyy/pythonify.py
@@ -227,7 +227,7 @@
     # needs to run first, so that the generic pythonizations can use them
     import _cppyy
     _cppyy._register_class(pycls)
-    _pythonize(pycls)
+    _pythonize(pycls, pycls.__cppname__)
     return pycls
 
 def make_cpptemplatetype(scope, template_name):
@@ -291,6 +291,27 @@
     raise AttributeError("'%s' has no attribute '%s'" % (str(scope), name))
 
 
+# helper for pythonization API
+def extract_namespace(name):
+    # find the namespace the named class lives in, take care of templates
+    tpl_open = 0
+    for pos in xrange(len(name)-1, 1, -1):
+        c = name[pos]
+
+        # count '<' and '>' to be able to skip template contents
+        if c == '>':
+            tpl_open += 1
+        elif c == '<':
+            tpl_open -= 1
+
+        # collect name up to "::"
+        elif tpl_open == 0 and c == ':' and name[pos-1] == ':':
+            # found the extend of the scope ... done
+            return name[:pos-1], name[pos+1:]
+
+    # no namespace; assume outer scope
+    return '', name
+
 # pythonization by decoration (move to their own file?)
 def python_style_getitem(self, idx):
     # python-style indexing: check for size and allow indexing from the back
@@ -314,15 +335,7 @@
     else:
         return python_style_getitem(self, slice_or_idx)
 
-
-_pythonizations = {}
-def _pythonize(pyclass):
-
-    try:
-        _pythonizations[pyclass.__name__](pyclass)
-    except KeyError:
-        pass
-
+def _pythonize(pyclass, name):
     # general note: use 'in pyclass.__dict__' rather than 'hasattr' to prevent
     # adding pythonizations multiple times in derived classes
 
@@ -363,10 +376,10 @@
     # map begin()/end() protocol to iter protocol on STL(-like) classes, but
     # not on vector, which is pythonized in the capi (interp-level; there is
     # also the fallback on the indexed __getitem__, but that is slower)
-    if not 'vector' in pyclass.__name__[:11] and \
+    if not 'vector' in name[:11] and \
             ('begin' in pyclass.__dict__ and 'end' in pyclass.__dict__):
-        if _cppyy._scope_byname(pyclass.__cppname__+'::iterator') or \
-                _cppyy._scope_byname(pyclass.__cppname__+'::const_iterator'):
+        if _cppyy._scope_byname(name+'::iterator') or \
+                _cppyy._scope_byname(name+'::const_iterator'):
             def __iter__(self):
                 i = self.begin()
                 while i != self.end():
@@ -386,7 +399,7 @@
             pyclass.__getitem__ = python_style_getitem
 
     # string comparisons
-    if pyclass.__name__ == _cppyy._std_string_name():
+    if name == _cppyy._std_string_name():
         def eq(self, other):
             if type(other) == pyclass:
                 return self.c_str() == other.c_str()
@@ -396,7 +409,7 @@
         pyclass.__str__ = pyclass.c_str
 
     # std::pair unpacking through iteration
-    if 'std::pair' == pyclass.__name__[:9] or 'pair' == pyclass.__name__[:4]:
+    if 'std::pair' == name[:9]:
         def getitem(self, idx):
             if idx == 0: return self.first
             if idx == 1: return self.second
@@ -406,6 +419,16 @@
         pyclass.__getitem__ = getitem
         pyclass.__len__     = return2
 
+    # user provided, custom pythonizations
+    try:
+        ns_name, cl_name = extract_namespace(name)
+        pythonizors = _pythonizations[ns_name]
+        name = cl_name
+    except KeyError:
+        pythonizors = _pythonizations['']   # global scope
+
+    for p in pythonizors:
+        p(pyclass, name)
 
 def _post_import_startup():
     # _cppyy should not be loaded at the module level, as that will trigger a
@@ -450,11 +473,26 @@
 
 
 # user-defined pythonizations interface
-_pythonizations = {}
-def add_pythonization(class_name, callback):
-    """Takes a class name and a callback. The callback should take a single
-    argument, the class proxy, and is called the first time the named class
-    is bound."""
-    if not callable(callback):
-        raise TypeError("given '%s' object is not callable" % str(callback))
-    _pythonizations[class_name] = callback
+_pythonizations = {'' : list()}
+def add_pythonization(pythonizor, scope = ''):
+    """<pythonizor> should be a callable taking two arguments: a class proxy,
+    and its C++ name. It is called on each time a named class from <scope>
+    (the global one by default, but a relevant C++ namespace is recommended)
+    is bound.
+    """
+    if not callable(pythonizor):
+        raise TypeError("given '%s' object is not callable" % str(pythonizor))
+    try:
+        _pythonizations[scope].append(pythonizor)
+    except KeyError:
+        _pythonizations[scope] = list()
+        _pythonizations[scope].append(pythonizor)
+
+def remove_pythonization(pythonizor, scope = ''):
+    """Remove previously registered <pythonizor> from <scope>.
+    """
+    try:
+        _pythonizations[scope].remove(pythonizor)
+        return True
+    except (KeyError, ValueError):
+        return False
diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile
--- a/pypy/module/_cppyy/test/Makefile
+++ b/pypy/module/_cppyy/test/Makefile
@@ -7,6 +7,7 @@
         fragileDict.so \
         operatorsDict.so \
         overloadsDict.so \
+        pythonizablesDict.so \
         stltypesDict.so \
         templatesDict.so
 
diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py
--- a/pypy/module/_cppyy/test/conftest.py
+++ b/pypy/module/_cppyy/test/conftest.py
@@ -10,7 +10,7 @@
             import os
             tst = os.path.basename(item.location[0])
             if not tst in ('test_helper.py', 'test_cppyy.py', 'test_pythonify.py',
-                           'test_datatypes.py'):
+                           'test_datatypes.py', 'test_pythonization.py'):
                 py.test.skip("genreflex is not installed")
             import re
             if tst == 'test_pythonify.py' and \
@@ -19,6 +19,9 @@
             elif tst == 'test_datatypes.py' and \
                 not re.search("AppTestDATATYPES.test0[1-7]", item.location[2]):
                 py.test.skip("genreflex is not installed")
+            elif tst == 'test_pythonization.py' and \
+                not re.search("AppTestPYTHONIZATION.test0[0]", item.location[2]):
+                py.test.skip("genreflex is not installed")
 
 def pytest_ignore_collect(path, config):
     if py.path.local.sysfind('genreflex') is None and config.option.runappdirect:
diff --git a/pypy/module/_cppyy/test/test_helper.py b/pypy/module/_cppyy/test/test_helper.py
--- a/pypy/module/_cppyy/test/test_helper.py
+++ b/pypy/module/_cppyy/test/test_helper.py
@@ -53,11 +53,12 @@
 
 
 def test_namespace_extraction():
-    assert helper.extract_namespace("vector")                        == ""
-    assert helper.extract_namespace("std::vector")                   == "std"
-    assert helper.extract_namespace("std::vector<double>")           == "std"
-    assert helper.extract_namespace("std::vector<std::vector>")      == "std"
-    assert helper.extract_namespace("vector<double>")                == ""
-    assert helper.extract_namespace("vector<std::vector>")           == ""
-    assert helper.extract_namespace("aap::noot::mies::zus")          == "aap::noot::mies"
+    from pypy.module._cppyy import pythonify
 
+    assert pythonify.extract_namespace("vector")[0]                     == ""
+    assert pythonify.extract_namespace("std::vector")[0]                == "std"
+    assert pythonify.extract_namespace("std::vector<double>")[0]        == "std"
+    assert pythonify.extract_namespace("std::vector<std::vector>")[0]   == "std"
+    assert pythonify.extract_namespace("vector<double>")[0]             == ""
+    assert pythonify.extract_namespace("vector<std::vector>")[0]        == ""
+    assert pythonify.extract_namespace("aap::noot::mies::zus")[0]       == "aap::noot::mies"
diff --git a/pypy/module/_cppyy/test/test_pythonify.py b/pypy/module/_cppyy/test/test_pythonify.py
--- a/pypy/module/_cppyy/test/test_pythonify.py
+++ b/pypy/module/_cppyy/test/test_pythonify.py
@@ -378,13 +378,13 @@
 
         import _cppyy
 
-        def example01a_pythonize(pyclass):
-            assert pyclass.__name__ == 'example01a'
-            def getitem(self, idx):
-                return self.addDataToInt(idx)
-            pyclass.__getitem__ = getitem
+        def example01a_pythonize(pyclass, name):
+            if name == 'example01a':
+                def getitem(self, idx):
+                    return self.addDataToInt(idx)
+                pyclass.__getitem__ = getitem
 
-        _cppyy.add_pythonization('example01a', example01a_pythonize)
+        _cppyy.add_pythonization(example01a_pythonize)
 
         e = _cppyy.gbl.example01a(1)
 

From pypy.commits at gmail.com  Wed May 16 07:59:07 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 May 2018 04:59:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge
Message-ID: <5afc1d0b.1c69fb81.d59ca.c8fa@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94598:e149edfccd49
Date: 2018-05-16 13:58 +0200
http://bitbucket.org/pypy/pypy/changeset/e149edfccd49/

Log:	merge

diff --git a/pypy/objspace/std/test/test_iterobject.py b/pypy/objspace/std/test/test_iterobject.py
--- a/pypy/objspace/std/test/test_iterobject.py
+++ b/pypy/objspace/std/test/test_iterobject.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.objspace.std.iterobject import W_SeqIterObject
 from pypy.interpreter.error import OperationError
 
@@ -11,8 +12,8 @@
         self.body0(w_iter)
 
     def body0(self, w_iter):
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
 
     def test_iter(self):
         w = self.space.wrap
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -120,8 +120,8 @@
         assert self.space.eq_w(self.space.next(w_iter), w(5))
         assert self.space.eq_w(self.space.next(w_iter), w(3))
         assert self.space.eq_w(self.space.next(w_iter), w(99))
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        py.test.raises(OperationError, self.space.next, w_iter)
+        py.test.raises(OperationError, self.space.next, w_iter)
 
     def test_contains(self):
         w = self.space.wrap
diff --git a/pypy/objspace/std/test/test_setstrategies.py b/pypy/objspace/std/test/test_setstrategies.py
--- a/pypy/objspace/std/test/test_setstrategies.py
+++ b/pypy/objspace/std/test/test_setstrategies.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.objspace.std.setobject import W_SetObject
 from pypy.objspace.std.setobject import (
     BytesIteratorImplementation, BytesSetStrategy, EmptySetStrategy,
@@ -58,7 +59,7 @@
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         s2 = W_SetObject(self.space, self.wrapped([4,5, "six", "seven"]))
         s3 = s1.intersect(s2)
-        skip("for now intersection with ObjectStrategy always results in another ObjectStrategy")
+        pytest.skip("for now intersection with ObjectStrategy always results in another ObjectStrategy")
         assert s3.strategy is self.space.fromcache(IntegerSetStrategy)
 
     def test_clear(self):
@@ -93,7 +94,7 @@
 
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         s1.descr_discard(self.space, self.space.wrap("five"))
-        skip("currently not supported")
+        pytest.skip("currently not supported")
         assert s1.strategy is self.space.fromcache(IntegerSetStrategy)
 
         set_discard__Set_ANY(self.space, s1, self.space.wrap(FakeInt(5)))
@@ -112,7 +113,7 @@
 
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         assert not s1.has_key(self.space.wrap("five"))
-        skip("currently not supported")
+        pytest.skip("currently not supported")
         assert s1.strategy is self.space.fromcache(IntegerSetStrategy)
 
         assert s1.has_key(self.space.wrap(FakeInt(2)))
diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py
--- a/pypy/objspace/std/test/test_tupleobject.py
+++ b/pypy/objspace/std/test/test_tupleobject.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std.tupleobject import W_TupleObject
 
@@ -42,8 +43,8 @@
         assert self.space.eq_w(self.space.next(w_iter), w(5))
         assert self.space.eq_w(self.space.next(w_iter), w(3))
         assert self.space.eq_w(self.space.next(w_iter), w(99))
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
 
     def test_contains(self):
         w = self.space.wrap

From pypy.commits at gmail.com  Wed May 16 07:59:04 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 May 2018 04:59:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix more raises/skip imports
Message-ID: <5afc1d08.1c69fb81.72cba.0e4b@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94597:c697fd791c27
Date: 2018-05-15 21:58 +0200
http://bitbucket.org/pypy/pypy/changeset/c697fd791c27/

Log:	fix more raises/skip imports

diff --git a/pypy/objspace/std/test/test_iterobject.py b/pypy/objspace/std/test/test_iterobject.py
--- a/pypy/objspace/std/test/test_iterobject.py
+++ b/pypy/objspace/std/test/test_iterobject.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.objspace.std.iterobject import W_SeqIterObject
 from pypy.interpreter.error import OperationError
 
@@ -11,8 +12,8 @@
         self.body0(w_iter)
 
     def body0(self, w_iter):
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
 
     def test_iter(self):
         w = self.space.wrap
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -120,8 +120,8 @@
         assert self.space.eq_w(self.space.next(w_iter), w(5))
         assert self.space.eq_w(self.space.next(w_iter), w(3))
         assert self.space.eq_w(self.space.next(w_iter), w(99))
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        py.test.raises(OperationError, self.space.next, w_iter)
+        py.test.raises(OperationError, self.space.next, w_iter)
 
     def test_contains(self):
         w = self.space.wrap
diff --git a/pypy/objspace/std/test/test_setstrategies.py b/pypy/objspace/std/test/test_setstrategies.py
--- a/pypy/objspace/std/test/test_setstrategies.py
+++ b/pypy/objspace/std/test/test_setstrategies.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.objspace.std.setobject import W_SetObject
 from pypy.objspace.std.setobject import (
     BytesIteratorImplementation, BytesSetStrategy, EmptySetStrategy,
@@ -58,7 +59,7 @@
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         s2 = W_SetObject(self.space, self.wrapped([4,5, "six", "seven"]))
         s3 = s1.intersect(s2)
-        skip("for now intersection with ObjectStrategy always results in another ObjectStrategy")
+        pytest.skip("for now intersection with ObjectStrategy always results in another ObjectStrategy")
         assert s3.strategy is self.space.fromcache(IntegerSetStrategy)
 
     def test_clear(self):
@@ -93,7 +94,7 @@
 
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         s1.descr_discard(self.space, self.space.wrap("five"))
-        skip("currently not supported")
+        pytest.skip("currently not supported")
         assert s1.strategy is self.space.fromcache(IntegerSetStrategy)
 
         set_discard__Set_ANY(self.space, s1, self.space.wrap(FakeInt(5)))
@@ -112,7 +113,7 @@
 
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         assert not s1.has_key(self.space.wrap("five"))
-        skip("currently not supported")
+        pytest.skip("currently not supported")
         assert s1.strategy is self.space.fromcache(IntegerSetStrategy)
 
         assert s1.has_key(self.space.wrap(FakeInt(2)))
diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py
--- a/pypy/objspace/std/test/test_tupleobject.py
+++ b/pypy/objspace/std/test/test_tupleobject.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std.tupleobject import W_TupleObject
 
@@ -42,8 +43,8 @@
         assert self.space.eq_w(self.space.next(w_iter), w(5))
         assert self.space.eq_w(self.space.next(w_iter), w(3))
         assert self.space.eq_w(self.space.next(w_iter), w(99))
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
 
     def test_contains(self):
         w = self.space.wrap

From pypy.commits at gmail.com  Wed May 16 08:39:21 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 May 2018 05:39:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge py3.5
Message-ID: <5afc2679.1c69fb81.a1ad9.0253@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94600:455b5c86ee7b
Date: 2018-05-16 14:38 +0200
http://bitbucket.org/pypy/pypy/changeset/455b5c86ee7b/

Log:	merge py3.5

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -423,6 +423,7 @@
     RegrTest('test_sys_setprofile.py', core=True),
     RegrTest('test_sys_settrace.py', core=True),
     RegrTest('test_sysconfig.py'),
+    RegrTest('test_sysconfig_pypy.py'),
     RegrTest('test_syslog.py'),
     RegrTest('test_tarfile.py'),
     RegrTest('test_tcl.py'),
diff --git a/pypy/interpreter/astcompiler/test/test_validate.py b/pypy/interpreter/astcompiler/test/test_validate.py
--- a/pypy/interpreter/astcompiler/test/test_validate.py
+++ b/pypy/interpreter/astcompiler/test/test_validate.py
@@ -1,4 +1,5 @@
 import os
+from pytest import raises
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.astcompiler import ast
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -375,7 +375,7 @@
         child.expect('>>>')
 
     def test_atexit(self):
-        skip("Python3 atexit is a builtin module")
+        py.test.skip("Python3 atexit is a builtin module")
         child = self.spawn([])
         child.expect('>>> ')
         child.sendline('def f(): print("foobye")')
@@ -525,9 +525,9 @@
 
     def test_options_i_m(self, monkeypatch):
         if sys.platform == "win32":
-            skip("close_fds is not supported on Windows platforms")
+            py.test.skip("close_fds is not supported on Windows platforms")
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'mymodule.py')
         p = os.path.abspath(p)
         monkeypatch.chdir(os.path.dirname(app_main))
@@ -557,7 +557,7 @@
 
     def test_options_u_i(self):
         if sys.platform == "win32":
-            skip("close_fds is not supported on Windows platforms")
+            py.test.skip("close_fds is not supported on Windows platforms")
         import subprocess, select, os
         pipe = subprocess.Popen([get_python3(), app_main, "-u", "-i"],
                                 stdout=subprocess.PIPE,
@@ -614,7 +614,7 @@
             del os.environ['PYTHONINSPECT_']
 
     def test_stdout_flushes_before_stdin_blocks(self):
-        skip("Python3 does not implement this behavior")
+        py.test.skip("Python3 does not implement this behavior")
         # This doesn't really test app_main.py, but a behavior that
         # can only be checked on top of py.py with pexpect.
         path = getscript("""
@@ -632,7 +632,7 @@
 
     def test_no_space_before_argument(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         child = self.spawn(['-cprint("hel" + "lo")'])
         child.expect('hello')
 
@@ -753,7 +753,7 @@
 
     def test_option_m(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'mymodule.py')
         p = os.path.abspath(p)
         monkeypatch.chdir(os.path.dirname(app_main))
@@ -767,7 +767,7 @@
 
     def test_option_m_package(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)),
                          'mypackage', '__main__.py')
         p = os.path.abspath(p)
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -168,6 +168,222 @@
     return decode_utf8(space, string, allow_surrogates=True)
 
 # ____________________________________________________________
+# utf-16
+
+def str_decode_utf_16(s, size, errors, final=True,
+                           errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "native",
+                                                         'utf-16-' + BYTEORDER2)
+    return result, length
+
+def str_decode_utf_16_be(s, size, errors, final=True,
+                              errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "big",
+                                                         'utf-16-be')
+    return result, length
+
+def str_decode_utf_16_le(s, size, errors, final=True,
+                              errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "little",
+                                                         'utf-16-le')
+    return result, length
+
+def str_decode_utf_16_helper(s, size, errors, final=True,
+                             errorhandler=None,
+                             byteorder="native",
+                             public_encoding_name='utf16'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+    bo = 0
+
+    if BYTEORDER == 'little':
+        ihi = 1
+        ilo = 0
+    else:
+        ihi = 0
+        ilo = 1
+
+    #  Check for BOM marks (U+FEFF) in the input and adjust current
+    #  byte order setting accordingly. In native mode, the leading BOM
+    #  mark is skipped, in all other modes, it is copied to the output
+    #  stream as-is (giving a ZWNBSP character).
+    pos = 0
+    if byteorder == 'native':
+        if size >= 2:
+            bom = (ord(s[ihi]) << 8) | ord(s[ilo])
+            if BYTEORDER == 'little':
+                if bom == 0xFEFF:
+                    pos += 2
+                    bo = -1
+                elif bom == 0xFFFE:
+                    pos += 2
+                    bo = 1
+            else:
+                if bom == 0xFEFF:
+                    pos += 2
+                    bo = 1
+                elif bom == 0xFFFE:
+                    pos += 2
+                    bo = -1
+    elif byteorder == 'little':
+        bo = -1
+    else:
+        bo = 1
+    if size == 0:
+        return u'', 0, bo
+    if bo == -1:
+        # force little endian
+        ihi = 1
+        ilo = 0
+
+    elif bo == 1:
+        # force big endian
+        ihi = 0
+        ilo = 1
+
+    result = UnicodeBuilder(size // 2)
+
+    #XXX I think the errors are not correctly handled here
+    while pos < size:
+        # remaining bytes at the end? (size should be even)
+        if len(s) - pos < 2:
+            if not final:
+                break
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "truncated data",
+                                  s, pos, len(s))
+            result.append(r)
+            if len(s) - pos < 2:
+                break
+        ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo])
+        pos += 2
+        if ch < 0xD800 or ch > 0xDFFF:
+            result.append(unichr(ch))
+            continue
+        # UTF-16 code pair:
+        if len(s) - pos < 2:
+            pos -= 2
+            if not final:
+                break
+            errmsg = "unexpected end of data"
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  errmsg, s, pos, len(s))
+            result.append(r)
+            if len(s) - pos < 2:
+                break
+        elif 0xD800 <= ch <= 0xDBFF:
+            ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo])
+            pos += 2
+            if 0xDC00 <= ch2 <= 0xDFFF:
+                if MAXUNICODE < 65536:
+                    result.append(unichr(ch))
+                    result.append(unichr(ch2))
+                else:
+                    result.append(UNICHR((((ch & 0x3FF)<<10) |
+                                           (ch2 & 0x3FF)) + 0x10000))
+                continue
+            else:
+                r, pos = errorhandler(errors, public_encoding_name,
+                                      "illegal UTF-16 surrogate",
+                                      s, pos - 4, pos - 2)
+                result.append(r)
+        else:
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "illegal encoding",
+                                  s, pos - 2, pos)
+            result.append(r)
+    return result.build(), pos, bo
+
+def _STORECHAR(result, CH, byteorder):
+    hi = chr(((CH) >> 8) & 0xff)
+    lo = chr((CH) & 0xff)
+    if byteorder == 'little':
+        result.append(lo)
+        result.append(hi)
+    else:
+        result.append(hi)
+        result.append(lo)
+
+def unicode_encode_utf_16_helper(s, size, errors,
+                                 errorhandler=None,
+                                 allow_surrogates=True,
+                                 byteorder='little',
+                                 public_encoding_name='utf16'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_encode
+    if size == 0:
+        if byteorder == 'native':
+            result = StringBuilder(2)
+            _STORECHAR(result, 0xFEFF, BYTEORDER)
+            return result.build()
+        return ""
+
+    result = StringBuilder(size * 2 + 2)
+    if byteorder == 'native':
+        _STORECHAR(result, 0xFEFF, BYTEORDER)
+        byteorder = BYTEORDER
+
+    pos = 0
+    while pos < size:
+        ch = ord(s[pos])
+        pos += 1
+
+        if ch < 0xD800:
+            _STORECHAR(result, ch, byteorder)
+        elif ch >= 0x10000:
+            _STORECHAR(result, 0xD800 | ((ch-0x10000) >> 10), byteorder)
+            _STORECHAR(result, 0xDC00 | ((ch-0x10000) & 0x3FF), byteorder)
+        elif ch >= 0xE000 or allow_surrogates:
+            _STORECHAR(result, ch, byteorder)
+        else:
+            ru, rs, pos = errorhandler(errors, public_encoding_name,
+                                       'surrogates not allowed',
+                                       s, pos-1, pos)
+            if rs is not None:
+                # py3k only
+                if len(rs) % 2 != 0:
+                    errorhandler('strict', public_encoding_name,
+                                 'surrogates not allowed',
+                                 s, pos-1, pos)
+                result.append(rs)
+                continue
+            for ch in ru:
+                if ord(ch) < 0xD800:
+                    _STORECHAR(result, ord(ch), byteorder)
+                else:
+                    errorhandler('strict', public_encoding_name,
+                                 'surrogates not allowed',
+                                 s, pos-1, pos)
+            continue
+
+    return result.build()
+
+def unicode_encode_utf_16(s, size, errors,
+                          errorhandler=None,
+                          allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "native",
+                                        'utf-16-' + BYTEORDER2)
+
+def unicode_encode_utf_16_be(s, size, errors,
+                             errorhandler=None,
+                             allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "big",
+                                        'utf-16-be')
+
+def unicode_encode_utf_16_le(s, size, errors,
+                             errorhandler=None,
+                             allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "little",
+                                        'utf-16-le')
+
+
+# ____________________________________________________________
 # utf-32
 
 def str_decode_utf_32(s, size, errors, final=True,
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -621,8 +621,6 @@
     try:
         func = getattr(unicodehelper, impl_name)
     except AttributeError:
-        if hasattr(runicode, 'py3k_' + impl_name):
-            impl_name = 'py3k_' + impl_name
         func = getattr(runicode, impl_name)
     return func
 
diff --git a/pypy/objspace/std/test/test_iterobject.py b/pypy/objspace/std/test/test_iterobject.py
--- a/pypy/objspace/std/test/test_iterobject.py
+++ b/pypy/objspace/std/test/test_iterobject.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.objspace.std.iterobject import W_SeqIterObject
 from pypy.interpreter.error import OperationError
 
@@ -11,8 +12,8 @@
         self.body0(w_iter)
 
     def body0(self, w_iter):
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
 
     def test_iter(self):
         w = self.space.wrap
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -120,8 +120,8 @@
         assert self.space.eq_w(self.space.next(w_iter), w(5))
         assert self.space.eq_w(self.space.next(w_iter), w(3))
         assert self.space.eq_w(self.space.next(w_iter), w(99))
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        py.test.raises(OperationError, self.space.next, w_iter)
+        py.test.raises(OperationError, self.space.next, w_iter)
 
     def test_contains(self):
         w = self.space.wrap
diff --git a/pypy/objspace/std/test/test_setstrategies.py b/pypy/objspace/std/test/test_setstrategies.py
--- a/pypy/objspace/std/test/test_setstrategies.py
+++ b/pypy/objspace/std/test/test_setstrategies.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.objspace.std.setobject import W_SetObject
 from pypy.objspace.std.setobject import (
     BytesIteratorImplementation, BytesSetStrategy, EmptySetStrategy,
@@ -58,7 +59,7 @@
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         s2 = W_SetObject(self.space, self.wrapped([4,5, "six", "seven"]))
         s3 = s1.intersect(s2)
-        skip("for now intersection with ObjectStrategy always results in another ObjectStrategy")
+        pytest.skip("for now intersection with ObjectStrategy always results in another ObjectStrategy")
         assert s3.strategy is self.space.fromcache(IntegerSetStrategy)
 
     def test_clear(self):
@@ -93,7 +94,7 @@
 
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         s1.descr_discard(self.space, self.space.wrap("five"))
-        skip("currently not supported")
+        pytest.skip("currently not supported")
         assert s1.strategy is self.space.fromcache(IntegerSetStrategy)
 
         set_discard__Set_ANY(self.space, s1, self.space.wrap(FakeInt(5)))
@@ -112,7 +113,7 @@
 
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         assert not s1.has_key(self.space.wrap("five"))
-        skip("currently not supported")
+        pytest.skip("currently not supported")
         assert s1.strategy is self.space.fromcache(IntegerSetStrategy)
 
         assert s1.has_key(self.space.wrap(FakeInt(2)))
diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py
--- a/pypy/objspace/std/test/test_tupleobject.py
+++ b/pypy/objspace/std/test/test_tupleobject.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std.tupleobject import W_TupleObject
 
@@ -42,8 +43,8 @@
         assert self.space.eq_w(self.space.next(w_iter), w(5))
         assert self.space.eq_w(self.space.next(w_iter), w(3))
         assert self.space.eq_w(self.space.next(w_iter), w(99))
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
 
     def test_contains(self):
         w = self.space.wrap

From pypy.commits at gmail.com  Wed May 16 08:39:19 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 16 May 2018 05:39:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: add missing file to unbreak lib-python
 tests
Message-ID: <5afc2677.1c69fb81.179ba.34ed@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94599:a6beb8e7ff9c
Date: 2018-05-16 14:38 +0200
http://bitbucket.org/pypy/pypy/changeset/a6beb8e7ff9c/

Log:	add missing file to unbreak lib-python tests

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -422,6 +422,7 @@
     RegrTest('test_sys_setprofile.py', core=True),
     RegrTest('test_sys_settrace.py', core=True),
     RegrTest('test_sysconfig.py'),
+    RegrTest('test_sysconfig_pypy.py'),
     RegrTest('test_syslog.py'),
     RegrTest('test_tarfile.py'),
     RegrTest('test_tcl.py'),

From pypy.commits at gmail.com  Wed May 16 12:25:42 2018
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 16 May 2018 09:25:42 -0700 (PDT)
Subject: [pypy-commit] pypy gc-more-logging: close merged branch
Message-ID: <5afc5b86.1c69fb81.fdbeb.f100@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-more-logging
Changeset: r94601:42a01bd21795
Date: 2018-05-16 18:24 +0200
http://bitbucket.org/pypy/pypy/changeset/42a01bd21795/

Log:	close merged branch


From pypy.commits at gmail.com  Wed May 16 12:25:44 2018
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 16 May 2018 09:25:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge the gc-more-logging branch,
 which logs some extra gc-minor and
Message-ID: <5afc5b88.1c69fb81.e40aa.d92d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r94602:49ede3b5afe6
Date: 2018-05-16 18:25 +0200
http://bitbucket.org/pypy/pypy/changeset/49ede3b5afe6/

Log:	merge the gc-more-logging branch, which logs some extra gc-minor and
	gc-collect-step info in the PYPYLOG

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1836,6 +1836,7 @@
         debug_print("minor collect, total memory used:", total_memory_used)
         debug_print("number of pinned objects:",
                     self.pinned_objects_in_nursery)
+        debug_print("total size of surviving objects:", self.nursery_surviving_size)
         if self.DEBUG >= 2:
             self.debug_check_consistency()     # expensive!
         #
@@ -2401,7 +2402,9 @@
                 # a total object size of at least '3 * nursery_size' bytes
                 # is processed.
                 limit = 3 * self.nursery_size // self.small_request_threshold
-                self.free_unvisited_rawmalloc_objects_step(limit)
+                nobjects = self.free_unvisited_rawmalloc_objects_step(limit)
+                debug_print("freeing raw objects:", limit-nobjects,
+                            "freed, limit was", limit)
                 done = False    # the 2nd half below must still be done
             else:
                 # Ask the ArenaCollection to visit a fraction of the objects.
@@ -2411,6 +2414,8 @@
                 limit = 3 * self.nursery_size // self.ac.page_size
                 done = self.ac.mass_free_incremental(self._free_if_unvisited,
                                                      limit)
+                status = done and "No more pages left." or "More to do."
+                debug_print("freeing GC objects, up to", limit, "pages.", status)
             # XXX tweak the limits above
             #
             if done:

From pypy.commits at gmail.com  Wed May 16 23:53:15 2018
From: pypy.commits at gmail.com (alex_gaynor)
Date: Wed, 16 May 2018 20:53:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove an unneeded call into OpenSSL
Message-ID: <5afcfcab.1c69fb81.ad457.6231@mx.google.com>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: py3.5
Changeset: r94603:f3d6e0d85a29
Date: 2018-05-16 20:39 -0700
http://bitbucket.org/pypy/pypy/changeset/f3d6e0d85a29/

Log:	Remove an unneeded call into OpenSSL

	This call doesn't do anything, on every version of OpenSSL going
	back to forever the call to ERR_clear_error() calls ERR_get_state(),
	so it's always been a no-op.

	Same change for CPython: https://github.com/python/cpython/pull/6887

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -220,7 +220,6 @@
         if server_hostname:
             self.server_hostname = server_hostname.decode('idna', 'strict')
 
-        lib.ERR_get_state()
         lib.ERR_clear_error()
         self.ssl = ssl = ffi.gc(lib.SSL_new(ctx), lib.SSL_free)
 

From pypy.commits at gmail.com  Wed May 16 23:53:17 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 May 2018 20:53:17 -0700 (PDT)
Subject: [pypy-commit] pypy
 alex_gaynor/remove-an-unneeded-call-into-openssl-th-1526429141011: close
 and document branch to be merged
Message-ID: <5afcfcad.1c69fb81.c3d22.e74e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: alex_gaynor/remove-an-unneeded-call-into-openssl-th-1526429141011
Changeset: r94604:710a263815fa
Date: 2018-05-16 20:42 -0700
http://bitbucket.org/pypy/pypy/changeset/710a263815fa/

Log:	close and document branch to be merged

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -12,3 +12,7 @@
 .. branch: py3.5-user-site-impl
 
 Use implementation-specific site directories in sysconfig like in Python2
+
+.. branch: alex_gaynor/remove-an-unneeded-call-into-openssl-th-1526429141011
+
+Remove an unneeded call into OpenSSL, from cpython https://github.com/python/cpython/pull/6887

From pypy.commits at gmail.com  Wed May 16 23:53:19 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 May 2018 20:53:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge ssl fix into py3.6
Message-ID: <5afcfcaf.1c69fb81.7ef92.3aec@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r94605:ceff6a6cf97b
Date: 2018-05-16 20:45 -0700
http://bitbucket.org/pypy/pypy/changeset/ceff6a6cf97b/

Log:	merge ssl fix into py3.6

diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -224,7 +224,6 @@
         if server_hostname:
             self.server_hostname = server_hostname.decode('idna', 'strict')
 
-        lib.ERR_get_state()
         lib.ERR_clear_error()
         self.ssl = ssl = ffi.gc(lib.SSL_new(ctx), lib.SSL_free)
 
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -12,3 +12,7 @@
 .. branch: py3.5-user-site-impl
 
 Use implementation-specific site directories in sysconfig like in Python2
+
+.. branch: alex_gaynor/remove-an-unneeded-call-into-openssl-th-1526429141011
+
+Remove an unneeded call into OpenSSL, from cpython https://github.com/python/cpython/pull/6887

From pypy.commits at gmail.com  Wed May 16 23:53:21 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 May 2018 20:53:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge py3.5 into py3.6
Message-ID: <5afcfcb1.1c69fb81.7ef92.3af1@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.6
Changeset: r94606:2a70f2b84450
Date: 2018-05-16 20:45 -0700
http://bitbucket.org/pypy/pypy/changeset/2a70f2b84450/

Log:	merge py3.5 into py3.6


From pypy.commits at gmail.com  Wed May 16 23:57:47 2018
From: pypy.commits at gmail.com (mattip)
Date: Wed, 16 May 2018 20:57:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <5afcfdbb.1c69fb81.7375e.9c5e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94607:24cdd1cfea2d
Date: 2018-05-16 20:55 -0700
http://bitbucket.org/pypy/pypy/changeset/24cdd1cfea2d/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -18,3 +18,7 @@
 .. branch: crypt_h
 
 Include crypt.h for crypt() on Linux
+
+.. branch: gc-more-logging
+
+Log additional gc-minor and gc-collect-step info in the PYPYLOG

From pypy.commits at gmail.com  Thu May 17 08:03:22 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 17 May 2018 05:03:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: obscure,
 but this fixes translation
Message-ID: <5afd6f8a.1c69fb81.4225b.9f9f@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94609:69b270a1b887
Date: 2018-05-17 14:02 +0200
http://bitbucket.org/pypy/pypy/changeset/69b270a1b887/

Log:	obscure, but this fixes translation

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -175,7 +175,7 @@
                 if opcode < HAVE_ARGUMENT:
                     raise BytecodeCorruption
                 next_instr += 2
-                oparg = (oparg << 8) | arg
+                oparg = (oparg * 256) | arg
 
             if opcode == opcodedesc.RETURN_VALUE.index:
                 w_returnvalue = self.popvalue()

From pypy.commits at gmail.com  Thu May 17 08:03:19 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 17 May 2018 05:03:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: merge py3.6
Message-ID: <5afd6f87.1c69fb81.bb8e6.2e82@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94608:0ecfc20943d3
Date: 2018-05-14 23:30 +0200
http://bitbucket.org/pypy/pypy/changeset/0ecfc20943d3/

Log:	merge py3.6

diff too long, truncating to 2000 out of 5669 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -51,3 +51,5 @@
 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0
 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0
 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1
+ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0
+fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -247,6 +247,7 @@
   Lukas Vacek
   Omer Katz
   Jacek Generowicz
+  Tomasz Dziopa
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -307,6 +308,7 @@
   Yury V. Zaytsev
   florinpapa
   Anders Sigfridsson
+  Matt Jackson
   Nikolay Zinov
   rafalgalczynski at gmail.com
   Joshua Gilbert
diff --git a/dotviewer/font/NOTICE b/dotviewer/font/COPYING.txt
rename from dotviewer/font/NOTICE
rename to dotviewer/font/COPYING.txt
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -7,6 +7,9 @@
 import time as _time
 import math as _math
 
+# for cpyext, use these as base classes
+from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop
+
 def _cmp(x, y):
     return 0 if x == y else 1 if x > y else -1
 
@@ -332,8 +335,7 @@
 
     return q
 
-
-class timedelta:
+class timedelta(deltainterop):
     """Represent the difference between two datetime objects.
 
     Supported operators:
@@ -446,7 +448,7 @@
         if abs(d) > 999999999:
             raise OverflowError("timedelta # of days is too large: %d" % d)
 
-        self = object.__new__(cls)
+        self = deltainterop.__new__(cls)
         self._days = d
         self._seconds = s
         self._microseconds = us
@@ -655,7 +657,7 @@
                           microseconds=999999)
 timedelta.resolution = timedelta(microseconds=1)
 
-class date:
+class date(dateinterop):
     """Concrete date type.
 
     Constructors:
@@ -695,12 +697,12 @@
         if month is None and isinstance(year, bytes) and len(year) == 4 and \
                 1 <= year[2] <= 12:
             # Pickle support
-            self = object.__new__(cls)
+            self = dateinterop.__new__(cls)
             self.__setstate(year)
             self._hashcode = -1
             return self
         year, month, day = _check_date_fields(year, month, day)
-        self = object.__new__(cls)
+        self = dateinterop.__new__(cls)
         self._year = year
         self._month = month
         self._day = day
@@ -1026,7 +1028,7 @@
 
 _tzinfo_class = tzinfo
 
-class time:
+class time(timeinterop):
     """Time with time zone.
 
     Constructors:
@@ -1063,14 +1065,14 @@
         """
         if isinstance(hour, bytes) and len(hour) == 6 and hour[0]&0x7F < 24:
             # Pickle support
-            self = object.__new__(cls)
+            self = timeinterop.__new__(cls)
             self.__setstate(hour, minute or None)
             self._hashcode = -1
             return self
         hour, minute, second, microsecond, fold = _check_time_fields(
             hour, minute, second, microsecond, fold)
         _check_tzinfo_arg(tzinfo)
-        self = object.__new__(cls)
+        self = timeinterop.__new__(cls)
         self._hour = hour
         self._minute = minute
         self._second = second
@@ -1376,7 +1378,7 @@
                 microsecond=0, tzinfo=None, *, fold=0):
         if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2]&0x7F <= 12:
             # Pickle support
-            self = object.__new__(cls)
+            self = dateinterop.__new__(cls)
             self.__setstate(year, month)
             self._hashcode = -1
             return self
@@ -1384,7 +1386,7 @@
         hour, minute, second, microsecond, fold = _check_time_fields(
             hour, minute, second, microsecond, fold)
         _check_tzinfo_arg(tzinfo)
-        self = object.__new__(cls)
+        self = dateinterop.__new__(cls)
         self._year = year
         self._month = month
         self._day = day
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -63,39 +63,9 @@
 
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
-    so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
-
-    g = {}
-    g['CC'] = "cc -pthread"
-    g['CXX'] = "c++ -pthread"
-    g['OPT'] = "-DNDEBUG -O2"
-    g['CFLAGS'] = "-DNDEBUG -O2"
-    g['CCSHARED'] = "-fPIC"
-    g['LDSHARED'] = "cc -pthread -shared"
-    g['EXT_SUFFIX'] = so_ext
-    g['SHLIB_SUFFIX'] = ".so"
-    g['SO'] = so_ext  # deprecated in Python 3, for backward compatibility
-    g['AR'] = "ar"
-    g['ARFLAGS'] = "rc"
-    g['EXE'] = ""
-    g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
-    g['VERSION'] = get_python_version()
-
-    if sys.platform[:6] == "darwin":
-        import platform
-        if platform.machine() == 'i386':
-            if platform.architecture()[0] == '32bit':
-                arch = 'i386'
-            else:
-                arch = 'x86_64'
-        else:
-            # just a guess
-            arch = platform.machine()
-        g['LDSHARED'] += ' -undefined dynamic_lookup'
-        g['CC'] += ' -arch %s' % (arch,)
-
+    from _sysconfigdata import build_time_vars
     global _config_vars
-    _config_vars = g
+    _config_vars = build_time_vars
 
 
 def _init_nt():
@@ -221,4 +191,3 @@
 
 from .sysconfig_cpython import (
     parse_makefile, _variable_rx, expand_makefile_vars)
-
diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
--- a/lib-python/3/sysconfig.py
+++ b/lib-python/3/sysconfig.py
@@ -20,30 +20,30 @@
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{installed_base}/lib/python{py_version_short}',
-        'platstdlib': '{platbase}/lib/python{py_version_short}',
-        'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
+        'stdlib': '{installed_base}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{platbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{base}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{platbase}/lib/{implementation_lower}{py_version_short}/site-packages',
         'include':
-            '{installed_base}/include/python{py_version_short}{abiflags}',
+            '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}',
         'platinclude':
-            '{installed_platbase}/include/python{py_version_short}{abiflags}',
+            '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'posix_home': {
-        'stdlib': '{installed_base}/lib/python',
-        'platstdlib': '{base}/lib/python',
-        'purelib': '{base}/lib/python',
-        'platlib': '{base}/lib/python',
-        'include': '{installed_base}/include/python',
-        'platinclude': '{installed_base}/include/python',
+        'stdlib': '{installed_base}/lib/{implementation_lower}',
+        'platstdlib': '{base}/lib/{implementation_lower}',
+        'purelib': '{base}/lib/{implementation_lower}',
+        'platlib': '{base}/lib/{implementation_lower}',
+        'include': '{installed_base}/include/{implementation_lower}',
+        'platinclude': '{installed_base}/include/{implementation_lower}',
         'scripts': '{base}/bin',
         'data': '{base}',
         },
     'pypy': {
-        'stdlib': '{installed_base}/lib-python',
-        'platstdlib': '{base}/lib-python',
+        'stdlib': '{installed_base}/lib-{implementation_lower}',
+        'platstdlib': '{base}/lib-{implementation_lower}',
         'purelib': '{base}/site-packages',
         'platlib': '{base}/site-packages',
         'include': '{installed_base}/include',
@@ -62,28 +62,28 @@
         'data': '{base}',
         },
     'nt_user': {
-        'stdlib': '{userbase}/Python{py_version_nodot}',
-        'platstdlib': '{userbase}/Python{py_version_nodot}',
-        'purelib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'include': '{userbase}/Python{py_version_nodot}/Include',
-        'scripts': '{userbase}/Python{py_version_nodot}/Scripts',
+        'stdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'platstdlib': '{userbase}/{implementation}{py_version_nodot}',
+        'purelib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'platlib': '{userbase}/{implementation}{py_version_nodot}/site-packages',
+        'include': '{userbase}/{implementation}{py_version_nodot}/Include',
+        'scripts': '{userbase}/{implementation}{py_version_nodot}/Scripts',
         'data': '{userbase}',
         },
     'posix_user': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
+        'stdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}{py_version_short}',
+        'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}/site-packages',
+        'include': '{userbase}/include/{implementation_lower}{py_version_short}',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
         },
     'osx_framework_user': {
-        'stdlib': '{userbase}/lib/python',
-        'platstdlib': '{userbase}/lib/python',
-        'purelib': '{userbase}/lib/python/site-packages',
-        'platlib': '{userbase}/lib/python/site-packages',
+        'stdlib': '{userbase}/lib/{implementation_lower}',
+        'platstdlib': '{userbase}/lib/{implementation_lower}',
+        'purelib': '{userbase}/lib/{implementation_lower}/site-packages',
+        'platlib': '{userbase}/lib/{implementation_lower}/site-packages',
         'include': '{userbase}/include',
         'scripts': '{userbase}/bin',
         'data': '{userbase}',
@@ -106,6 +106,11 @@
 _USER_BASE = None
 
 
+def _get_implementation():
+    if '__pypy__' in sys.builtin_module_names:
+        return 'PyPy'
+    return 'Python'
+
 def _safe_realpath(path):
     try:
         return realpath(path)
@@ -556,6 +561,8 @@
         except AttributeError:
             # sys.abiflags may not be defined on all platforms.
             _CONFIG_VARS['abiflags'] = ''
+        _CONFIG_VARS['implementation'] = _get_implementation()
+        _CONFIG_VARS['implementation_lower'] = _get_implementation().lower()
 
         if os.name == 'nt':
             _init_non_posix(_CONFIG_VARS)
@@ -731,6 +738,8 @@
     _print_dict('Paths', get_paths())
     print()
     _print_dict('Variables', get_config_vars())
+    print
+    _print_dict('User', get_paths('%s_user' % os.name))
 
 
 if __name__ == '__main__':
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -4,6 +4,7 @@
 import subprocess
 import shutil
 from copy import copy
+from distutils.spawn import find_executable
 
 from test.support import (run_unittest,
                           import_module, TESTFN, unlink, check_warnings,
@@ -298,6 +299,30 @@
 
         self.assertIn(ldflags, ldshared)
 
+    @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows")
+    def test_cc_values(self):
+        """ CC and CXX should be set for pypy """
+        for var in ["CC", "CXX"]:
+            assert sysconfig.get_config_var(var) is not None
+
+    @unittest.skipIf(not find_executable("gcc"),
+        "Does not apply to machines without gcc installed"
+    )
+    def test_gcc_values(self):
+        """ if gcc is installed on the box, gcc values should be set. """
+        assert "gcc" in sysconfig.get_config_var("CC")
+        assert sysconfig.get_config_var("GNULD") == "yes"
+        assert "gcc" in sysconfig.get_config_var("LDSHARED")
+
+
+    @unittest.skipIf(not find_executable("g++"),
+        "Does not apply to machines without g++ installed"
+    )
+    def test_gplusplus_values(self):
+        """ if g++ is installed on the box, g++ values should be set. """
+        assert "g++" in sysconfig.get_config_var("CXX")
+
+
     @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX")
     def test_platform_in_subprocess(self):
         my_platform = sysconfig.get_platform()
diff --git a/lib-python/3/test/test_sysconfig_pypy.py b/lib-python/3/test/test_sysconfig_pypy.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3/test/test_sysconfig_pypy.py
@@ -0,0 +1,17 @@
+import os
+import sys
+import unittest
+import site
+
+
+class TestSysConfigPypy(unittest.TestCase):
+    def test_install_schemes(self):
+        # User-site etc. paths should have "pypy" and not "python"
+        # inside them.
+        if site.ENABLE_USER_SITE:
+            parts = site.USER_SITE.lower().split(os.path.sep)
+            assert any(x.startswith('pypy') for x in parts[-2:]), parts
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
--- a/lib_pypy/_cffi_ssl/README.md
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -14,6 +14,8 @@
 
 * ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1)
 
+* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates()
+
 
 # Tests?
 
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/pypy_win32_extra.py
@@ -0,0 +1,84 @@
+#
+# An extra bit of logic for the Win32-only functionality that is missing from the
+# version from cryptography.
+#
+
+import sys
+
+INCLUDES = """
+#include <Wincrypt.h>
+"""
+
+TYPES = """
+typedef ... *HCERTSTORE;
+typedef ... *HCRYPTPROV_LEGACY;
+
+typedef struct {
+    DWORD      dwCertEncodingType;
+    BYTE       *pbCertEncoded;
+    DWORD      cbCertEncoded;
+    ...;
+} CERT_CONTEXT, *PCCERT_CONTEXT;
+
+typedef struct {
+    DWORD      dwCertEncodingType;
+    BYTE       *pbCrlEncoded;
+    DWORD      cbCrlEncoded;
+    ...;
+} CRL_CONTEXT, *PCCRL_CONTEXT;
+
+typedef struct {
+    DWORD cUsageIdentifier;
+    LPSTR *rgpszUsageIdentifier;
+    ...;
+} CERT_ENHKEY_USAGE, *PCERT_ENHKEY_USAGE;
+"""
+
+FUNCTIONS = """
+HCERTSTORE WINAPI CertOpenStore(
+         LPCSTR            lpszStoreProvider,
+         DWORD             dwMsgAndCertEncodingType,
+         HCRYPTPROV_LEGACY hCryptProv,
+         DWORD             dwFlags,
+         const char        *pvPara
+);
+PCCERT_CONTEXT WINAPI CertEnumCertificatesInStore(
+         HCERTSTORE     hCertStore,
+         PCCERT_CONTEXT pPrevCertContext
+);
+BOOL WINAPI CertFreeCertificateContext(
+         PCCERT_CONTEXT pCertContext
+);
+BOOL WINAPI CertFreeCRLContext(
+         PCCRL_CONTEXT pCrlContext
+);
+BOOL WINAPI CertCloseStore(
+         HCERTSTORE hCertStore,
+         DWORD      dwFlags
+);
+BOOL WINAPI CertGetEnhancedKeyUsage(
+         PCCERT_CONTEXT     pCertContext,
+         DWORD              dwFlags,
+         PCERT_ENHKEY_USAGE pUsage,
+         DWORD              *pcbUsage
+);
+PCCRL_CONTEXT WINAPI CertEnumCRLsInStore(
+         HCERTSTORE    hCertStore,
+         PCCRL_CONTEXT pPrevCrlContext
+);
+"""
+
+MACROS = """
+#define CERT_STORE_READONLY_FLAG ...
+#define CERT_SYSTEM_STORE_LOCAL_MACHINE ...
+#define CRYPT_E_NOT_FOUND ...
+#define CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG ...
+#define CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG ...
+#define X509_ASN_ENCODING ...
+#define PKCS_7_ASN_ENCODING ...
+
+static const LPCSTR CERT_STORE_PROV_SYSTEM_A;
+"""
+
+CUSTOMIZATIONS = """
+"""
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -24,6 +24,7 @@
 from enum import IntEnum as _IntEnum
 
 if sys.platform == 'win32':
+    from _cffi_ssl._stdssl.win32_extra import enum_certificates, enum_crls
     HAVE_POLL = False
 else:
     from select import poll, POLLIN, POLLOUT
diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
@@ -0,0 +1,101 @@
+from _pypy_openssl import lib, ffi
+
+
+def enum_certificates(store_name):
+    """Retrieve certificates from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type, trust) tuples.  The encoding_type flag can be interpreted
+with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
+a set of OIDs or the boolean True.
+    """
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name, "ascii"))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+    
+    result = []
+    pCertCtx = ffi.NULL
+    try:
+        while True:
+            pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
+            if pCertCtx == ffi.NULL:
+                break
+            cert = ffi.buffer(pCertCtx.pbCertEncoded, pCertCtx.cbCertEncoded)[:]
+            enc = certEncodingType(pCertCtx.dwCertEncodingType)
+            keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
+            if keyusage is True:
+                keyusage = parseKeyUsage(pCertCtx, lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
+            result.append((cert, enc, keyusage))
+    finally:
+        if pCertCtx != ffi.NULL:
+            lib.CertFreeCertificateContext(pCertCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def enum_crls(store_name):
+    """Retrieve CRLs from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'.  The system may provide
+more cert storages, too.  The function returns a list of (bytes,
+encoding_type) tuples.  The encoding_type flag can be interpreted with
+X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
+    hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+                               lib.CERT_STORE_READONLY_FLAG | lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                               bytes(store_name, "ascii"))
+    if hStore == ffi.NULL:
+        raise WindowsError(*ffi.getwinerror())
+
+    result = []
+    pCrlCtx = ffi.NULL
+    try:
+        while True:
+            pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
+            if pCrlCtx == ffi.NULL:
+                break
+            crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
+            enc = certEncodingType(pCrlCtx.dwCertEncodingType)
+            result.append((crl, enc))
+    finally:
+        if pCrlCtx != ffi.NULL:
+            lib.CertFreeCRLContext(pCrlCtx)
+        if not lib.CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise WindowsError(*ffi.getwinerror())
+    return result
+
+
+def certEncodingType(encodingType):
+    if encodingType == lib.X509_ASN_ENCODING:
+        return "x509_asn"
+    if encodingType == lib.PKCS_7_ASN_ENCODING:
+        return "pkcs_7_asn"
+    return encodingType
+
+def parseKeyUsage(pCertCtx, flags):
+    pSize = ffi.new("DWORD *")
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    pUsageMem = ffi.new("char[]", pSize[0])
+    pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
+    if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
+        error_with_message = ffi.getwinerror()
+        if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+            return True
+        raise WindowsError(*error_with_message)
+
+    retval = set()
+    for i in range(pUsage.cUsageIdentifier):
+        if pUsage.rgpszUsageIdentifier[i]:
+            oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
+            retval.add(oid)
+    return retval
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -81,8 +81,11 @@
     def _CData_output(self, resarray, base=None, index=-1):
         from _rawffi.alt import types
         # If a char_p or unichar_p is received, skip the string interpretation
-        if base._ffiargtype != types.Pointer(types.char_p) and \
-           base._ffiargtype != types.Pointer(types.unichar_p):
+        try:
+            deref = type(base)._deref_ffiargtype()
+        except AttributeError:
+            deref = None
+        if deref != types.char_p and deref != types.unichar_p:
             # this seems to be a string if we're array of char, surprise!
             from ctypes import c_char, c_wchar
             if self._type_ is c_char:
@@ -127,6 +130,12 @@
                 value = self(*value)
         return _CDataMeta.from_param(self, value)
 
+    def _build_ffiargtype(self):
+        return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+    def _deref_ffiargtype(self):
+        return self._type_.get_ffi_argtype()
+
 def array_get_slice_params(self, index):
     if hasattr(self, '_length_'):
         start, stop, step = index.indices(self._length_)
@@ -254,6 +263,5 @@
             _type_ = base
         )
         cls = ArrayMeta(name, (Array,), tpdict)
-        cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype())
         ARRAY_CACHE[key] = cls
         return cls
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -49,10 +49,13 @@
         else:
             return self.from_param(as_parameter)
 
+    def _build_ffiargtype(self):
+        return _shape_to_ffi_type(self._ffiargshape_)
+
     def get_ffi_argtype(self):
         if self._ffiargtype:
             return self._ffiargtype
-        self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_)
+        self._ffiargtype = self._build_ffiargtype()
         return self._ffiargtype
 
     def _CData_output(self, resbuffer, base=None, index=-1):
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -70,7 +70,12 @@
         self._ffiarray = ffiarray
         self.__init__ = __init__
         self._type_ = TP
-        self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype())
+
+    def _build_ffiargtype(self):
+        return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+    def _deref_ffiargtype(self):
+        return self._type_.get_ffi_argtype()
 
     from_address = cdata_from_address
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -160,6 +160,10 @@
             raise AttributeError("_fields_ is final")
         if self in [f[1] for f in value]:
             raise AttributeError("Structure or union cannot contain itself")
+        if self._ffiargtype is not None:
+            raise NotImplementedError("Too late to set _fields_: we already "
+                        "said to libffi that the structure type %s is opaque"
+                        % (self,))
         names_and_fields(
             self,
             value, self.__bases__[0],
diff --git a/lib_pypy/_ssl/__init__.py b/lib_pypy/_ssl/__init__.py
--- a/lib_pypy/_ssl/__init__.py
+++ b/lib_pypy/_ssl/__init__.py
@@ -16,12 +16,14 @@
     RAND_egd          = builtinify(RAND_egd)
 
 import sys
-if sys.platform == "win32" and 'enum_certificates' not in globals():
-    def enum_certificates(*args, **kwds):
-        import warnings
-        warnings.warn("ssl.enum_certificates() is not implemented")
-        return []
-    def enum_crls(*args, **kwds):
-        import warnings
-        warnings.warn("ssl.enum_crls() is not implemented")
-        return []
+if sys.platform == "win32":
+    if 'enum_certificates' not in globals():
+        def enum_certificates(*args, **kwds):
+            import warnings
+            warnings.warn("ssl.enum_certificates() is not implemented")
+            return []
+    if 'enum_crls' not in globals():
+        def enum_crls(*args, **kwds):
+            import warnings
+            warnings.warn("ssl.enum_crls() is not implemented")
+            return []
diff --git a/lib_pypy/_ssl_build.py b/lib_pypy/_ssl_build.py
--- a/lib_pypy/_ssl_build.py
+++ b/lib_pypy/_ssl_build.py
@@ -5,6 +5,11 @@
 from _cffi_ssl._cffi_src.build_openssl import (build_ffi_for_binding,
         _get_openssl_libraries, extra_link_args, compiler_type)
 
+if sys.platform == "win32":
+    pypy_win32_extra = ["pypy_win32_extra"]
+else:
+    pypy_win32_extra = []
+
 ffi = build_ffi_for_binding(
     module_name="_pypy_openssl",
     module_prefix="_cffi_src.openssl.",
@@ -44,10 +49,10 @@
         "x509_vfy",
         "pkcs7",
         "callbacks",
-    ],
+    ] + pypy_win32_extra,
     libraries=_get_openssl_libraries(sys.platform),
     extra_link_args=extra_link_args(compiler_type()),
 )
 
 if __name__ == '__main__':
-    ffi.compile()
+    ffi.compile(verbose=True)
diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -1,10 +1,47 @@
 import _imp
+import os
+import sys
+from distutils.spawn import find_executable
 
 so_ext = _imp.extension_suffixes()[0]
 
+
 build_time_vars = {
-    "EXT_SUFFIX": so_ext,
-    "SHLIB_SUFFIX": so_ext,
     "SOABI": '-'.join(so_ext.split('.')[1].split('-')[:2]),
-    "SO": so_ext  # deprecated in Python 3, for backward compatibility
+    "SO": so_ext,  # deprecated in Python 3, for backward compatibility
+    'CC': "cc -pthread",
+    'CXX': "c++ -pthread",
+    'OPT': "-DNDEBUG -O2",
+    'CFLAGS': "-DNDEBUG -O2",
+    'CCSHARED': "-fPIC",
+    'LDSHARED': "cc -pthread -shared",
+    'EXT_SUFFIX': so_ext,
+    'SHLIB_SUFFIX': ".so",
+    'AR': "ar",
+    'ARFLAGS': "rc",
+    'EXE': "",
+    'LIBDIR': os.path.join(sys.prefix, 'lib'),
+    'VERSION': sys.version[:3]
 }
+
+if sys.platform[:6] == "darwin":
+    import platform
+    if platform.machine() == 'i386':
+        if platform.architecture()[0] == '32bit':
+            arch = 'i386'
+        else:
+            arch = 'x86_64'
+    else:
+        # just a guess
+        arch = platform.machine()
+    build_time_vars['LDSHARED'] += ' -undefined dynamic_lookup'
+    build_time_vars['CC'] += ' -arch %s' % (arch,)
+
+if find_executable("gcc"):
+    build_time_vars.update({
+        "CC": "gcc -pthread",
+        "GNULD": "yes",
+        "LDSHARED": "gcc -pthread -shared",
+    })
+    if find_executable("g++"):
+        build_time_vars["CXX"] = "g++ -pthread"
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,7 +39,7 @@
     "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
-    "_csv", "_pypyjson", "_posixsubprocess", # "_cppyy", "micronumpy"
+    "_csv", "_pypyjson", "_posixsubprocess", "_cppyy", # "micronumpy",
     "_jitlog",
 ])
 
diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -73,3 +73,63 @@
 This division between bytecode evaluator and object space gives a lot of
 flexibility.  One can plug in different :doc:`object spaces <objspace>` to get
 different or enriched behaviours of the Python objects.
+
+Layers
+------
+
+RPython
+~~~~~~~
+:ref:`RPython <rpython:language>` is the language in which we write interpreters.
+Not the entire PyPy project is written in RPython, only the parts that are
+compiled in the translation process. The interesting point is that RPython
+has no parser, it's compiled from the live python objects, which makes it
+possible to do all kinds of metaprogramming during import time. In short,
+Python is a meta programming language for RPython.
+
+The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+Consult `Getting Started with RPython`_ for further reading
+
+Translation
+~~~~~~~~~~~
+The translation toolchain - this is the part that takes care of translating
+RPython to flow graphs and then to C. There is more in the
+:doc:`architecture <architecture>` document written about it.
+
+It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+and ``rtyper``.
+
+PyPy Interpreter
+~~~~~~~~~~~~~~~~
+This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
+interpreter for Python written in RPython.  The fact that it is
+RPython is not apparent at first.  Built-in modules are written in
+``pypy/module/*``.  Some modules that CPython implements in C are
+simply written in pure Python; they are in the top-level ``lib_pypy``
+directory.  The standard library of Python (with a few changes to
+accomodate PyPy) is in ``lib-python``.
+
+JIT Compiler
+~~~~~~~~~~~~
+:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
+interpreter written in RPython, rather than the user program that it
+interprets.  As a result it applies to any interpreter, i.e. any
+language.  But getting it to work correctly is not trivial: it
+requires a small number of precise "hints" and possibly some small
+refactorings of the interpreter.  The JIT itself also has several
+almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+that turns it into machine code.  Writing a new backend is a
+traditional way to get into the project.
+
+Garbage Collectors
+~~~~~~~~~~~~~~~~~~
+Garbage Collectors (GC): as you may notice if you are used to CPython's
+C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+:ref:`rpython:garbage-collection` is inserted
+during translation.  Moreover, this is not reference counting; it is a real
+GC written as more RPython code.  The best one we have so far is in
+``rpython/memory/gc/incminimark.py``.
+
+.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -267,14 +267,14 @@
 * PyPy 2.5.1 or earlier: normal users would see permission errors.
   Installers need to run ``pypy -c "import gdbm"`` and other similar
   commands at install time; the exact list is in
-  :source:`pypy/tool/release/package.py <package.py>`.  Users
+  :source:`pypy/tool/release/package.py`.  Users
   seeing a broken installation of PyPy can fix it after-the-fact if they
   have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
 
 * PyPy 2.6 and later: anyone would get ``ImportError: no module named
   _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
   ``lib_pypy`` directory during the installation process (plus others;
-  see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+  see the exact list in :source:`pypy/tool/release/package.py`).
   Users seeing a broken
   installation of PyPy can fix it after-the-fact, by running ``pypy
   /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -539,7 +539,7 @@
 
     hg help branch
 
-.. _official wiki: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: https://www.mercurial-scm.org/wiki/
 
 
 .. _using-development-tracker:
@@ -547,15 +547,7 @@
 Using the development bug/feature tracker
 -----------------------------------------
 
-We have a `development tracker`_, based on Richard Jones'
-`roundup`_ application.  You can file bugs,
-feature requests or see what's going on
-for the next milestone, both from an E-Mail and from a
-web interface.
-
-.. _development tracker: https://bugs.pypy.org/
-.. _roundup: http://roundup.sourceforge.net/
-
+We use bitbucket for :source:`issues` tracking and :source:`pull-requests`.
 
 .. _testing:
 
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -8,3 +8,4 @@
    :maxdepth: 1
 
    man/pypy.1.rst
+   man/pypy3.1.rst
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -66,9 +66,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '5.8'
+version = '6.0'
 # The full version, including alpha/beta/rc tags.
-release = '5.8.0'
+release = '6.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/config/objspace.usemodules._cppyy.txt b/pypy/doc/config/objspace.usemodules._cppyy.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules._cppyy.txt
@@ -0,0 +1,1 @@
+The internal backend for cppyy
diff --git a/pypy/doc/config/objspace.usemodules._rawffi.txt b/pypy/doc/config/objspace.usemodules._rawffi.txt
--- a/pypy/doc/config/objspace.usemodules._rawffi.txt
+++ b/pypy/doc/config/objspace.usemodules._rawffi.txt
@@ -1,3 +1,3 @@
-An experimental module providing very low-level interface to
+A module providing very low-level interface to
 C-level libraries, for use when implementing ctypes, not
-intended for a direct use at all.
\ No newline at end of file
+intended for a direct use at all.
diff --git a/pypy/doc/config/objspace.usemodules.cpyext.txt b/pypy/doc/config/objspace.usemodules.cpyext.txt
--- a/pypy/doc/config/objspace.usemodules.cpyext.txt
+++ b/pypy/doc/config/objspace.usemodules.cpyext.txt
@@ -1,1 +1,1 @@
-Use (experimental) cpyext module, that tries to load and run CPython extension modules
+Use cpyext module to load and run CPython extension modules
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/contributing.rst
@@ -0,0 +1,472 @@
+Contributing Guidelines
+===========================
+
+.. contents::
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy's
+  approach is great if you do TDD, and not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below.
+
+The first thing to remember is that PyPy project is very different than most
+projects out there. It's also different from a classic compiler project,
+so academic courses about compilers often don't apply or lead in the wrong
+direction. However, if you want to understand how designing & building a runtime
+works in the real world then this is a great project!
+
+Getting involved
+^^^^^^^^^^^^^^^^
+
+PyPy employs a relatively standard open-source development process. You are
+encouraged as a first step to join our `pypy-dev mailing list`_ and IRC channel,
+details of which can be found in our :ref:`contact <contact>` section. The folks
+there are very friendly, and can point you in the right direction.
+
+We give out commit rights usually fairly liberally, so if you want to do something
+with PyPy, you can become a committer. We also run frequent coding sprints which
+are separately announced and often happen around Python conferences such as
+EuroPython or PyCon. Upcoming events are usually announced on `the blog`_.
+
+Further Reading: :ref:`Contact <contact>`
+
+.. _the blog: http://morepypy.blogspot.com
+.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
+
+
+Your first contribution
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The first and most important rule how **not** to contribute to PyPy is
+"just hacking a feature". This won't work, and you'll find your PR will typically
+require a lot of re-work. There are a few reasons why not:
+
+* build times are large
+* PyPy has very thick layer separation
+* context of the cPython runtime is often required
+
+Instead, reach out on the dev mailing list or the IRC channel, and we're more
+than happy to help! :)
+
+Some ideas for first contributions are:
+
+* Documentation - this will give you an understanding of the pypy architecture
+* Test failures - find a failing test in the `nightly builds`_, and fix it
+* Missing language features - these are listed in our `issue tracker`_
+
+.. _nightly builds: http://buildbot.pypy.org/nightly/
+.. _issue tracker: https://bitbucket.org/pypy/pypy/issues
+
+Source Control
+--------------
+
+PyPy development is based a typical fork/pull request based workflow, centered
+around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
+before, a good introduction can be found here:
+
+    https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow
+
+The cycle for a new PyPy contributor goes typically like this:
+
+Fork & Clone
+------------
+
+* Make an account on bitbucket_.
+
+* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
+  icons).  You get a fork of the repository, e.g. in
+  `https://bitbucket.org/yourname/pypy/`.
+
+* Clone your new repo (i.e. the fork) to your local machine with the command
+  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
+  operation but only ever needs to be done once.  See also
+  http://pypy.org/download.html#building-from-source .
+  If you already cloned
+  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
+  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
+  your clone to contain the line ``default =
+  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
+  up``.  If you already have such a clone but don't want to change it,
+  you can clone that copy with ``hg clone /path/to/other/copy``, and
+  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
+
+* Now you have a complete copy of the PyPy repo.  Make a branch
+  with a command like ``hg branch name_of_your_branch``.
+
+Edit
+----
+
+* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
+  to make Mercurial aware of new files you added, e.g. new test files.
+  Use ``hg status`` to see if there are such files.  Write and run tests!
+  (See the rest of this page.)
+
+* Commit regularly with ``hg commit``.  A one-line commit message is
+  fine.  We love to have tons of commits; make one as soon as you have
+  some progress, even if it is only some new test that doesn't pass yet,
+  or fixing things even if not all tests pass.  Step by step, you are
+  building the history of your changes, which is the point of a version
+  control system.  (There are commands like ``hg log`` and ``hg up``
+  that you should read about later, to learn how to navigate this
+  history.)
+
+* The commits stay on your machine until you do ``hg push`` to "push"
+  them back to the repo named in the file ``.hg/hgrc``.  Repos are
+  basically just collections of commits (a commit is also called a
+  changeset): there is one repo per url, plus one for each local copy on
+  each local machine.  The commands ``hg push`` and ``hg pull`` copy
+  commits around, with the goal that all repos in question end up with
+  the exact same set of commits.  By opposition, ``hg up`` only updates
+  the "working copy" by reading the local repository, i.e. it makes the
+  files that you see correspond to the latest (or any other) commit
+  locally present.
+
+* You should push often; there is no real reason not to.  Remember that
+  even if they are pushed, with the setup above, the commits are (1)
+  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
+  named.  Yes, they are publicly visible, but don't worry about someone
+  walking around the thousands of repos on bitbucket saying "hah, look
+  at the bad coding style of that guy".  Try to get into the mindset
+  that your work is not secret and it's fine that way.  We might not
+  accept it as is for PyPy, asking you instead to improve some things,
+  but we are not going to judge you.
+
+Pull Request
+------------
+
+* The final step is to open a pull request, so that we know that you'd
+  like to merge that branch back to the original ``pypy/pypy`` repo.
+  This can also be done several times if you have interesting
+  intermediate states, but if you get there, then we're likely to
+  proceed to the next stage, which is...
+
+* Get a regular account for pushing directly to
+  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
+  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
+  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
+  then be pushed directly to the official repo, but (if you follow these
+  rules) they are still on a branch, and we can still review the
+  branches you want to merge.
+
+* If you get closer to the regular day-to-day development, you'll notice
+  that we generally push small changes as one or a few commits directly
+  to the branch ``default``.  Also, we often collaborate even if we are
+  on other branches, which do not really "belong" to anyone.  At this
+  point you'll need ``hg merge`` and learn how to resolve conflicts that
+  sometimes occur when two people try to push different commits in
+  parallel on the same branch.  But it is likely an issue for later ``:-)``
+
+.. _bitbucket: https://bitbucket.org/
+
+
+Architecture
+^^^^^^^^^^^^
+
+PyPy has layers. Just like ogres or onions. Those layers help us keep the
+respective parts separated enough to be worked on independently and make the
+complexity manageable. This is, again, just a sanity requirement for such
+a complex project. For example writing a new optimization for the JIT usually
+does **not** involve touching a Python interpreter at all or the JIT assembler
+backend or the garbage collector. Instead it requires writing small tests in
+``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
+After that, you can just compile PyPy and things should just work.
+
+Further Reading: :doc:`architecture <architecture>`
+
+Where to start?
+---------------
+
+PyPy is made from parts that are relatively independent of each other.
+You should start looking at the part that attracts you most (all paths are
+relative to the PyPy top level directory).  You may look at our
+:doc:`directory reference <dir-reference>` or start off at one of the following
+points:
+
+*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
+   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
+   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
+   function objects and argument passing in :source:`pypy/interpreter/function.py`
+   and :source:`pypy/interpreter/argument.py`, the object space interface
+   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
+   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
+   Core types supporting the bytecode interpreter are defined in
+   :source:`pypy/interpreter/typedef.py`.
+
+*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
+   and grammar files that allow it to parse the syntax of various Python
+   versions. Once the grammar has been processed, the parser can be
+   translated by the above machinery into efficient code.
+
+*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
+   contains a modified version of the compiler package from CPython
+   that fixes some bugs and is translatable.
+
+*  :source:`pypy/objspace/std` contains the
+   :ref:`Standard object space <standard-object-space>`.  The main file
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)
+
+Building
+^^^^^^^^
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Further Reading: :doc:`Build <build>`
+
+Coding Guide
+------------
+
+As well as the usual pep8 and formatting standards, there are a number of
+naming conventions and coding styles that are important to understand before
+browsing the source.
+
+Further Reading: :doc:`Coding Guide <coding-guide>`
+
+Testing
+^^^^^^^
+
+Test driven development
+-----------------------
+
+Instead, we practice a lot of test driven development. This is partly because
+of very high quality requirements for compilers and partly because there is
+simply no other way to get around such complex project, that will keep you sane.
+There are probably people out there who are smart enough not to need it, we're
+not one of those. You may consider familiarizing yourself with `pytest`_,
+since this is a tool we use for tests.
+This leads to the next issue:
+
+.. _pytest: http://pytest.org/
+
+py.test and the py lib
+----------------------
+
+The `py.test testing tool`_ drives all our testing needs.
+
+We use the `py library`_ for filesystem path manipulations, terminal
+writing, logging and some other support  functionality.
+
+You don't necessarily need to install these two libraries because
+we also ship them inlined in the PyPy source tree.
+
+.. _py library: http://pylib.readthedocs.org/
+
+Running PyPy's unit tests
+-------------------------
+
+PyPy development always was and is still thoroughly test-driven.
+We use the flexible `py.test testing tool`_ which you can `install independently
+<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
+
+The PyPy source tree comes with an inlined version of ``py.test``
+which you can invoke by typing::
+
+    python pytest.py -h
+
+This is usually equivalent to using an installed version::
+
+    py.test -h
+
+If you encounter problems with the installed version
+make sure you have the correct version installed which
+you can find out with the ``--version`` switch.
+
+You will need the `build requirements`_ to run tests successfully, since many of
+them compile little pieces of PyPy and then run the tests inside that minimal
+interpreter. The `cpyext` tests also require `pycparser`, and many tests build
+cases with `hypothesis`.
+
+Now on to running some tests.  PyPy has many different test directories
+and you can use shell completion to point at directories or files::
+
+    py.test pypy/interpreter/test/test_pyframe.py
+
+    # or for running tests of a whole subdirectory
+    py.test pypy/interpreter/
+
+See `py.test usage and invocations`_ for some more generic info
+on how you can run tests.
+
+Beware trying to run "all" pypy tests by pointing to the root
+directory or even the top level subdirectory ``pypy``.  It takes
+hours and uses huge amounts of RAM and is not recommended.
+
+To run CPython regression tests you can point to the ``lib-python``
+directory::
+
+    py.test lib-python/2.7/test/test_datetime.py
+
+This will usually take a long time because this will run
+the PyPy Python interpreter on top of CPython.  On the plus
+side, it's usually still faster than doing a full translation
+and running the regression test with the translated PyPy Python
+interpreter.
+
+.. _py.test testing tool: http://pytest.org
+.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
+.. _`build requirements`: build.html#install-build-time-dependencies
+
+Testing After Translation
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While the usual invocation of `pytest` translates a piece of RPython code and
+runs it, we have a test extension to run tests without translation, directly
+on the host python. This is very convenient for modules such as `cpyext`, to
+compare and contrast test results between CPython and PyPy. Untranslated tests
+are invoked by using the `-A` or `--runappdirect` option to `pytest`::
+
+    python2 pytest.py -A pypy/module/cpyext/test
+
+where `python2` can be either `python2` or `pypy2`. On the `py3` branch, the
+collection phase must be run with `python2` so untranslated tests are run
+with::
+
+    cpython2 pytest.py -A pypy/module/cpyext/test --python=path/to/pypy3
+
+
+Tooling & Utilities
+^^^^^^^^^^^^^^^^^^^
+
+If you are interested in the inner workings of the PyPy Python interpreter,
+there are some features of the untranslated Python interpreter that allow you
+to introspect its internals.
+
+
+Interpreter-level console
+-------------------------
+
+To start interpreting Python with PyPy, install a C compiler that is
+supported by distutils and use Python 2.7 or greater to run PyPy::
+
+    cd pypy
+    python bin/pyinteractive.py
+
+After a few seconds (remember: this is running on top of CPython), you should
+be at the PyPy prompt, which is the same as the Python prompt, but with an
+extra ">".
+
+If you press
+<Ctrl-C> on the console you enter the interpreter-level console, a
+usual CPython console.  You can then access internal objects of PyPy
+(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
+prompt with the prefix ``w_``::
+
+    >>>> a = 123
+    >>>> <Ctrl-C>
+    *** Entering interpreter-level console ***
+    >>> w_a
+    W_IntObject(123)
+
+The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
+
+    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
+    >>> <Ctrl-D>
+    *** Leaving interpreter-level console ***
+
+    KeyboardInterrupt
+    >>>> l
+    [1, 'abc']
+
+Note that the prompt of the interpreter-level console is only '>>>' since
+it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
+Linux) or <Ctrl-Z>, <Enter> (under Windows).
+
+Also note that not all modules are available by default in this mode (for
+example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
+``--withmod-...`` command line options.
+
+You may be interested in reading more about the distinction between
+:ref:`interpreter-level and app-level <interpreter-level>`.
+
+pyinteractive.py options
+------------------------
+
+To list the PyPy interpreter command line options, type::
+
+    cd pypy
+    python bin/pyinteractive.py --help
+
+pyinteractive.py supports most of the options that CPython supports too (in addition to a
+large amount of options that can be used to customize pyinteractive.py).
+As an example of using PyPy from the command line, you could type::
+
+    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
+
+Alternatively, as with regular Python, you can simply give a
+script name on the command line::
+
+    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
+
+See our :doc:`configuration sections <config/index>` for details about what all the commandline
+options do.
+
+
+.. _trace example:
+
+Tracing bytecode and operations on objects
+------------------------------------------
+
+You can use a simple tracing mode to monitor the interpretation of
+bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
+PyPy console::
+
+    >>>> __pytrace__ = 1
+    Tracing enabled
+    >>>> x = 5
+            <module>:           LOAD_CONST    0 (5)
+            <module>:           STORE_NAME    0 (x)
+            <module>:           LOAD_CONST    1 (None)
+            <module>:           RETURN_VALUE    0
+    >>>> x
+            <module>:           LOAD_NAME    0 (x)
+            <module>:           PRINT_EXPR    0
+    5
+            <module>:           LOAD_CONST    0 (None)
+            <module>:           RETURN_VALUE    0
+    >>>>
+
+
+Demos
+^^^^^
+
+The `example-interpreter`_ repository contains an example interpreter
+written using the RPython translation toolchain.
+
+.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
+
+
+graphviz & pygame for flow graph viewing (highly recommended)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+graphviz and pygame are both necessary if you want to look at generated flow
+graphs:
+
+    graphviz: http://www.graphviz.org/Download.php
+
+    pygame: http://www.pygame.org/download.shtml
+
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -214,6 +214,7 @@
   Lukas Vacek
   Omer Katz
   Jacek Generowicz
+  Tomasz Dziopa
   Sylvain Thenault
   Jakub Stasiak
   Andrew Dalke
@@ -274,6 +275,7 @@
   Yury V. Zaytsev
   florinpapa
   Anders Sigfridsson
+  Matt Jackson
   Nikolay Zinov
   rafalgalczynski at gmail.com
   Joshua Gilbert
diff --git a/pypy/doc/discussion/ctypes-implementation.rst b/pypy/doc/discussion/ctypes-implementation.rst
--- a/pypy/doc/discussion/ctypes-implementation.rst
+++ b/pypy/doc/discussion/ctypes-implementation.rst
@@ -141,28 +141,3 @@
 
 .. _pyglet: http://pyglet.org/
 
-
-ctypes configure
------------------
-
-We also released ``ctypes-configure``, which is an experimental package
-trying to approach the portability issues of ctypes-based code.
-
-idea
-~~~~
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes c
-compiler (via distutils) for various platform-dependent details like
-exact sizes of types (for example size_t), ``#defines``, exact outline of
-structures etc. It replaces in this regard code generator (h2py).
-
-installation
-~~~~~~~~~~~~
-
-``easy_install ctypes_configure``
-
-usage
-~~~~~
-
-:source:`ctypes_configure/doc/sample.py` explains in details how to use it.
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -1,5 +1,5 @@
-Embedding PyPy
-==============
+Embedding PyPy (DEPRECATED)
+===========================
 
 PyPy has a very minimal and a very strange embedding interface, based on
 the usage of `cffi`_ and the philosophy that Python is a better language than
diff --git a/pypy/doc/eventhistory.rst b/pypy/doc/eventhistory.rst
--- a/pypy/doc/eventhistory.rst
+++ b/pypy/doc/eventhistory.rst
@@ -40,11 +40,9 @@
 Main focus of the sprint will be on the goals of the upcoming June 0.9
 release.
 
-Read more in `the sprint announcement`__, see who is  planning to attend
-on the `people page`_.
+Read more about `the sprint`__
 
-__ https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/announce.html
-.. _people page: https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/people.txt
+__ https://bitbucket.org/pypy/extradoc/src/extradoc/sprintinfo/ddorf2006/
 
 
 PyPy sprint at Akihabara (Tokyo, Japan)
diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst
--- a/pypy/doc/extradoc.rst
+++ b/pypy/doc/extradoc.rst
@@ -75,12 +75,12 @@
 .. _A Way Forward in Parallelising Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2014/position-paper.pdf
 .. _Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf
 .. _Allocation Removal by Partial Evaluation in a Tracing JIT: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/pepm2011/bolz-allocation-removal.pdf
-.. _Towards a Jitting VM for Prolog Execution: http://www.stups.uni-duesseldorf.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
+.. _Towards a Jitting VM for Prolog Execution: http://stups.hhu.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
 .. _High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages: http://buildbot.pypy.org/misc/antocuni-thesis.pdf
 .. _How to *not* write Virtual Machines for Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dyla2007/dyla.pdf
 .. _`Tracing the Meta-Level: PyPy's Tracing JIT Compiler`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009/bolz-tracing-jit.pdf
 .. _`Faster than C#: Efficient Implementation of Dynamic Languages on .NET`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009-dotnet/cli-jit.pdf
-.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: http://stups.hhu.de/mediawiki/images/b/b9/Master_bolz.pdf
+.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: https://www.researchgate.net/profile/Davide_Ancona/publication/252023163_Automatic_generation_of_JIT_compilers_for_dynamic_languages_in_NET/links/53f2098e0cf2bc0c40e70023/Automatic-generation-of-JIT-compilers-for-dynamic-languages-in-NET.pdf
 .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/DynamicLanguages_abstracts.html#AACM-DLS07
 .. _EU Reports: index-report.html
 .. _Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution: http://sabi.net/nriley/pubs/dls6-riley.pdf
@@ -368,6 +368,6 @@
 .. _LLVM: http://llvm.org/
 .. _IronPython: http://ironpython.codeplex.com/
 .. _Dynamic Native Optimization of Native Interpreters: http://people.csail.mit.edu/gregs/dynamorio.html
-.. _JikesRVM: http://jikesrvm.org/
+.. _JikesRVM: http://www.jikesrvm.org/
 .. _Tunes: http://tunes.org
 .. _old Tunes Wiki: http://buildbot.pypy.org/misc/cliki.tunes.org/
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -67,7 +67,7 @@
 you may need to run the command with `sudo` for a global installation.
 The other commands of ``setup.py`` are available too, like ``build``.
 
-.. _PyPI: https://pypi.python.org/pypi
+.. _PyPI: https://pypi.org
 .. _`use virtualenv (as documented here)`: install.html#installing-using-virtualenv
 
 
@@ -360,7 +360,7 @@
 (produced during a sprint).  On the `PyPy bitbucket page`_ there is also a
 Scheme and an Io implementation; both of these are unfinished at the moment.
 
-.. _Topaz: http://topazruby.com/
+.. _Topaz: http://docs.topazruby.com/en/latest/
 .. _Hippy: http://morepypy.blogspot.ch/2012/07/hello-everyone.html
 .. _JavaScript interpreter: https://bitbucket.org/pypy/lang-js/
 .. _Prolog interpreter: https://bitbucket.org/cfbolz/pyrolog/
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -152,7 +152,7 @@
 to wait until it reaches a point in which the interpreter is in a known state
 and calling user-defined code is harmless.  It might happen that multiple
 events occur before the hook is invoked: in this case, you can inspect the
-value ``stats.count`` to know how many times the event occured since the last
+value ``stats.count`` to know how many times the event occurred since the last
 time the hook was called.  Similarly, ``stats.duration`` contains the
 **total** time spent by the GC for this specific event since the last time the
 hook was called.
@@ -163,7 +163,7 @@
 The attributes for ``GcMinorStats`` are:
 
 ``count``
-    The number of minor collections occured since the last hook call.
+    The number of minor collections occurred since the last hook call.
 
 ``duration``
     The total time spent inside minor collections since the last hook
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
deleted file mode 100644
--- a/pypy/doc/getting-started-dev.rst
+++ /dev/null
@@ -1,345 +0,0 @@
-Getting Started Developing With PyPy
-====================================
-
-.. contents::
-
-
-Using Mercurial
----------------
-
-PyPy development is based on Mercurial (hg).  If you are not used to
-version control, the cycle for a new PyPy contributor goes typically
-like this:
-
-* Make an account on bitbucket_.
-
-* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
-  icons).  You get a fork of the repository, e.g. in
-  https://bitbucket.org/yourname/pypy/.
-
-* Clone this new repo (i.e. the fork) to your local machine with the command 
-  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
-  operation but only ever needs to be done once.  See also 
-  http://pypy.org/download.html#building-from-source .
-  If you already cloned
-  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
-  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
-  your clone to contain the line ``default =
-  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
-  up``.  If you already have such a clone but don't want to change it,
-  you can clone that copy with ``hg clone /path/to/other/copy``, and
-  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
-
-* Now you have a complete copy of the PyPy repo.  Make a branch
-  with a command like ``hg branch name_of_your_branch``.
-
-* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
-  to make Mercurial aware of new files you added, e.g. new test files.
-  Use ``hg status`` to see if there are such files.  Write and run tests!
-  (See the rest of this page.)
-
-* Commit regularly with ``hg commit``.  A one-line commit message is
-  fine.  We love to have tons of commits; make one as soon as you have
-  some progress, even if it is only some new test that doesn't pass yet,
-  or fixing things even if not all tests pass.  Step by step, you are
-  building the history of your changes, which is the point of a version
-  control system.  (There are commands like ``hg log`` and ``hg up``
-  that you should read about later, to learn how to navigate this
-  history.)
-
-* The commits stay on your machine until you do ``hg push`` to "push"
-  them back to the repo named in the file ``.hg/hgrc``.  Repos are
-  basically just collections of commits (a commit is also called a
-  changeset): there is one repo per url, plus one for each local copy on
-  each local machine.  The commands ``hg push`` and ``hg pull`` copy
-  commits around, with the goal that all repos in question end up with
-  the exact same set of commits.  By opposition, ``hg up`` only updates
-  the "working copy" by reading the local repository, i.e. it makes the
-  files that you see correspond to the latest (or any other) commit
-  locally present.
-
-* You should push often; there is no real reason not to.  Remember that
-  even if they are pushed, with the setup above, the commits are (1)
-  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
-  named.  Yes, they are publicly visible, but don't worry about someone
-  walking around the thousands of repos on bitbucket saying "hah, look
-  at the bad coding style of that guy".  Try to get into the mindset
-  that your work is not secret and it's fine that way.  We might not
-  accept it as is for PyPy, asking you instead to improve some things,
-  but we are not going to judge you.
-
-* The final step is to open a pull request, so that we know that you'd
-  like to merge that branch back to the original ``pypy/pypy`` repo.
-  This can also be done several times if you have interesting
-  intermediate states, but if you get there, then we're likely to
-  proceed to the next stage, which is...
-
-* Get a regular account for pushing directly to
-  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
-  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
-  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
-  then be pushed directly to the official repo, but (if you follow these
-  rules) they are still on a branch, and we can still review the
-  branches you want to merge.
-
-* If you get closer to the regular day-to-day development, you'll notice
-  that we generally push small changes as one or a few commits directly
-  to the branch ``default``.  Also, we often collaborate even if we are
-  on other branches, which do not really "belong" to anyone.  At this
-  point you'll need ``hg merge`` and learn how to resolve conflicts that
-  sometimes occur when two people try to push different commits in
-  parallel on the same branch.  But it is likely an issue for later ``:-)``
-
-.. _bitbucket: https://bitbucket.org/
-
-
-Running PyPy's unit tests
--------------------------
-
-PyPy development always was and is still thoroughly test-driven.
-We use the flexible `py.test testing tool`_ which you can `install independently
-<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
-
-The PyPy source tree comes with an inlined version of ``py.test``
-which you can invoke by typing::
-
-    python pytest.py -h
-
-This is usually equivalent to using an installed version::
-
-    py.test -h
-
-If you encounter problems with the installed version
-make sure you have the correct version installed which
-you can find out with the ``--version`` switch.
-
-You will need the `build requirements`_ to run tests successfully, since many of
-them compile little pieces of PyPy and then run the tests inside that minimal
-interpreter
-
-Now on to running some tests.  PyPy has many different test directories
-and you can use shell completion to point at directories or files::
-
-    py.test pypy/interpreter/test/test_pyframe.py
-
-    # or for running tests of a whole subdirectory
-    py.test pypy/interpreter/
-
-See `py.test usage and invocations`_ for some more generic info
-on how you can run tests.
-
-Beware trying to run "all" pypy tests by pointing to the root
-directory or even the top level subdirectory ``pypy``.  It takes
-hours and uses huge amounts of RAM and is not recommended.
-
-To run CPython regression tests you can point to the ``lib-python``
-directory::
-
-    py.test lib-python/2.7/test/test_datetime.py
-
-This will usually take a long time because this will run
-the PyPy Python interpreter on top of CPython.  On the plus
-side, it's usually still faster than doing a full translation
-and running the regression test with the translated PyPy Python
-interpreter.
-
-.. _py.test testing tool: http://pytest.org
-.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
-.. _`build requirements`: build.html#install-build-time-dependencies
-
-Special Introspection Features of the Untranslated Python Interpreter
----------------------------------------------------------------------
-
-If you are interested in the inner workings of the PyPy Python interpreter,
-there are some features of the untranslated Python interpreter that allow you
-to introspect its internals.
-
-
-Interpreter-level console
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To start interpreting Python with PyPy, install a C compiler that is
-supported by distutils and use Python 2.7 or greater to run PyPy::
-
-    cd pypy
-    python bin/pyinteractive.py
-
-After a few seconds (remember: this is running on top of CPython), you should
-be at the PyPy prompt, which is the same as the Python prompt, but with an
-extra ">".
-
-If you press
-<Ctrl-C> on the console you enter the interpreter-level console, a
-usual CPython console.  You can then access internal objects of PyPy
-(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
-prompt with the prefix ``w_``::
-
-    >>>> a = 123
-    >>>> <Ctrl-C>
-    *** Entering interpreter-level console ***
-    >>> w_a
-    W_IntObject(123)
-
-The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
-
-    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
-    >>> <Ctrl-D>
-    *** Leaving interpreter-level console ***
-
-    KeyboardInterrupt
-    >>>> l
-    [1, 'abc']
-
-Note that the prompt of the interpreter-level console is only '>>>' since
-it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
-Linux) or <Ctrl-Z>, <Enter> (under Windows).
-
-Also note that not all modules are available by default in this mode (for
-example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
-``--withmod-...`` command line options.
-
-You may be interested in reading more about the distinction between
-:ref:`interpreter-level and app-level <interpreter-level>`.
-
-pyinteractive.py options
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-To list the PyPy interpreter command line options, type::
-
-    cd pypy
-    python bin/pyinteractive.py --help
-
-pyinteractive.py supports most of the options that CPython supports too (in addition to a
-large amount of options that can be used to customize pyinteractive.py).
-As an example of using PyPy from the command line, you could type::
-
-    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
-
-Alternatively, as with regular Python, you can simply give a
-script name on the command line::
-
-    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
-
-The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
-default almost none of them are, because initializing them takes time.
-If you want anyway to enable all built-in modules, you can use
-``--allworkingmodules``.
-
-See our :doc:`configuration sections <config/index>` for details about what all the commandline
-options do.
-
-
-.. _trace example:
-
-Tracing bytecode and operations on objects
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can use a simple tracing mode to monitor the interpretation of
-bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
-PyPy console::
-
-    >>>> __pytrace__ = 1
-    Tracing enabled
-    >>>> x = 5
-            <module>:           LOAD_CONST    0 (5)
-            <module>:           STORE_NAME    0 (x)
-            <module>:           LOAD_CONST    1 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>> x
-            <module>:           LOAD_NAME    0 (x)
-            <module>:           PRINT_EXPR    0 
-    5
-            <module>:           LOAD_CONST    0 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>>
-
-
-Demos
------
-
-The `example-interpreter`_ repository contains an example interpreter
-written using the RPython translation toolchain.
-
-.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
-
-
-Additional Tools for running (and hacking) PyPy
------------------------------------------------
-
-We use some optional tools for developing PyPy. They are not required to run
-the basic tests or to get an interactive PyPy prompt but they help to
-understand  and debug PyPy especially for the translation process.
-
-
-graphviz & pygame for flow graph viewing (highly recommended)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-graphviz and pygame are both necessary if you
-want to look at generated flow graphs:
-
-	graphviz: http://www.graphviz.org/Download.php
-
-	pygame: http://www.pygame.org/download.shtml
-
-
-py.test and the py lib
-~~~~~~~~~~~~~~~~~~~~~~
-
-The `py.test testing tool`_ drives all our testing needs.
-
-We use the `py library`_ for filesystem path manipulations, terminal
-writing, logging and some other support  functionality.
-
-You don't necessarily need to install these two libraries because
-we also ship them inlined in the PyPy source tree.
-
-.. _py library: http://pylib.readthedocs.org/
-
-
-Getting involved
-----------------
-
-PyPy employs an open development process.  You are invited to join our
-`pypy-dev mailing list`_ or look at the other :ref:`contact
-possibilities <contact>`.  Usually we give out commit rights fairly liberally, so if you
-want to do something with PyPy, you can become a committer. We also run frequent
-coding sprints which are separately announced and often happen around Python
-conferences such as EuroPython or PyCon. Upcoming events are usually announced
-on `the blog`_.
-
-.. _the blog: http://morepypy.blogspot.com
-.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
-
-
-.. _start-reading-sources:
-
-Where to start reading the sources
-----------------------------------
-
-PyPy is made from parts that are relatively independent of each other.
-You should start looking at the part that attracts you most (all paths are
-relative to the PyPy top level directory).  You may look at our :doc:`directory reference <dir-reference>`
-or start off at one of the following points:
-
-*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
-   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
-   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
-   function objects and argument passing in :source:`pypy/interpreter/function.py`
-   and :source:`pypy/interpreter/argument.py`, the object space interface
-   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
-   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
-   Core types supporting the bytecode interpreter are defined in :source:`pypy/interpreter/typedef.py`.
-
-*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
-   and grammar files that allow it to parse the syntax of various Python
-   versions. Once the grammar has been processed, the parser can be
-   translated by the above machinery into efficient code.
-
-*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
-   contains a modified version of the compiler package from CPython
-   that fixes some bugs and is translatable.
-
-*  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
-   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
-   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
-   as a first approximation.  (Some types have multiple implementations.)
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
deleted file mode 100644
--- a/pypy/doc/how-to-contribute.rst
+++ /dev/null
@@ -1,93 +0,0 @@
-How to contribute to PyPy
-=========================
-
-This page describes how to contribute to the PyPy project. The first thing
-to remember is that PyPy project is very different than most projects out there.
-It's also different from a classic compiler project, so academic courses
-about compilers often don't apply or lead in the wrong direction.
-
-
-Don't just hack
----------------
-
-The first and most important rule how not to contribute to PyPy is
-"just hacking". This won't work. There are two major reasons why not
--- build times are large and PyPy has very thick layer separation which
-make it harder to "just hack a feature".
-
-
-Test driven development
------------------------
-
-Instead, we practice a lot of test driven development. This is partly because
-of very high quality requirements for compilers and partly because there is
-simply no other way to get around such complex project, that will keep you sane.
-There are probably people out there who are smart enough not to need it, we're
-not one of those. You may consider familiarizing yourself with `pytest`_,
-since this is a tool we use for tests.
-This leads to the next issue:
-
-.. _pytest: http://pytest.org/
-
-
-Layers
-------
-
-PyPy has layers. Just like Ogres or onions.
-Those layers help us keep the respective parts separated enough
-to be worked on independently and make the complexity manageable. This is,
-again, just a sanity requirement for such a complex project. For example writing
-a new optimization for the JIT usually does **not** involve touching a Python
-interpreter at all or the JIT assembler backend or the garbage collector.
-Instead it requires writing small tests in
-``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
-After that, you can just compile PyPy and things should just work.
-
-The short list of layers for further reading. For each of those layers, a good
-entry point is a test subdirectory in respective directories. It usually
-describes (better or worse) the interfaces between the submodules. For the
-``pypy`` subdirectory, most tests are small snippets of python programs that
-check for correctness (calls ``AppTestXxx``) that will call the appropriate
-part of the interpreter. For the ``rpython`` directory, most tests are small
-RPython interpreters that perform certain tasks. To see how they translate
-to low-level graphs, run them with ``--view``. To see small interpreters
-with a JIT compiler, use ``--viewloops`` option.
-
-* **python interpreter** - it's the part implemented in the ``pypy/`` directory.
-  It's implemented in RPython, which is a high level static language with
-  classes, garbage collection, just-in-time compiler generation and the ability
-  to call C. A cool part about it is that it can be run untranslated, so all
-  the tests are runnable without translating PyPy.
-
-  **interpreter** contains the interpreter core
-
-  **objspace** contains implementations of various objects exported to
-  the Python layer
-
-  **module** directory contains extension modules written in RPython
-

From pypy.commits at gmail.com  Thu May 17 08:03:25 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 17 May 2018 05:03:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: merge py3.6
Message-ID: <5afd6f8d.1c69fb81.30369.b306@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94610:df08b57a6b4b
Date: 2018-05-17 14:02 +0200
http://bitbucket.org/pypy/pypy/changeset/df08b57a6b4b/

Log:	merge py3.6

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -423,6 +423,7 @@
     RegrTest('test_sys_setprofile.py', core=True),
     RegrTest('test_sys_settrace.py', core=True),
     RegrTest('test_sysconfig.py'),
+    RegrTest('test_sysconfig_pypy.py'),
     RegrTest('test_syslog.py'),
     RegrTest('test_tarfile.py'),
     RegrTest('test_tcl.py'),
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -224,7 +224,6 @@
         if server_hostname:
             self.server_hostname = server_hostname.decode('idna', 'strict')
 
-        lib.ERR_get_state()
         lib.ERR_clear_error()
         self.ssl = ssl = ffi.gc(lib.SSL_new(ctx), lib.SSL_free)
 
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -12,3 +12,7 @@
 .. branch: py3.5-user-site-impl
 
 Use implementation-specific site directories in sysconfig like in Python2
+
+.. branch: alex_gaynor/remove-an-unneeded-call-into-openssl-th-1526429141011
+
+Remove an unneeded call into OpenSSL, from cpython https://github.com/python/cpython/pull/6887
diff --git a/pypy/interpreter/astcompiler/test/test_validate.py b/pypy/interpreter/astcompiler/test/test_validate.py
--- a/pypy/interpreter/astcompiler/test/test_validate.py
+++ b/pypy/interpreter/astcompiler/test/test_validate.py
@@ -1,4 +1,5 @@
 import os
+from pytest import raises
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.astcompiler import ast
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -375,7 +375,7 @@
         child.expect('>>>')
 
     def test_atexit(self):
-        skip("Python3 atexit is a builtin module")
+        py.test.skip("Python3 atexit is a builtin module")
         child = self.spawn([])
         child.expect('>>> ')
         child.sendline('def f(): print("foobye")')
@@ -525,9 +525,9 @@
 
     def test_options_i_m(self, monkeypatch):
         if sys.platform == "win32":
-            skip("close_fds is not supported on Windows platforms")
+            py.test.skip("close_fds is not supported on Windows platforms")
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'mymodule.py')
         p = os.path.abspath(p)
         monkeypatch.chdir(os.path.dirname(app_main))
@@ -557,7 +557,7 @@
 
     def test_options_u_i(self):
         if sys.platform == "win32":
-            skip("close_fds is not supported on Windows platforms")
+            py.test.skip("close_fds is not supported on Windows platforms")
         import subprocess, select, os
         pipe = subprocess.Popen([get_python3(), app_main, "-u", "-i"],
                                 stdout=subprocess.PIPE,
@@ -614,7 +614,7 @@
             del os.environ['PYTHONINSPECT_']
 
     def test_stdout_flushes_before_stdin_blocks(self):
-        skip("Python3 does not implement this behavior")
+        py.test.skip("Python3 does not implement this behavior")
         # This doesn't really test app_main.py, but a behavior that
         # can only be checked on top of py.py with pexpect.
         path = getscript("""
@@ -632,7 +632,7 @@
 
     def test_no_space_before_argument(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         child = self.spawn(['-cprint("hel" + "lo")'])
         child.expect('hello')
 
@@ -753,7 +753,7 @@
 
     def test_option_m(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'mymodule.py')
         p = os.path.abspath(p)
         monkeypatch.chdir(os.path.dirname(app_main))
@@ -767,7 +767,7 @@
 
     def test_option_m_package(self, monkeypatch):
         if not hasattr(runpy, '_run_module_as_main'):
-            skip("requires CPython >= 2.6")
+            py.test.skip("requires CPython >= 2.6")
         p = os.path.join(os.path.realpath(os.path.dirname(__file__)),
                          'mypackage', '__main__.py')
         p = os.path.abspath(p)
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -168,6 +168,222 @@
     return decode_utf8(space, string, allow_surrogates=True)
 
 # ____________________________________________________________
+# utf-16
+
+def str_decode_utf_16(s, size, errors, final=True,
+                           errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "native",
+                                                         'utf-16-' + BYTEORDER2)
+    return result, length
+
+def str_decode_utf_16_be(s, size, errors, final=True,
+                              errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "big",
+                                                         'utf-16-be')
+    return result, length
+
+def str_decode_utf_16_le(s, size, errors, final=True,
+                              errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "little",
+                                                         'utf-16-le')
+    return result, length
+
+def str_decode_utf_16_helper(s, size, errors, final=True,
+                             errorhandler=None,
+                             byteorder="native",
+                             public_encoding_name='utf16'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+    bo = 0
+
+    if BYTEORDER == 'little':
+        ihi = 1
+        ilo = 0
+    else:
+        ihi = 0
+        ilo = 1
+
+    #  Check for BOM marks (U+FEFF) in the input and adjust current
+    #  byte order setting accordingly. In native mode, the leading BOM
+    #  mark is skipped, in all other modes, it is copied to the output
+    #  stream as-is (giving a ZWNBSP character).
+    pos = 0
+    if byteorder == 'native':
+        if size >= 2:
+            bom = (ord(s[ihi]) << 8) | ord(s[ilo])
+            if BYTEORDER == 'little':
+                if bom == 0xFEFF:
+                    pos += 2
+                    bo = -1
+                elif bom == 0xFFFE:
+                    pos += 2
+                    bo = 1
+            else:
+                if bom == 0xFEFF:
+                    pos += 2
+                    bo = 1
+                elif bom == 0xFFFE:
+                    pos += 2
+                    bo = -1
+    elif byteorder == 'little':
+        bo = -1
+    else:
+        bo = 1
+    if size == 0:
+        return u'', 0, bo
+    if bo == -1:
+        # force little endian
+        ihi = 1
+        ilo = 0
+
+    elif bo == 1:
+        # force big endian
+        ihi = 0
+        ilo = 1
+
+    result = UnicodeBuilder(size // 2)
+
+    #XXX I think the errors are not correctly handled here
+    while pos < size:
+        # remaining bytes at the end? (size should be even)
+        if len(s) - pos < 2:
+            if not final:
+                break
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "truncated data",
+                                  s, pos, len(s))
+            result.append(r)
+            if len(s) - pos < 2:
+                break
+        ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo])
+        pos += 2
+        if ch < 0xD800 or ch > 0xDFFF:
+            result.append(unichr(ch))
+            continue
+        # UTF-16 code pair:
+        if len(s) - pos < 2:
+            pos -= 2
+            if not final:
+                break
+            errmsg = "unexpected end of data"
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  errmsg, s, pos, len(s))
+            result.append(r)
+            if len(s) - pos < 2:
+                break
+        elif 0xD800 <= ch <= 0xDBFF:
+            ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo])
+            pos += 2
+            if 0xDC00 <= ch2 <= 0xDFFF:
+                if MAXUNICODE < 65536:
+                    result.append(unichr(ch))
+                    result.append(unichr(ch2))
+                else:
+                    result.append(UNICHR((((ch & 0x3FF)<<10) |
+                                           (ch2 & 0x3FF)) + 0x10000))
+                continue
+            else:
+                r, pos = errorhandler(errors, public_encoding_name,
+                                      "illegal UTF-16 surrogate",
+                                      s, pos - 4, pos - 2)
+                result.append(r)
+        else:
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "illegal encoding",
+                                  s, pos - 2, pos)
+            result.append(r)
+    return result.build(), pos, bo
+
+def _STORECHAR(result, CH, byteorder):
+    hi = chr(((CH) >> 8) & 0xff)
+    lo = chr((CH) & 0xff)
+    if byteorder == 'little':
+        result.append(lo)
+        result.append(hi)
+    else:
+        result.append(hi)
+        result.append(lo)
+
+def unicode_encode_utf_16_helper(s, size, errors,
+                                 errorhandler=None,
+                                 allow_surrogates=True,
+                                 byteorder='little',
+                                 public_encoding_name='utf16'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_encode
+    if size == 0:
+        if byteorder == 'native':
+            result = StringBuilder(2)
+            _STORECHAR(result, 0xFEFF, BYTEORDER)
+            return result.build()
+        return ""
+
+    result = StringBuilder(size * 2 + 2)
+    if byteorder == 'native':
+        _STORECHAR(result, 0xFEFF, BYTEORDER)
+        byteorder = BYTEORDER
+
+    pos = 0
+    while pos < size:
+        ch = ord(s[pos])
+        pos += 1
+
+        if ch < 0xD800:
+            _STORECHAR(result, ch, byteorder)
+        elif ch >= 0x10000:
+            _STORECHAR(result, 0xD800 | ((ch-0x10000) >> 10), byteorder)
+            _STORECHAR(result, 0xDC00 | ((ch-0x10000) & 0x3FF), byteorder)
+        elif ch >= 0xE000 or allow_surrogates:
+            _STORECHAR(result, ch, byteorder)
+        else:
+            ru, rs, pos = errorhandler(errors, public_encoding_name,
+                                       'surrogates not allowed',
+                                       s, pos-1, pos)
+            if rs is not None:
+                # py3k only
+                if len(rs) % 2 != 0:
+                    errorhandler('strict', public_encoding_name,
+                                 'surrogates not allowed',
+                                 s, pos-1, pos)
+                result.append(rs)
+                continue
+            for ch in ru:
+                if ord(ch) < 0xD800:
+                    _STORECHAR(result, ord(ch), byteorder)
+                else:
+                    errorhandler('strict', public_encoding_name,
+                                 'surrogates not allowed',
+                                 s, pos-1, pos)
+            continue
+
+    return result.build()
+
+def unicode_encode_utf_16(s, size, errors,
+                          errorhandler=None,
+                          allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "native",
+                                        'utf-16-' + BYTEORDER2)
+
+def unicode_encode_utf_16_be(s, size, errors,
+                             errorhandler=None,
+                             allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "big",
+                                        'utf-16-be')
+
+def unicode_encode_utf_16_le(s, size, errors,
+                             errorhandler=None,
+                             allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "little",
+                                        'utf-16-le')
+
+
+# ____________________________________________________________
 # utf-32
 
 def str_decode_utf_32(s, size, errors, final=True,
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -621,8 +621,6 @@
     try:
         func = getattr(unicodehelper, impl_name)
     except AttributeError:
-        if hasattr(runicode, 'py3k_' + impl_name):
-            impl_name = 'py3k_' + impl_name
         func = getattr(runicode, impl_name)
     return func
 
diff --git a/pypy/objspace/std/test/test_iterobject.py b/pypy/objspace/std/test/test_iterobject.py
--- a/pypy/objspace/std/test/test_iterobject.py
+++ b/pypy/objspace/std/test/test_iterobject.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.objspace.std.iterobject import W_SeqIterObject
 from pypy.interpreter.error import OperationError
 
@@ -11,8 +12,8 @@
         self.body0(w_iter)
 
     def body0(self, w_iter):
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
 
     def test_iter(self):
         w = self.space.wrap
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -120,8 +120,8 @@
         assert self.space.eq_w(self.space.next(w_iter), w(5))
         assert self.space.eq_w(self.space.next(w_iter), w(3))
         assert self.space.eq_w(self.space.next(w_iter), w(99))
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        py.test.raises(OperationError, self.space.next, w_iter)
+        py.test.raises(OperationError, self.space.next, w_iter)
 
     def test_contains(self):
         w = self.space.wrap
diff --git a/pypy/objspace/std/test/test_setstrategies.py b/pypy/objspace/std/test/test_setstrategies.py
--- a/pypy/objspace/std/test/test_setstrategies.py
+++ b/pypy/objspace/std/test/test_setstrategies.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.objspace.std.setobject import W_SetObject
 from pypy.objspace.std.setobject import (
     BytesIteratorImplementation, BytesSetStrategy, EmptySetStrategy,
@@ -58,7 +59,7 @@
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         s2 = W_SetObject(self.space, self.wrapped([4,5, "six", "seven"]))
         s3 = s1.intersect(s2)
-        skip("for now intersection with ObjectStrategy always results in another ObjectStrategy")
+        pytest.skip("for now intersection with ObjectStrategy always results in another ObjectStrategy")
         assert s3.strategy is self.space.fromcache(IntegerSetStrategy)
 
     def test_clear(self):
@@ -93,7 +94,7 @@
 
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         s1.descr_discard(self.space, self.space.wrap("five"))
-        skip("currently not supported")
+        pytest.skip("currently not supported")
         assert s1.strategy is self.space.fromcache(IntegerSetStrategy)
 
         set_discard__Set_ANY(self.space, s1, self.space.wrap(FakeInt(5)))
@@ -112,7 +113,7 @@
 
         s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
         assert not s1.has_key(self.space.wrap("five"))
-        skip("currently not supported")
+        pytest.skip("currently not supported")
         assert s1.strategy is self.space.fromcache(IntegerSetStrategy)
 
         assert s1.has_key(self.space.wrap(FakeInt(2)))
diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py
--- a/pypy/objspace/std/test/test_tupleobject.py
+++ b/pypy/objspace/std/test/test_tupleobject.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std.tupleobject import W_TupleObject
 
@@ -42,8 +43,8 @@
         assert self.space.eq_w(self.space.next(w_iter), w(5))
         assert self.space.eq_w(self.space.next(w_iter), w(3))
         assert self.space.eq_w(self.space.next(w_iter), w(99))
-        raises(OperationError, self.space.next, w_iter)
-        raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
+        pytest.raises(OperationError, self.space.next, w_iter)
 
     def test_contains(self):
         w = self.space.wrap

From pypy.commits at gmail.com  Thu May 17 12:57:26 2018
From: pypy.commits at gmail.com (wlav)
Date: Thu, 17 May 2018 09:57:26 -0700 (PDT)
Subject: [pypy-commit] pypy cppyy-packaging: add support for unsigned char
 arrays and general cleanup of array binding code + tests
Message-ID: <5afdb476.1c69fb81.7a3c9.4d43@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r94611:eb16b0aff1e3
Date: 2018-05-17 09:39 -0700
http://bitbucket.org/pypy/pypy/changeset/eb16b0aff1e3/

Log:	add support for unsigned char arrays and general cleanup of array
	binding code + tests

diff --git a/pypy/module/_cppyy/capi/__init__.py b/pypy/module/_cppyy/capi/__init__.py
--- a/pypy/module/_cppyy/capi/__init__.py
+++ b/pypy/module/_cppyy/capi/__init__.py
@@ -11,6 +11,3 @@
     assert lltype.typeOf(ptr) == C_OBJECT
     address = rffi.cast(rffi.CCHARP, ptr)
     return rffi.cast(C_OBJECT, lltype.direct_ptradd(address, offset))
-
-def exchange_address(ptr, cif_descr, index):
-    return rffi.ptradd(ptr, cif_descr.exchange_args[index])
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -7,7 +7,7 @@
 from rpython.rlib import rfloat, rawrefcount
 
 from pypy.module._rawffi.interp_rawffi import letter2tp
-from pypy.module._rawffi.array import W_Array, W_ArrayInstance
+from pypy.module._rawffi.array import W_ArrayInstance
 
 from pypy.module._cppyy import helper, capi, ffitypes
 
@@ -130,20 +130,6 @@
         pass
 
 
-class ArrayCache(object):
-    def __init__(self, space):
-        self.space = space
-    def __getattr__(self, name):
-        if name.startswith('array_'):
-            typecode = name[len('array_'):]
-            arr = self.space.interp_w(W_Array, letter2tp(self.space, typecode))
-            setattr(self, name, arr)
-            return arr
-        raise AttributeError(name)
-
-    def _freeze_(self):
-        return True
-
 class ArrayTypeConverterMixin(object):
     _mixin_ = True
     _immutable_fields_ = ['size']
@@ -162,9 +148,7 @@
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONG, address_value)
-        cache = space.fromcache(ArrayCache)
-        arr = getattr(cache, 'array_' + self.typecode)
-        return arr.fromaddress(space, address, self.size)
+        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address)
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy the full array (uses byte copy for now)
@@ -205,17 +189,15 @@
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONGP, address_value)
-        cache = space.fromcache(ArrayCache)
-        arr = getattr(cache, 'array_' + self.typecode)
-        return arr.fromaddress(space, address[0], self.size)
+        return W_ArrayInstance(space, letter2tp(space, self.typecode), self.size, address[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         # copy only the pointer value
         rawobject = get_rawobject_nonnull(space, w_obj)
-        byteptr = rffi.cast(rffi.CCHARPP, capi.direct_ptradd(rawobject, offset))
+        byteptr = rffi.cast(rffi.VOIDPP, capi.direct_ptradd(rawobject, offset))
         buf = space.getarg_w('s*', w_value)
         try:
-            byteptr[0] = buf.get_raw_address()
+            byteptr[0] = rffi.cast(rffi.VOIDP, buf.get_raw_address())
         except ValueError:
             raise oefmt(space.w_TypeError,
                         "raw buffer interface not supported")
@@ -337,6 +319,10 @@
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         address[0] = self._unwrap_object(space, w_value)
 
+
+class UCharConverter(ffitypes.typeid(rffi.UCHAR), CharConverter):
+    pass
+
 class FloatConverter(ffitypes.typeid(rffi.FLOAT), FloatTypeConverterMixin, TypeConverter):
     _immutable_fields_ = ['default']
 
@@ -449,12 +435,12 @@
         # returned as a long value for the address (INTPTR_T is not proper
         # per se, but rffi does not come with a PTRDIFF_T)
         address = self._get_raw_address(space, w_obj, offset)
-        ptrval = rffi.cast(rffi.ULONG, rffi.cast(rffi.VOIDPP, address)[0])
-        if ptrval == 0:
+        ptrval = rffi.cast(rffi.ULONGP, address)[0]
+        if ptrval == rffi.cast(rffi.ULONG, 0):
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
-        arr = space.interp_w(W_Array, letter2tp(space, 'P'))
-        return arr.fromaddress(space, ptrval, sys.maxint)
+        shape = letter2tp(space, 'P')
+        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
@@ -797,6 +783,7 @@
 
 _converters["bool"]                     = BoolConverter
 _converters["char"]                     = CharConverter
+_converters["unsigned char"]            = UCharConverter
 _converters["float"]                    = FloatConverter
 _converters["const float&"]             = ConstFloatRefConverter
 _converters["double"]                   = DoubleConverter
@@ -886,6 +873,7 @@
     "NOT_RPYTHON"
     array_info = (
         ('b', rffi.sizeof(rffi.UCHAR),      ("bool",)),    # is debatable, but works ...
+        ('B', rffi.sizeof(rffi.UCHAR),      ("unsigned char",)),
         ('h', rffi.sizeof(rffi.SHORT),      ("short int", "short")),
         ('H', rffi.sizeof(rffi.USHORT),     ("unsigned short int", "unsigned short")),
         ('i', rffi.sizeof(rffi.INT),        ("int",)),
@@ -901,9 +889,11 @@
 
     for tcode, tsize, names in array_info:
         class ArrayConverter(ArrayTypeConverterMixin, TypeConverter):
+            _immutable_fields_ = ['typecode', 'typesize']
             typecode = tcode
             typesize = tsize
         class PtrConverter(PtrTypeConverterMixin, TypeConverter):
+            _immutable_fields_ = ['typecode', 'typesize']
             typecode = tcode
             typesize = tsize
         for name in names:
@@ -919,7 +909,6 @@
 def _add_aliased_converters():
     "NOT_RPYTHON"
     aliases = (
-        ("char",                            "unsigned char"), # TODO: check
         ("char",                            "signed char"),   # TODO: check
         ("const char*",                     "char*"),
 
diff --git a/pypy/module/_cppyy/executor.py b/pypy/module/_cppyy/executor.py
--- a/pypy/module/_cppyy/executor.py
+++ b/pypy/module/_cppyy/executor.py
@@ -5,7 +5,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib import jit_libffi
 
-from pypy.module._rawffi.interp_rawffi import unpack_simple_shape
+from pypy.module._rawffi.interp_rawffi import letter2tp
 from pypy.module._rawffi.array import W_Array, W_ArrayInstance
 
 from pypy.module._cppyy import helper, capi, ffitypes
@@ -56,11 +56,11 @@
             raise NotImplementedError
         lresult = capi.c_call_l(space, cppmethod, cppthis, num_args, args)
         ptrval = rffi.cast(rffi.ULONG, lresult)
-        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.newtext(self.typecode)))
-        if ptrval == 0:
+        if ptrval == rffi.cast(rffi.ULONG, 0):
             from pypy.module._cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
-        return arr.fromaddress(space, ptrval, sys.maxint)
+        shape = letter2tp(space, self.typecode)
+        return W_ArrayInstance(space, shape, sys.maxint/shape.size, ptrval)
 
 
 class VoidExecutor(FunctionExecutor):
diff --git a/pypy/module/_cppyy/ffitypes.py b/pypy/module/_cppyy/ffitypes.py
--- a/pypy/module/_cppyy/ffitypes.py
+++ b/pypy/module/_cppyy/ffitypes.py
@@ -74,6 +74,37 @@
         # allow int to pass to char and make sure that str is of length 1
         if space.isinstance_w(w_value, space.w_int):
             ival = space.c_int_w(w_value)
+            if ival < -128 or 127 < ival:
+                raise oefmt(space.w_ValueError, "char arg not in range(-128,128)")
+
+            value = rffi.cast(rffi.CHAR, space.c_int_w(w_value))
+        else:
+            value = space.text_w(w_value)
+            if len(value) != 1:
+                raise oefmt(space.w_ValueError,
+                        "char expected, got string of size %d", len(value))
+
+        value = rffi.cast(rffi.CHAR, value[0])
+        return value     # turn it into a "char" to the annotator
+
+    def cffi_type(self, space):
+        state = space.fromcache(State)
+        return state.c_char
+
+class UCharTypeMixin(object):
+    _mixin_     = True
+    _immutable_fields_ = ['c_type', 'c_ptrtype']
+
+    c_type      = rffi.UCHAR
+    c_ptrtype   = rffi.CCHARP           # there's no such thing as rffi.UCHARP
+
+    def _wrap_object(self, space, obj):
+        return space.newbytes(obj)
+
+    def _unwrap_object(self, space, w_value):
+        # allow int to pass to char and make sure that str is of length 1
+        if space.isinstance_w(w_value, space.w_int):
+            ival = space.c_int_w(w_value)
             if ival < 0 or 256 <= ival:
                 raise oefmt(space.w_ValueError, "char arg not in range(256)")
 
@@ -277,6 +308,7 @@
     "NOT_RPYTHON"
     if c_type == bool:            return BoolTypeMixin
     if c_type == rffi.CHAR:       return CharTypeMixin
+    if c_type == rffi.UCHAR:      return UCharTypeMixin
     if c_type == rffi.SHORT:      return ShortTypeMixin
     if c_type == rffi.USHORT:     return UShortTypeMixin
     if c_type == rffi.INT:        return IntTypeMixin
diff --git a/pypy/module/_cppyy/interp_cppyy.py b/pypy/module/_cppyy/interp_cppyy.py
--- a/pypy/module/_cppyy/interp_cppyy.py
+++ b/pypy/module/_cppyy/interp_cppyy.py
@@ -267,11 +267,12 @@
     def do_fast_call(self, cppthis, args_w, call_local):
         if self.cif_descr == lltype.nullptr(jit_libffi.CIF_DESCRIPTION):
             raise FastCallNotPossible
+        jit.promote(self)
         cif_descr = self.cif_descr
         buffer = lltype.malloc(rffi.CCHARP.TO, cif_descr.exchange_size, flavor='raw')
         try:
             # this pointer
-            data = capi.exchange_address(buffer, cif_descr, 0)
+            data = rffi.ptradd(buffer, cif_descr.exchange_args[0])
             x = rffi.cast(rffi.LONGP, data)       # LONGP needed for test_zjit.py
             x[0] = rffi.cast(rffi.LONG, cppthis)
 
@@ -280,11 +281,11 @@
             for i in range(len(args_w)):
                 conv = self.converters[i]
                 w_arg = args_w[i]
-                data = capi.exchange_address(buffer, cif_descr, i+1)
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[i+1])
                 conv.convert_argument_libffi(self.space, w_arg, data, call_local)
             for j in range(i+1, len(self.arg_defs)):
                 conv = self.converters[j]
-                data = capi.exchange_address(buffer, cif_descr, j+1)
+                data = rffi.ptradd(buffer, cif_descr.exchange_args[j+1])
                 conv.default_argument_libffi(self.space, data)
 
             assert self._funcaddr
@@ -296,7 +297,7 @@
         return w_res
 
     # from ctypefunc; have my own version for annotater purposes and to disable
-    # memory tracking (method live time is longer than the tests)
+    # memory tracking (method life time is longer than the tests)
     @jit.dont_look_inside
     def _rawallocate(self, builder):
         builder.space = self.space
@@ -309,7 +310,7 @@
         # allocate the buffer
         if we_are_translated():
             rawmem = lltype.malloc(rffi.CCHARP.TO, builder.nb_bytes,
-                                   flavor='raw', track_allocation=False)
+                                   flavor='raw')
             rawmem = rffi.cast(jit_libffi.CIF_DESCRIPTION_P, rawmem)
         else:
             # gross overestimation of the length below, but too bad
@@ -352,7 +353,7 @@
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
         funcaddr = capi.c_function_address_from_index(self.space, self.scope, self.index)
-        if funcaddr and cppthis:      # methods only for now
+        if funcaddr and cppthis:      # TODO: methods only for now
             state = self.space.fromcache(ffitypes.State)
 
             # argument type specification (incl. cppthis)
diff --git a/pypy/module/_cppyy/test/datatypes.cxx b/pypy/module/_cppyy/test/datatypes.cxx
--- a/pypy/module/_cppyy/test/datatypes.cxx
+++ b/pypy/module/_cppyy/test/datatypes.cxx
@@ -29,6 +29,7 @@
     m_voidp   = (void*)0;
 
     m_bool_array2   = new bool[N];
+    m_uchar_array2  = new unsigned char[N];
     m_short_array2  = new short[N];
     m_ushort_array2 = new unsigned short[N];
     m_int_array2    = new int[N];
@@ -42,6 +43,8 @@
     for (int i = 0; i < N; ++i) {
         m_bool_array[i]    =  bool(i%2);
         m_bool_array2[i]   =  bool((i+1)%2);
+        m_uchar_array[i]   =   1u*i;
+        m_uchar_array2[i]  =   2u*i;
         m_short_array[i]   =  -1*i;
         m_short_array2[i]  =  -2*i;
         m_ushort_array[i]  =   3u*i;
@@ -114,6 +117,8 @@
 
 bool*           CppyyTestData::get_bool_array()    { return m_bool_array; }
 bool*           CppyyTestData::get_bool_array2()   { return m_bool_array2; }
+unsigned char*  CppyyTestData::get_uchar_array()   { return m_uchar_array; }
+unsigned char*  CppyyTestData::get_uchar_array2()  { return m_uchar_array2; }
 short*          CppyyTestData::get_short_array()   { return m_short_array; }
 short*          CppyyTestData::get_short_array2()  { return m_short_array2; }
 unsigned short* CppyyTestData::get_ushort_array()  { return m_ushort_array; }
@@ -233,6 +238,7 @@
 void CppyyTestData::set_enum_cr(const EWhat& w)                  { m_enum    = w; }
 
 //- passers -----------------------------------------------------------------
+unsigned char*  CppyyTestData::pass_array(unsigned char* a)  { return a; }
 short*          CppyyTestData::pass_array(short* a)          { return a; }
 unsigned short* CppyyTestData::pass_array(unsigned short* a) { return a; }
 int*            CppyyTestData::pass_array(int* a)            { return a; }
diff --git a/pypy/module/_cppyy/test/datatypes.h b/pypy/module/_cppyy/test/datatypes.h
--- a/pypy/module/_cppyy/test/datatypes.h
+++ b/pypy/module/_cppyy/test/datatypes.h
@@ -99,6 +99,8 @@
 
     bool*           get_bool_array();
     bool*           get_bool_array2();
+    unsigned char*  get_uchar_array();
+    unsigned char*  get_uchar_array2();
     short*          get_short_array();
     short*          get_short_array2();
     unsigned short* get_ushort_array();
@@ -217,6 +219,7 @@
     void set_enum_cr(const EWhat&);
 
 // passers
+    unsigned char*  pass_array(unsigned char*);
     short*          pass_array(short*);
     unsigned short* pass_array(unsigned short*);
     int*            pass_array(int*);
@@ -226,6 +229,7 @@
     float*          pass_array(float*);
     double*         pass_array(double*);
 
+    unsigned char*  pass_void_array_B(void* a) { return pass_array((unsigned char*)a); }
     short*          pass_void_array_h(void* a) { return pass_array((short*)a); }
     unsigned short* pass_void_array_H(void* a) { return pass_array((unsigned short*)a); }
     int*            pass_void_array_i(void* a) { return pass_array((int*)a); }
@@ -265,6 +269,8 @@
 // array types
     bool            m_bool_array[N];
     bool*           m_bool_array2;
+    unsigned char   m_uchar_array[N];
+    unsigned char*  m_uchar_array2;
     short           m_short_array[N];
     short*          m_short_array2;
     unsigned short  m_ushort_array[N];
diff --git a/pypy/module/_cppyy/test/test_datatypes.py b/pypy/module/_cppyy/test/test_datatypes.py
--- a/pypy/module/_cppyy/test/test_datatypes.py
+++ b/pypy/module/_cppyy/test/test_datatypes.py
@@ -73,8 +73,8 @@
             assert c.get_bool_array2()[i]   ==   bool((i+1)%2)
 
         # reading of integer array types
-        names = [ 'short', 'ushort',    'int', 'uint',    'long',  'ulong']
-        alpha = [(-1, -2),   (3, 4), (-5, -6), (7, 8), (-9, -10), (11, 12)]
+        names = ['uchar',  'short', 'ushort',    'int', 'uint',    'long',  'ulong']
+        alpha = [ (1, 2), (-1, -2),   (3, 4), (-5, -6), (7, 8), (-9, -10), (11, 12)]
         for j in range(self.N):
             assert getattr(c, 'm_%s_array'    % names[i])[i]   == alpha[i][0]*i
             assert getattr(c, 'get_%s_array'  % names[i])()[i] == alpha[i][0]*i
@@ -89,6 +89,7 @@
             assert round(c.m_double_array2[k] + 16.*k, 8) == 0
 
         # out-of-bounds checks
+        raises(IndexError, c.m_uchar_array.__getitem__,  self.N)
         raises(IndexError, c.m_short_array.__getitem__,  self.N)
         raises(IndexError, c.m_ushort_array.__getitem__, self.N)
         raises(IndexError, c.m_int_array.__getitem__,    self.N)
@@ -177,10 +178,10 @@
         c.destroy_arrays()
 
         # integer arrays
-        names = ['short', 'ushort', 'int', 'uint', 'long', 'ulong']
+        names = ['uchar', 'short', 'ushort', 'int', 'uint', 'long', 'ulong']
         import array
         a = range(self.N)
-        atypes = ['h', 'H', 'i', 'I', 'l', 'L' ]
+        atypes = ['B', 'h', 'H', 'i', 'I', 'l', 'L' ]
         for j in range(len(names)):
             b = array.array(atypes[j], a)
             setattr(c, 'm_'+names[j]+'_array', b)     # buffer copies
@@ -188,7 +189,7 @@
                 assert eval('c.m_%s_array[i]' % names[j]) == b[i]
 
             setattr(c, 'm_'+names[j]+'_array2', b)    # pointer copies
-            b[i] = 28
+            b[3] = 28
             for i in range(self.N):
                 assert eval('c.m_%s_array2[i]' % names[j]) == b[i]
 
@@ -681,6 +682,7 @@
 
         c = CppyyTestData()
         for func in ['get_bool_array',   'get_bool_array2',
+                     'get_uchar_array',   'get_uchar_array2',
                      'get_ushort_array', 'get_ushort_array2',
                      'get_int_array',    'get_int_array2',
                      'get_uint_array',   'get_uint_array2',
diff --git a/pypy/module/_cppyy/test/test_zjit.py b/pypy/module/_cppyy/test/test_zjit.py
--- a/pypy/module/_cppyy/test/test_zjit.py
+++ b/pypy/module/_cppyy/test/test_zjit.py
@@ -17,19 +17,6 @@
 # (note that the module is not otherwise used in the test itself)
 import pypy.module.cpyext
 
-# change capi's direct_ptradd and exchange_address to being jit-opaque
- at jit.dont_look_inside
-def _opaque_direct_ptradd(ptr, offset):
-    address = rffi.cast(rffi.CCHARP, ptr)
-    return rffi.cast(capi.C_OBJECT, lltype.direct_ptradd(address, offset))
-capi.direct_ptradd = _opaque_direct_ptradd
-
- at jit.dont_look_inside
-def _opaque_exchange_address(ptr, cif_descr, index):
-    offset = rffi.cast(rffi.LONG, cif_descr.exchange_args[index])
-    return rffi.ptradd(ptr, offset)
-capi.exchange_address = _opaque_exchange_address
-
 # add missing alt_errno (??)
 def get_tlobj(self):
     try:

From pypy.commits at gmail.com  Fri May 18 04:11:16 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 18 May 2018 01:11:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: cleanup in dict implementation: don't
 use has_* flags, put fallback
Message-ID: <5afe8aa4.1c69fb81.7a3c9.068a@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94612:353420330f35
Date: 2018-05-18 10:10 +0200
http://bitbucket.org/pypy/pypy/changeset/353420330f35/

Log:	cleanup in dict implementation: don't use has_* flags, put fallback
	implementations into base methods instead

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -270,13 +270,7 @@
         """Not exposed directly to app-level, but via __pypy__.reversed_dict().
         """
         strategy = self.get_strategy()
-        if strategy.has_iterreversed:
-            it = strategy.iterreversed(self)
-            return W_DictMultiIterKeysObject(space, it)
-        else:
-            # fall-back
-            w_keys = self.w_keys()
-            return space.call_method(w_keys, '__reversed__')
+        return strategy.iterreversed(self)
 
     def nondescr_delitem_if_value_is(self, space, w_key, w_value):
         """Not exposed directly to app-level, but used by
@@ -306,32 +300,7 @@
         """Not exposed directly to app-level, but via __pypy__.move_to_end().
         """
         strategy = self.get_strategy()
-        if strategy.has_move_to_end:
-            strategy.move_to_end(self, w_key, last_flag)
-        else:
-            # fall-back
-            w_value = self.getitem(w_key)
-            if w_value is None:
-                space.raise_key_error(w_key)
-            else:
-                self.internal_delitem(w_key)
-                if last_flag:
-                    self.setitem(w_key, w_value)
-                else:
-                    # *very slow* fall-back
-                    keys_w = []
-                    values_w = []
-                    iteratorimplementation = self.iteritems()
-                    while True:
-                        w_k, w_v = iteratorimplementation.next_item()
-                        if w_k is None:
-                            break
-                        keys_w.append(w_k)
-                        values_w.append(w_v)
-                    self.clear()
-                    self.setitem(w_key, w_value)
-                    for i in range(len(keys_w)):
-                        self.setitem(keys_w[i], values_w[i])
+        strategy.move_to_end(self, w_key, last_flag)
 
     def nondescr_popitem_first(self, space):
         """Not exposed directly to app-level, but via __pypy__.popitem_first().
@@ -358,21 +327,10 @@
         otherwise KeyError is raised
         """
         strategy = self.get_strategy()
-        if strategy.has_pop:
-            try:
-                return strategy.pop(self, w_key, w_default)
-            except KeyError:
-                raise space.raise_key_error(w_key)
-        # fall-back
-        w_item = self.getitem(w_key)
-        if w_item is None:
-            if w_default is not None:
-                return w_default
-            else:
-                space.raise_key_error(w_key)
-        else:
-            self.internal_delitem(w_key)
-            return w_item
+        try:
+            return strategy.pop(self, w_key, w_default)
+        except KeyError:
+            raise space.raise_key_error(w_key)
 
     def descr_popitem(self, space):
         """D.popitem() -> (k, v), remove and return some (key, value) pair as
@@ -612,7 +570,6 @@
         raise NotImplementedError
 
     has_iterreversed = False
-    has_move_to_end = False
     has_pop = False
     # ^^^ no default implementation available for these methods
 
@@ -627,6 +584,48 @@
     def prepare_update(self, w_dict, num_extra):
         pass
 
+    def move_to_end(self, w_dict, w_key, last_flag):
+        # fall-back
+        w_value = w_dict.getitem(w_key)
+        if w_value is None:
+            self.space.raise_key_error(w_key)
+        else:
+            w_dict.internal_delitem(w_key)
+            if last_flag:
+                w_dict.setitem(w_key, w_value)
+            else:
+                # *very slow* fall-back
+                keys_w = []
+                values_w = []
+                iteratorimplementation = w_dict.iteritems()
+                while True:
+                    w_k, w_v = iteratorimplementation.next_item()
+                    if w_k is None:
+                        break
+                    keys_w.append(w_k)
+                    values_w.append(w_v)
+                w_dict.clear()
+                w_dict.setitem(w_key, w_value)
+                for i in range(len(keys_w)):
+                    w_dict.setitem(keys_w[i], values_w[i])
+
+
+    def pop(self, w_dict, w_key, w_default):
+        # fall-back
+        w_item = w_dict.getitem(w_key)
+        if w_item is None:
+            if w_default is not None:
+                return w_default
+            else:
+                raise KeyError
+        else:
+            w_dict.internal_delitem(w_key)
+            return w_item
+
+    def iterreversed(self, w_dict):
+        # fall-back if getiterreversed is not present
+        w_keys = self.w_keys(w_dict)
+        return self.space.call_method(w_keys, '__reversed__')
 
 class EmptyDictStrategy(DictStrategy):
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -724,6 +723,12 @@
     def view_as_kwargs(self, w_dict):
         return ([], [])
 
+    def move_to_end(self, w_dict, w_key, last_flag):
+        self.space.raise_key_error(w_key)
+
+    def pop(self, w_dict, w_key, w_default):
+        raise KeyError
+
     # ---------- iterator interface ----------------
 
     def getiterkeys(self, w_dict):
@@ -881,15 +886,10 @@
 
     if hasattr(dictimpl, 'getiterreversed'):
         def iterreversed(self, w_dict):
-            return IterClassReversed(self.space, self, w_dict)
+            return W_DictMultiIterKeysObject(
+                    self.space,
+                    IterClassReversed(self.space, self, w_dict))
         dictimpl.iterreversed = iterreversed
-        dictimpl.has_iterreversed = True
-
-    if hasattr(dictimpl, 'move_to_end'):
-        dictimpl.has_move_to_end = True
-
-    if hasattr(dictimpl, 'pop'):
-        dictimpl.has_pop = True
 
     @jit.look_inside_iff(lambda self, w_dict, w_updatedict:
                          w_dict_unrolling_heuristic(w_dict))
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -227,6 +227,26 @@
         raises(KeyError, d.pop, "abc")
         assert len(d) == 2
 
+    def test_pop_kwargs(self):
+        def kw(**d): return d
+        d = kw(o=2, t=4)
+        dd = d.copy()
+        result = dd.pop("o")
+        assert result == 2
+        assert len(dd) == 1
+        dd = d.copy()
+        result = dd.pop("o", 44)
+        assert result == 2
+        assert len(dd) == 1
+        result = dd.pop("o", 44)
+        assert result == 44
+        assert len(dd) == 1
+        raises(KeyError, dd.pop, "33")
+
+        assert d.pop("abc", None) is None
+        raises(KeyError, d.pop, "abc")
+        assert len(d) == 2
+
     def test_has_key(self):
         d = {1: 2, 3: 4}
         assert d.has_key(1)
@@ -262,7 +282,8 @@
 
     def test_reversed_dict(self):
         import __pypy__
-        for d in [{}, {1: 2, 3: 4, 5: 6}, {"a": 5, "b": 2, "c": 6}]:
+        def kw(**d): return d
+        for d in [{}, {1: 2, 3: 4, 5: 6}, {"a": 5, "b": 2, "c": 6}, kw(a=1, b=2)]:
             assert list(__pypy__.reversed_dict(d)) == d.keys()[::-1]
         raises(TypeError, __pypy__.reversed_dict, 42)
 

From pypy.commits at gmail.com  Fri May 18 05:30:49 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 18 May 2018 02:30:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: fix co_firstlineno when
 annotations are present
Message-ID: <5afe9d49.1c69fb81.4ba74.8ccd@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94613:c6b57116d338
Date: 2018-05-18 11:28 +0200
http://bitbucket.org/pypy/pypy/changeset/c6b57116d338/

Log:	fix co_firstlineno when annotations are present

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -307,11 +307,10 @@
         # if the scope contained an annotated variable assignemt,
         # this will emit the requisite SETUP_ANNOTATIONS
         if self.scope.contains_annotated and not isinstance(self, AbstractFunctionCodeGenerator):
-            self.emit_op(ops.SETUP_ANNOTATIONS)
+            return self.emit_op(ops.SETUP_ANNOTATIONS)
 
     def visit_Module(self, mod):
-        if not self._handle_body(mod.body):
-            self.first_lineno = self.lineno = 1
+        self._handle_body(mod.body)
 
     def visit_Interactive(self, mod):
         self.interactive = True
@@ -1685,6 +1684,12 @@
                                      symbols, compile_info, qualname=None)
 
     def _compile(self, tree):
+        if isinstance(tree, ast.Module):
+            if tree.body:
+                self.first_lineno = tree.body[0].lineno
+            else:
+                self.first_lineno = self.lineno = 1
+
         self._maybe_setup_annotations()
         tree.walkabout(self)
 
diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py
--- a/pypy/interpreter/test/test_annotations.py
+++ b/pypy/interpreter/test/test_annotations.py
@@ -132,3 +132,11 @@
             ''')
         """)
 
+    def test_lineno(self):
+        s = """
+
+a: int
+        """
+        c = compile(s, "f", "exec")
+        assert c.co_firstlineno == 3
+

From pypy.commits at gmail.com  Fri May 18 06:04:43 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 18 May 2018 03:04:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix test_peepholer by undoing pypy-change
Message-ID: <5afea53b.1c69fb81.e43be.8f5f@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94614:b71f7b72b31d
Date: 2018-05-18 12:03 +0200
http://bitbucket.org/pypy/pypy/changeset/b71f7b72b31d/

Log:	fix test_peepholer by undoing pypy-change

diff --git a/lib-python/3/test/test_peepholer.py b/lib-python/3/test/test_peepholer.py
--- a/lib-python/3/test/test_peepholer.py
+++ b/lib-python/3/test/test_peepholer.py
@@ -81,11 +81,10 @@
         # On CPython, "a,b,c=1,2,3" turns into "a,b,c=<constant (1,2,3)>"
         # but on PyPy, it turns into "a=1;b=2;c=3".
         for line, elem in (
-            ('a = 1,2,3', '((1, 2, 3))'),
-            ('("a","b","c")', "(('a', 'b', 'c'))"),
-            ('a,b,c = 1,2,3', '((1, 2, 3))'),
-            ('(None, 1, None)', '((None, 1, None))'),
-            ('((1, 2), 3, 4)', '(((1, 2), 3, 4))'),
+            ('a = 1,2,3', (1, 2, 3)),
+            ('("a","b","c")', ('a', 'b', 'c')),
+            ('(None, 1, None)', (None, 1, None)),
+            ('((1, 2), 3, 4)', ((1, 2), 3, 4)),
             ):
             code = compile(line,'','single')
             self.assertInBytecode(code, 'LOAD_CONST', elem)

From pypy.commits at gmail.com  Fri May 18 06:06:26 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 18 May 2018 03:06:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: merge py3.6
Message-ID: <5afea5a2.1c69fb81.efe8b.4cbc@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94615:2d9f8bdb0649
Date: 2018-05-18 12:05 +0200
http://bitbucket.org/pypy/pypy/changeset/2d9f8bdb0649/

Log:	merge py3.6

diff --git a/lib-python/3/test/test_peepholer.py b/lib-python/3/test/test_peepholer.py
--- a/lib-python/3/test/test_peepholer.py
+++ b/lib-python/3/test/test_peepholer.py
@@ -81,11 +81,10 @@
         # On CPython, "a,b,c=1,2,3" turns into "a,b,c=<constant (1,2,3)>"
         # but on PyPy, it turns into "a=1;b=2;c=3".
         for line, elem in (
-            ('a = 1,2,3', '((1, 2, 3))'),
-            ('("a","b","c")', "(('a', 'b', 'c'))"),
-            ('a,b,c = 1,2,3', '((1, 2, 3))'),
-            ('(None, 1, None)', '((None, 1, None))'),
-            ('((1, 2), 3, 4)', '(((1, 2), 3, 4))'),
+            ('a = 1,2,3', (1, 2, 3)),
+            ('("a","b","c")', ('a', 'b', 'c')),
+            ('(None, 1, None)', (None, 1, None)),
+            ('((1, 2), 3, 4)', ((1, 2), 3, 4)),
             ):
             code = compile(line,'','single')
             self.assertInBytecode(code, 'LOAD_CONST', elem)

From pypy.commits at gmail.com  Fri May 18 11:55:19 2018
From: pypy.commits at gmail.com (mattip)
Date: Fri, 18 May 2018 08:55:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: add _Py_Dealloc to c function list to
 export it as _PyPy_Dealloc
Message-ID: <5afef767.1c69fb81.2ef6a.3010@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94616:a8f64c4ae1ec
Date: 2018-05-18 08:53 -0700
http://bitbucket.org/pypy/pypy/changeset/a8f64c4ae1ec/

Log:	add _Py_Dealloc to c function list to export it as _PyPy_Dealloc

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -645,7 +645,7 @@
     '_PyObject_New', '_PyObject_NewVar',
     '_PyObject_GC_New', '_PyObject_GC_NewVar',
     'PyObject_Init', 'PyObject_InitVar', 'PyInt_FromLong',
-    'PyTuple_New',
+    'PyTuple_New', '_Py_Dealloc',
 ]
 TYPES = {}
 FORWARD_DECLS = []

From pypy.commits at gmail.com  Sat May 19 06:55:03 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 19 May 2018 03:55:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: more precise tests,
 also fixes the raises problem
Message-ID: <5b000287.1c69fb81.cceb0.5efa@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94617:21081eaa7731
Date: 2018-05-19 12:54 +0200
http://bitbucket.org/pypy/pypy/changeset/21081eaa7731/

Log:	more precise tests, also fixes the raises problem

diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -525,19 +525,19 @@
         w_app_g3_i = space.wrap(app_g3_i)
         assert space.eq_w(space.call_function(w_app_g3_i,w(1)),w(1))
         assert space.eq_w(space.call_function(w_app_g3_i,w(1L)),w(1))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w(sys.maxint*2))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w(None))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w("foo"))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w(1.0))
+        space.raises_w(space.w_OverflowError, space.call_function,w_app_g3_i,w(sys.maxint*2))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_i,w(None))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_i,w("foo"))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_i,w(1.0))
 
         app_g3_s = gateway.interp2app_temp(g3_id,
                                          unwrap_spec=[gateway.ObjSpace,
                                                       'text'])
         w_app_g3_s = space.wrap(app_g3_s)
         assert space.eq_w(space.call_function(w_app_g3_s,w("foo")),w("foo"))
-        raises(gateway.OperationError,space.call_function,w_app_g3_s,w(None))
-        raises(gateway.OperationError,space.call_function,w_app_g3_s,w(1))
-        raises(gateway.OperationError,space.call_function,w_app_g3_s,w(1.0))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_s,w(None))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_s,w(1))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_s,w(1.0))
 
         app_g3_f = gateway.interp2app_temp(g3_id,
                                          unwrap_spec=[gateway.ObjSpace,
@@ -546,14 +546,14 @@
         assert space.eq_w(space.call_function(w_app_g3_f,w(1.0)),w(1.0))
         assert space.eq_w(space.call_function(w_app_g3_f,w(1)),w(1.0))
         assert space.eq_w(space.call_function(w_app_g3_f,w(1L)),w(1.0))
-        raises(gateway.OperationError,space.call_function,w_app_g3_f,w(None))
-        raises(gateway.OperationError,space.call_function,w_app_g3_f,w("foo"))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_f,w(None))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_f,w("foo"))
 
         app_g3_r = gateway.interp2app_temp(g3_id,
                                            unwrap_spec=[gateway.ObjSpace,
                                                         r_longlong])
         w_app_g3_r = space.wrap(app_g3_r)
-        raises(gateway.OperationError,space.call_function,w_app_g3_r,w(1.0))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_r,w(1.0))
 
     def test_interp2app_unwrap_spec_unicode(self):
         space = self.space
@@ -570,9 +570,9 @@
         assert self.space.eq_w(
             space.call_function(w_app_g3_u, w("baz")),
             w(3))
-        raises(gateway.OperationError, space.call_function, w_app_g3_u,
+        space.raises_w(space.w_TypeError, space.call_function, w_app_g3_u,
                w(None))
-        raises(gateway.OperationError, space.call_function, w_app_g3_u,
+        space.raises_w(space.w_TypeError, space.call_function, w_app_g3_u,
                w(42))
 
     def test_interp2app_unwrap_spec_unwrapper(self):
@@ -589,7 +589,7 @@
                                                       Unwrapper])
         assert self.space.eq_w(
             space.call_function(w(app_g3_u), w(42)), w(43))
-        raises(gateway.OperationError, space.call_function,
+        space.raises_w(space.w_TypeError, space.call_function,
                w(app_g3_u), w(None))
 
     def test_interp2app_classmethod(self):

From pypy.commits at gmail.com  Mon May 21 08:11:35 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 05:11:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge py3.5
Message-ID: <5b02b777.1c69fb81.894bc.0ee4@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94618:c4057f244c87
Date: 2018-05-21 14:10 +0200
http://bitbucket.org/pypy/pypy/changeset/c4057f244c87/

Log:	merge py3.5

diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -525,19 +525,19 @@
         w_app_g3_i = space.wrap(app_g3_i)
         assert space.eq_w(space.call_function(w_app_g3_i,w(1)),w(1))
         assert space.eq_w(space.call_function(w_app_g3_i,w(1L)),w(1))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w(sys.maxint*2))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w(None))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w("foo"))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w(1.0))
+        space.raises_w(space.w_OverflowError, space.call_function,w_app_g3_i,w(sys.maxint*2))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_i,w(None))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_i,w("foo"))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_i,w(1.0))
 
         app_g3_s = gateway.interp2app_temp(g3_id,
                                          unwrap_spec=[gateway.ObjSpace,
                                                       'text'])
         w_app_g3_s = space.wrap(app_g3_s)
         assert space.eq_w(space.call_function(w_app_g3_s,w("foo")),w("foo"))
-        raises(gateway.OperationError,space.call_function,w_app_g3_s,w(None))
-        raises(gateway.OperationError,space.call_function,w_app_g3_s,w(1))
-        raises(gateway.OperationError,space.call_function,w_app_g3_s,w(1.0))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_s,w(None))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_s,w(1))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_s,w(1.0))
 
         app_g3_f = gateway.interp2app_temp(g3_id,
                                          unwrap_spec=[gateway.ObjSpace,
@@ -546,14 +546,14 @@
         assert space.eq_w(space.call_function(w_app_g3_f,w(1.0)),w(1.0))
         assert space.eq_w(space.call_function(w_app_g3_f,w(1)),w(1.0))
         assert space.eq_w(space.call_function(w_app_g3_f,w(1L)),w(1.0))
-        raises(gateway.OperationError,space.call_function,w_app_g3_f,w(None))
-        raises(gateway.OperationError,space.call_function,w_app_g3_f,w("foo"))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_f,w(None))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_f,w("foo"))
 
         app_g3_r = gateway.interp2app_temp(g3_id,
                                            unwrap_spec=[gateway.ObjSpace,
                                                         r_longlong])
         w_app_g3_r = space.wrap(app_g3_r)
-        raises(gateway.OperationError,space.call_function,w_app_g3_r,w(1.0))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_r,w(1.0))
 
     def test_interp2app_unwrap_spec_unicode(self):
         space = self.space
@@ -570,9 +570,9 @@
         assert self.space.eq_w(
             space.call_function(w_app_g3_u, w("baz")),
             w(3))
-        raises(gateway.OperationError, space.call_function, w_app_g3_u,
+        space.raises_w(space.w_TypeError, space.call_function, w_app_g3_u,
                w(None))
-        raises(gateway.OperationError, space.call_function, w_app_g3_u,
+        space.raises_w(space.w_TypeError, space.call_function, w_app_g3_u,
                w(42))
 
     def test_interp2app_unwrap_spec_unwrapper(self):
@@ -589,7 +589,7 @@
                                                       Unwrapper])
         assert self.space.eq_w(
             space.call_function(w(app_g3_u), w(42)), w(43))
-        raises(gateway.OperationError, space.call_function,
+        space.raises_w(space.w_TypeError, space.call_function,
                w(app_g3_u), w(None))
 
     def test_interp2app_classmethod(self):

From pypy.commits at gmail.com  Mon May 21 08:28:16 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 05:28:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: the property name changed
Message-ID: <5b02bb60.1c69fb81.77fc6.4c3b@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94619:7f0060a706fd
Date: 2018-05-21 14:27 +0200
http://bitbucket.org/pypy/pypy/changeset/7f0060a706fd/

Log:	the property name changed

diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -427,6 +427,6 @@
     def test_builtin_readonly_property(self):
         import sys
         x = lambda: 5
-        e = raises(AttributeError, 'x.func_globals = {}')
+        e = raises(AttributeError, 'x.__globals__ = {}')
         if '__pypy__' in sys.builtin_module_names:
-            assert str(e.value) == "readonly attribute 'func_globals'"
+            assert str(e.value) == "readonly attribute '__globals__'"

From pypy.commits at gmail.com  Mon May 21 08:34:48 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 05:34:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: py2 fix
Message-ID: <5b02bce8.1c69fb81.c1edb.9c69@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94620:f43a1c80ee44
Date: 2018-05-18 21:26 +0200
http://bitbucket.org/pypy/pypy/changeset/f43a1c80ee44/

Log:	py2 fix

diff --git a/pypy/tool/dis3.py b/pypy/tool/dis3.py
--- a/pypy/tool/dis3.py
+++ b/pypy/tool/dis3.py
@@ -394,6 +394,8 @@
     lineno = code.co_firstlineno
     addr = 0
     for byte_incr, line_incr in zip(byte_increments, line_increments):
+        byte_incr = ord(byte_incr)
+        line_incr = ord(line_incr)
         if byte_incr:
             if lineno != lastlineno:
                 yield (addr, lineno)

From pypy.commits at gmail.com  Mon May 21 08:34:50 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 05:34:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: another place that inspects the
 bytecode
Message-ID: <5b02bcea.1c69fb81.ade6f.76eb@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94621:8bd5845552f9
Date: 2018-05-19 11:24 +0200
http://bitbucket.org/pypy/pypy/changeset/8bd5845552f9/

Log:	another place that inspects the bytecode

diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -691,6 +691,7 @@
         endblock = [-1]     # current finally/except block stack
         addr = 0
         while addr < len(code):
+            assert addr & 1 == 0
             op = ord(code[addr])
             if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
                       SETUP_ASYNC_WITH):
@@ -713,10 +714,7 @@
             if addr == self.last_instr:
                 f_lasti_handler_addr = endblock[-1]
 
-            if op >= HAVE_ARGUMENT:
-                addr += 3
-            else:
-                addr += 1
+            addr += 2
 
         if len(blockstack) != 0 or len(endblock) != 1:
             raise oefmt(space.w_SystemError,

From pypy.commits at gmail.com  Mon May 21 08:34:57 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 05:34:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: also emit BUILD_CONST_KEY_MAP
 for regular dicts (the implementation could be faster, but later)
Message-ID: <5b02bcf1.1c69fb81.a0222.b8b4@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94624:761a52b54b30
Date: 2018-05-21 14:09 +0200
http://bitbucket.org/pypy/pypy/changeset/761a52b54b30/

Log:	also emit BUILD_CONST_KEY_MAP for regular dicts (the implementation
	could be faster, but later)

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1296,28 +1296,46 @@
         containers = 0
         elements = 0
         is_unpacking = False
+        all_constant_keys_w = None
         if d.values:
+            if len(d.keys) < 0xffff:
+                all_constant_keys_w = []
+                for key in d.keys:
+                    if key is None or key.as_constant() is None:
+                        all_constant_keys_w = None
+                        break
+                    else:
+                        all_constant_keys_w.append(key.as_constant())
             for i in range(len(d.values)):
                 key = d.keys[i]
                 is_unpacking = key is None
                 if elements == 0xFFFF or (elements and is_unpacking):
+                    assert all_constant_keys_w is None
                     self.emit_op_arg(ops.BUILD_MAP, elements)
                     containers += 1
                     elements = 0
                 if is_unpacking:
+                    assert all_constant_keys_w is None
                     d.values[i].walkabout(self)
                     containers += 1
                 else:
-                    key.walkabout(self)
+                    if not all_constant_keys_w:
+                        key.walkabout(self)
                     d.values[i].walkabout(self)
                     elements += 1
         if elements or containers == 0:
-            self.emit_op_arg(ops.BUILD_MAP, elements)
-            containers += 1
+            if all_constant_keys_w:
+                w_tup = self.space.newtuple(all_constant_keys_w)
+                self.load_const(w_tup)
+                self.emit_op_arg(ops.BUILD_CONST_KEY_MAP, elements)
+            else:
+                self.emit_op_arg(ops.BUILD_MAP, elements)
+                containers += 1
         # If there is more than one dict, they need to be merged into
         # a new dict. If there is one dict and it's an unpacking, then
         #it needs to be copied into a new dict.
         while containers > 1 or is_unpacking:
+            assert all_constant_keys_w is None
             oparg = min(containers, 255)
             self.emit_op_arg(ops.BUILD_MAP_UNPACK, oparg)
             containers -= (oparg - 1)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1491,3 +1491,18 @@
             del []
         """
         generate_function_code(source, self.space)
+
+    def test_make_constant_map(self):
+        source = """def f():
+            return {"A": 1, "b": 2}
+        """
+        counts = self.count_instructions(source)
+        assert ops.BUILD_MAP not in counts
+        source = """def f():
+            return {"a": 1, "b": {}, 1: {"a": x}}
+        """
+        counts = self.count_instructions(source)
+        assert counts[ops.BUILD_MAP] == 1 # the empty dict
+        assert counts[ops.BUILD_CONST_KEY_MAP] == 2
+
+
diff --git a/pypy/interpreter/test/test_pycode.py b/pypy/interpreter/test/test_pycode.py
--- a/pypy/interpreter/test/test_pycode.py
+++ b/pypy/interpreter/test/test_pycode.py
@@ -14,6 +14,6 @@
     finally:
         sys.stdout = stdout
     print '>>>\n' + output + '\n<<<'
-    assert ' 1 (7)' in output
+    assert ' 0 (7)' in output
     assert ' 4 (None)' in output
     assert ' 16 RETURN_VALUE' in output

From pypy.commits at gmail.com  Mon May 21 08:34:52 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 05:34:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: there is yet another piece of
 disassembling code, for the nowadays rarely used
Message-ID: <5b02bcec.1c69fb81.74596.b8d9@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94622:c1abcb494f75
Date: 2018-05-21 13:33 +0200
http://bitbucket.org/pypy/pypy/changeset/c1abcb494f75/

Log:	there is yet another piece of disassembling code, for the nowadays
	rarely used __pytrace__ = True support in pyinteractive.py Fix it
	somewhat, delete code that's less useful.

diff --git a/pypy/interpreter/interactive.py b/pypy/interpreter/interactive.py
--- a/pypy/interpreter/interactive.py
+++ b/pypy/interpreter/interactive.py
@@ -213,7 +213,7 @@
             ec.bytecode_only_trace = self._orig_bytecode_only_trace
 
     def _do_bytecode_only_trace(self, frame):
-        from pypy.tool.pydis import Bytecode, HAVE_ARGUMENT
+        from pypy.tool import opcode3, dis3
 
         if frame.hide():
             return
@@ -221,18 +221,12 @@
         self.unsettrace()
         next_instr = frame.last_instr
         opcode = ord(frame.pycode.co_code[next_instr])
+        oparg = ord(frame.pycode.co_code[next_instr+1])
 
-        oparg = 0
-        if opcode >= HAVE_ARGUMENT:
-            lo = ord(frame.pycode.co_code[next_instr+1])
-            hi = ord(frame.pycode.co_code[next_instr+2])
-            oparg = (hi * 256) | lo
-
-        class fake:
-            code = frame.pycode
-        bytecode = Bytecode(fake, next_instr, oparg, 0)
+        argrepr = reprargstring(self.space, frame.pycode, opcode, oparg)
+        oprepr = opcode3.opname[opcode] + argrepr.ljust(5)
         print '\t%-19s %s' % (str(frame.pycode.co_name) + ':',
-                              bytecode.repr_with_space(self.space))
+                              oprepr)
         self.settrace()
 
     def checktrace(self):
@@ -255,3 +249,26 @@
 
 class IncompleteInput(Exception):
     pass
+
+
+def reprargstring(space, pycode, opcode, oparg):
+    """ return a string representation of any arguments. (empty for no args)"""
+    from pypy.tool import opcode3
+    if oparg is None:
+        return ''
+    s = repr(oparg).rjust(5) + " "
+    if opcode in opcode3.hasconst:
+        r = space.text_w(space.repr(pycode.co_consts_w[oparg]))
+        s += '(' + r + ')'
+    elif opcode in opcode3.hasname:
+        s +=  '(' + pycode.co_names[oparg] + ')'
+    elif opcode in opcode3.hasjrel:
+        s +=  '(to ' + repr(self.index + oparg) + ')'
+    elif opcode in opcode3.haslocal:
+        s +=  '(' + pycode.co_varnames[oparg] + ')'
+    elif opcode in opcode3.hascompare:
+        s +=  '(' + opcode3.cmp_op[oparg] + ')'
+    elif opcode in opcode3.hasfree:
+        free = pycode.co_cellvars + pycode.co_freevars
+        s +=  '(' + free[oparg] + ')'
+    return s
diff --git a/pypy/interpreter/test/test_zpy.py b/pypy/interpreter/test/test_zpy.py
--- a/pypy/interpreter/test/test_zpy.py
+++ b/pypy/interpreter/test/test_zpy.py
@@ -122,6 +122,3 @@
     # '5\n' --- this line sent to stderr
     assert ('\t<module>:           LOAD_NAME    0 (x)\n'
             '\t<module>:           PRINT_EXPR    0 \n') in output
-    assert ('\t<module>:           LOAD_CONST    0 (None)\n'
-            '\t<module>:           RETURN_VALUE    0 \n'
-            '>>>> ') in output
diff --git a/pypy/tool/pydis.py b/pypy/tool/pydis.py
deleted file mode 100644
--- a/pypy/tool/pydis.py
+++ /dev/null
@@ -1,202 +0,0 @@
-"""disassembler of Python byte code into mnemonics.
-
-XXX this only works for python-2.3 because of the linenumber 
-    optimization 
-
-"""
-
-import sys
-
-from pypy.tool import stdlib_opcode
-from pypy.tool.stdlib_opcode import *
-
-__all__ = ["dis","pydisassemble","distb","disco"] + stdlib_opcode.__all__
-
-EXTENDED_ARG = stdlib_opcode.opcodedesc.EXTENDED_ARG.index
-
-
-class Bytecode:
-    def __init__(self, disresult, bytecodeindex, oparg, lineno):
-        self.disresult = disresult 
-        self.index = bytecodeindex
-        self.op = ord(disresult.code.co_code[self.index])
-        self.name = opname[self.op]
-        self.oparg = oparg
-        self.lineno = lineno
-
-    def __eq__(self, other):
-        return (self.__class__ == other.__class__ and 
-                self.index == other.index and
-                self.op == other.op and
-                self.name == other.name and
-                self.oparg == other.oparg)
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def reprargstring(self, space = None):
-        """ return a string representation of any arguments. (empty for no args)"""
-        oparg = self.oparg
-        if oparg is None:
-            return ''
-        co = self.disresult.code
-        op = self.op
-        
-        s = repr(oparg).rjust(5) + " "
-        if op in hasconst:
-            consts = self.get_consts(space)
-            s += '(' + consts[oparg] + ')'
-        elif op in hasname:
-            s +=  '(' + co.co_names[oparg] + ')'
-        elif op in hasjrel:
-            s +=  '(to ' + repr(self.index + oparg) + ')'
-        elif op in haslocal:
-            s +=  '(' + co.co_varnames[oparg] + ')'
-        elif op in hascompare:
-            s +=  '(' + cmp_op[oparg] + ')'
-        elif op in hasfree:
-            #if free is None:
-            free = co.co_cellvars + co.co_freevars
-            s +=  '(' + free[oparg] + ')'
-        return s 
-
-    def get_consts(self, space=None):
-        # support both real code objects and PyCode objects
-        co = self.disresult.code
-        if hasattr(co, "co_consts"):
-            return [repr(c) for c in co.co_consts]
-
-        if space is None:
-            return [repr(c) for c in co.co_consts_w]
-        
-        r = lambda x: space.str_w(space.repr(x))
-        return [r(c) for c in co.co_consts_w]
-
-    def repr_with_space(self, space):
-        return self.name + self.reprargstring(space)
-
-    def __repr__(self):
-        return self.name + self.reprargstring()
-
-class DisResult:
-    """ an instance of this class gets returned for disassembling 
-        objects/functions/code objects whatever.    
-    """
-    def __init__(self, code):
-        self.code = code
-        self.bytecodes = []
-   
-    def append(self,  bytecodeindex, oparg, lineno):
-        """ append bytecode anaylsis information ..."""
-        bc = Bytecode(self, bytecodeindex, oparg, lineno)
-        self.bytecodes.append(bc)
-
-    def getbytecode(self, index):
-        """ return bytecode instance matching the given index. """
-        for bytecode in self.bytecodes:
-            if bytecode.index == index:
-                return bytecode
-        raise ValueError("no bytecode found on index %s in code \n%s" % (
-                index, pydis(self.code)))
-
-    def format(self):
-        lastlineno = -1
-        labels = findlabels(self.code.co_code)
-        lines = []
-        for bc in self.bytecodes:
-            l = []
-            if bc.lineno != lastlineno:
-                lastlineno = bc.lineno
-                l.append("%3d" % bc.lineno)
-            else:
-                l.append("   ")
-            l.append(bc.index in labels and ">>" or "  ")
-            l.append(repr(bc.index).rjust(4)) 
-            l.append(bc.name.ljust(20))
-            l.append(bc.reprargstring())
-            lines.append(" ".join(l))
-        return "\n".join(lines)
-
-    __repr__ = format
-
-def pydis(co): 
-    """return result of dissassembling a code object. """
-
-    if hasattr(co, 'func_code'):
-        co = co.func_code 
-
-    if hasattr(co, 'code'):
-        co = co.code 
-
-    disresult = DisResult(co)
-    code = co.co_code
-
-    byte_increments = [ord(c) for c in co.co_lnotab[0::2]]
-    line_increments = [ord(c) for c in co.co_lnotab[1::2]]
-    table_length = len(byte_increments)
-
-    lineno = co.co_firstlineno
-    table_index = 0
-    while (table_index < table_length
-           and byte_increments[table_index] == 0):
-        lineno += line_increments[table_index]
-        table_index += 1
-    addr = 0
-    line_incr = 0
-
-    n = len(code)
-    i = 0
-    extended_arg = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-
-        if i >= addr:
-            lineno += line_incr
-            while table_index < table_length:
-                addr += byte_increments[table_index]
-                line_incr = line_increments[table_index]
-                table_index += 1
-                if line_incr:
-                    break
-            else:
-                addr = sys.maxint
-        current_bytecodeindex = i
-        i = i+1
-        oparg = None
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256 + extended_arg
-            extended_arg = 0
-            i = i+2
-            if op == EXTENDED_ARG:
-                extended_arg = oparg*65536L
-
-        disresult.append(current_bytecodeindex, oparg, lineno)
-    assert disresult is not None
-    return disresult
-
-def findlabels(code):
-    """Detect all offsets in a byte code which are jump targets.
-
-    Return the list of offsets.
-
-    """
-    labels = []
-    n = len(code)
-    i = 0
-    while i < n:
-        c = code[i]
-        op = ord(c)
-        i = i+1
-        if op >= HAVE_ARGUMENT:
-            oparg = ord(code[i]) + ord(code[i+1])*256
-            i = i+2
-            label = -1
-            if op in hasjrel:
-                label = i+oparg
-            elif op in hasjabs:
-                label = oparg
-            if label >= 0:
-                if label not in labels:
-                    labels.append(label)
-    return labels

From pypy.commits at gmail.com  Mon May 21 08:34:59 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 05:34:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: merge py3.5
Message-ID: <5b02bcf3.1c69fb81.7e0ef.b2b6@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94625:38388e5c10c5
Date: 2018-05-21 14:34 +0200
http://bitbucket.org/pypy/pypy/changeset/38388e5c10c5/

Log:	merge py3.5

diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -525,19 +525,19 @@
         w_app_g3_i = space.wrap(app_g3_i)
         assert space.eq_w(space.call_function(w_app_g3_i,w(1)),w(1))
         assert space.eq_w(space.call_function(w_app_g3_i,w(1L)),w(1))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w(sys.maxint*2))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w(None))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w("foo"))
-        raises(gateway.OperationError,space.call_function,w_app_g3_i,w(1.0))
+        space.raises_w(space.w_OverflowError, space.call_function,w_app_g3_i,w(sys.maxint*2))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_i,w(None))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_i,w("foo"))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_i,w(1.0))
 
         app_g3_s = gateway.interp2app_temp(g3_id,
                                          unwrap_spec=[gateway.ObjSpace,
                                                       'text'])
         w_app_g3_s = space.wrap(app_g3_s)
         assert space.eq_w(space.call_function(w_app_g3_s,w("foo")),w("foo"))
-        raises(gateway.OperationError,space.call_function,w_app_g3_s,w(None))
-        raises(gateway.OperationError,space.call_function,w_app_g3_s,w(1))
-        raises(gateway.OperationError,space.call_function,w_app_g3_s,w(1.0))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_s,w(None))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_s,w(1))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_s,w(1.0))
 
         app_g3_f = gateway.interp2app_temp(g3_id,
                                          unwrap_spec=[gateway.ObjSpace,
@@ -546,14 +546,14 @@
         assert space.eq_w(space.call_function(w_app_g3_f,w(1.0)),w(1.0))
         assert space.eq_w(space.call_function(w_app_g3_f,w(1)),w(1.0))
         assert space.eq_w(space.call_function(w_app_g3_f,w(1L)),w(1.0))
-        raises(gateway.OperationError,space.call_function,w_app_g3_f,w(None))
-        raises(gateway.OperationError,space.call_function,w_app_g3_f,w("foo"))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_f,w(None))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_f,w("foo"))
 
         app_g3_r = gateway.interp2app_temp(g3_id,
                                            unwrap_spec=[gateway.ObjSpace,
                                                         r_longlong])
         w_app_g3_r = space.wrap(app_g3_r)
-        raises(gateway.OperationError,space.call_function,w_app_g3_r,w(1.0))
+        space.raises_w(space.w_TypeError, space.call_function,w_app_g3_r,w(1.0))
 
     def test_interp2app_unwrap_spec_unicode(self):
         space = self.space
@@ -570,9 +570,9 @@
         assert self.space.eq_w(
             space.call_function(w_app_g3_u, w("baz")),
             w(3))
-        raises(gateway.OperationError, space.call_function, w_app_g3_u,
+        space.raises_w(space.w_TypeError, space.call_function, w_app_g3_u,
                w(None))
-        raises(gateway.OperationError, space.call_function, w_app_g3_u,
+        space.raises_w(space.w_TypeError, space.call_function, w_app_g3_u,
                w(42))
 
     def test_interp2app_unwrap_spec_unwrapper(self):
@@ -589,7 +589,7 @@
                                                       Unwrapper])
         assert self.space.eq_w(
             space.call_function(w(app_g3_u), w(42)), w(43))
-        raises(gateway.OperationError, space.call_function,
+        space.raises_w(space.w_TypeError, space.call_function,
                w(app_g3_u), w(None))
 
     def test_interp2app_classmethod(self):
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -427,6 +427,6 @@
     def test_builtin_readonly_property(self):
         import sys
         x = lambda: 5
-        e = raises(AttributeError, 'x.func_globals = {}')
+        e = raises(AttributeError, 'x.__globals__ = {}')
         if '__pypy__' in sys.builtin_module_names:
-            assert str(e.value) == "readonly attribute 'func_globals'"
+            assert str(e.value) == "readonly attribute '__globals__'"

From pypy.commits at gmail.com  Mon May 21 08:34:54 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 05:34:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: implement the new MAKE_FUNCTION
 bytecode, that subsumes MAKE_FUNCTION and
Message-ID: <5b02bcee.1c69fb81.d988a.2832@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94623:fdaa88d1d204
Date: 2018-05-21 13:34 +0200
http://bitbucket.org/pypy/pypy/changeset/fdaa88d1d204/

Log:	implement the new MAKE_FUNCTION bytecode, that subsumes
	MAKE_FUNCTION and MAKE_CLOSURE.

diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -216,6 +216,7 @@
 def_op('BUILD_SET_UNPACK', 153)
 
 def_op('FORMAT_VALUE', 155)   # in CPython 3.6, but available in PyPy from 3.5
+def_op('BUILD_CONST_KEY_MAP', 156)
 def_op('BUILD_STRING', 157)   # in CPython 3.6, but available in PyPy from 3.5
 
 # pypy modification, experimental bytecode
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -757,7 +757,7 @@
     return -2 - _num_args(arg) - ((arg >> 16) & 0xFFFF)
 
 def _compute_MAKE_FUNCTION(arg):
-    return -1 - _num_args(arg) - ((arg >> 16) & 0xFFFF)
+    return -1 - bool(arg & 0x01) - bool(arg & 0x02) - bool(arg & 0x04) - bool(arg & 0x08)
 
 def _compute_BUILD_SLICE(arg):
     if arg == 3:
@@ -794,6 +794,9 @@
 def _compute_BUILD_STRING(arg):
     return 1 - arg
 
+def _compute_BUILD_CONST_KEY_MAP(arg):
+    return  -arg
+
 
 _stack_effect_computers = {}
 for name, func in globals().items():
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -320,10 +320,11 @@
         self.add_none_to_final_return = False
         mod.body.walkabout(self)
 
-    def _make_function(self, code, num_defaults=0, qualname=None):
+    def _make_function(self, code, oparg=0, qualname=None):
         """Emit the opcodes to turn a code object into a function."""
         w_qualname = self.space.newtext(qualname or code.co_name)
         if code.co_freevars:
+            oparg = oparg | 0x08
             # Load cell and free vars to pass on.
             for free in code.co_freevars:
                 free_scope = self.scope.lookup(free)
@@ -334,24 +335,25 @@
                     index = self.free_vars[free]
                 self.emit_op_arg(ops.LOAD_CLOSURE, index)
             self.emit_op_arg(ops.BUILD_TUPLE, len(code.co_freevars))
-            self.load_const(code)
-            self.load_const(w_qualname)
-            self.emit_op_arg(ops.MAKE_CLOSURE, num_defaults)
-        else:
-            self.load_const(code)
-            self.load_const(w_qualname)
-            self.emit_op_arg(ops.MAKE_FUNCTION, num_defaults)
+        self.load_const(code)
+        self.load_const(w_qualname)
+        self.emit_op_arg(ops.MAKE_FUNCTION, oparg)
 
     def _visit_kwonlydefaults(self, args):
         defaults = 0
+        keys_w = []
         for i, default in enumerate(args.kw_defaults):
             if default:
                 kwonly = args.kwonlyargs[i]
                 assert isinstance(kwonly, ast.arg)
                 mangled = self.scope.mangle(kwonly.arg)
-                self.load_const(self.space.newtext(mangled))
+                keys_w.append(self.space.newtext(mangled))
                 default.walkabout(self)
                 defaults += 1
+        if keys_w:
+            w_tup = self.space.newtuple(keys_w)
+            self.load_const(w_tup)
+            self.emit_op_arg(ops.BUILD_CONST_KEY_MAP, len(keys_w))
         return defaults
 
     def _visit_arg_annotation(self, name, ann, names):
@@ -386,7 +388,7 @@
                 self.error("too many annotations", func)
             w_tup = space.newtuple([space.newtext(name) for name in names])
             self.load_const(w_tup)
-            l += 1
+            self.emit_op_arg(ops.BUILD_CONST_KEY_MAP, l)
         return l
 
     @specialize.arg(2)
@@ -395,16 +397,25 @@
         # Load decorators first, but apply them after the function is created.
         self.visit_sequence(func.decorator_list)
         args = func.args
+
         assert isinstance(args, ast.arguments)
+
+        oparg = 0
         self.visit_sequence(args.defaults)
-        kw_default_count = 0
+
+        if args.defaults is not None and len(args.defaults):
+            oparg = oparg | 0x01
+            self.emit_op_arg(ops.BUILD_TUPLE, len(args.defaults))
+
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
+            if kw_default_count:
+                oparg = oparg | 0x02
+
         num_annotations = self._visit_annotations(func, args, func.returns)
-        num_defaults = len(args.defaults) if args.defaults is not None else 0
-        oparg = num_defaults
-        oparg |= kw_default_count << 8
-        oparg |= num_annotations << 16
+        if num_annotations:
+            oparg = oparg | 0x04
+
         code, qualname = self.sub_scope(function_code_generator, func.name,
                                         func, func.lineno)
         self._make_function(code, oparg, qualname=qualname)
@@ -424,15 +435,20 @@
         self.update_position(lam.lineno)
         args = lam.args
         assert isinstance(args, ast.arguments)
+
         self.visit_sequence(args.defaults)
-        kw_default_count = 0
+
+        oparg = 0
+        if args.defaults is not None and len(args.defaults):
+            oparg = oparg | 0x01
+            self.emit_op_arg(ops.BUILD_TUPLE, len(args.defaults))
+
         if args.kwonlyargs:
             kw_default_count = self._visit_kwonlydefaults(args)
-        default_count = len(args.defaults) if args.defaults is not None else 0
+            if kw_default_count:
+                oparg = oparg | 0x02
         code, qualname = self.sub_scope(
             LambdaCodeGenerator, "<lambda>", lam, lam.lineno)
-        oparg = default_count
-        oparg |= kw_default_count << 8
         self._make_function(code, oparg, qualname=qualname)
 
     def visit_ClassDef(self, cls):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -39,7 +39,7 @@
 # time you make pyc files incompatible.  This value ends up in the frozen
 # importlib, via MAGIC_NUMBER in module/_frozen_importlib/__init__.
 
-pypy_incremental_magic = 144 # bump it by 16
+pypy_incremental_magic = 160 # bump it by 16
 assert pypy_incremental_magic % 16 == 0
 assert pypy_incremental_magic < 3000 # the magic number of Python 3. There are
                                      # no known magic numbers below this value
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -247,6 +247,8 @@
                 self.BINARY_TRUE_DIVIDE(oparg, next_instr)
             elif opcode == opcodedesc.BINARY_XOR.index:
                 self.BINARY_XOR(oparg, next_instr)
+            elif opcode == opcodedesc.BUILD_CONST_KEY_MAP.index:
+                self.BUILD_CONST_KEY_MAP(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_LIST.index:
                 self.BUILD_LIST(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_LIST_FROM_ARG.index:
@@ -357,8 +359,6 @@
                 self.LOAD_NAME(oparg, next_instr)
             elif opcode == opcodedesc.LOOKUP_METHOD.index:
                 self.LOOKUP_METHOD(oparg, next_instr)
-            elif opcode == opcodedesc.MAKE_CLOSURE.index:
-                self.MAKE_CLOSURE(oparg, next_instr)
             elif opcode == opcodedesc.MAKE_FUNCTION.index:
                 self.MAKE_FUNCTION(oparg, next_instr)
             elif opcode == opcodedesc.MAP_ADD.index:
@@ -1341,47 +1341,42 @@
         self.call_function(oparg, w_varkw, has_vararg=True)
 
     @jit.unroll_safe
-    def _make_function(self, oparg, freevars=None):
+    def MAKE_FUNCTION(self, oparg, next_instr):
         space = self.space
         w_qualname = self.popvalue()
         qualname = self.space.unicode_w(w_qualname)
         w_codeobj = self.popvalue()
         codeobj = self.space.interp_w(PyCode, w_codeobj)
-        if freevars is not None:
-            # Pop freevars
-            self.popvalue()
-        posdefaults = oparg & 0xFF
-        kwdefaults = (oparg >> 8) & 0xFF
-        num_annotations = (oparg >> 16) & 0xFF
-        w_ann = None
-        if num_annotations:
-            names_w = space.fixedview(self.popvalue())
-            w_ann = space.newdict(strdict=True)
-            for i in range(len(names_w) - 1, -1, -1):
-                space.setitem(w_ann, names_w[i], self.popvalue())
-        kw_defs_w = None
-        if kwdefaults:
-            kw_defs_w = []
-            for i in range(kwdefaults):
-                w_defvalue = self.popvalue()
-                w_defname = self.popvalue()
-                kw_defs_w.append((w_defname, w_defvalue))
-        defaultarguments = self.popvalues(posdefaults)
+        assert 0 <= oparg <= 0x0F
+        if oparg & 0x08:
+            w_freevarstuple = self.popvalue()
+            # XXX this list copy is expensive, it's purely for the annotator
+            freevars = [self.space.interp_w(Cell, cell)
+                        for cell in self.space.fixedview(w_freevarstuple)]
+        else:
+            freevars = None
+        if oparg & 0x04:
+            w_ann = self.popvalue()
+        else:
+            w_ann = None
+        if oparg & 0x02:
+            w_kw_defs = self.popvalue()
+            # XXX
+            kw_defs_w = [space.unpackiterable(w_tup)
+                            for w_tup in space.fixedview(
+                                space.call_method(w_kw_defs, 'items'))]
+        else:
+            kw_defs_w = None
+        if oparg & 0x01:
+            defaultarguments = space.fixedview(self.popvalue())
+        else:
+            defaultarguments = []
+
         fn = function.Function(space, codeobj, self.get_w_globals(),
                                defaultarguments,
                                kw_defs_w, freevars, w_ann, qualname=qualname)
         self.pushvalue(fn)
 
-    def MAKE_FUNCTION(self, oparg, next_instr):
-        return self._make_function(oparg)
-
-    @jit.unroll_safe
-    def MAKE_CLOSURE(self, oparg, next_instr):
-        w_freevarstuple = self.peekvalue(2)
-        freevars = [self.space.interp_w(Cell, cell)
-                    for cell in self.space.fixedview(w_freevarstuple)]
-        self._make_function(oparg, freevars)
-
     def BUILD_SLICE(self, numargs, next_instr):
         if numargs == 3:
             w_step = self.popvalue()
@@ -1431,7 +1426,18 @@
             w_value = self.peekvalue(2 * i)
             w_key = self.peekvalue(2 * i + 1)
             self.space.setitem(w_dict, w_key, w_value)
-        self.popvalues(2 * itemcount)
+        self.dropvalues(2 * itemcount)
+        self.pushvalue(w_dict)
+
+    @jit.unroll_safe
+    def BUILD_CONST_KEY_MAP(self, itemcount, next_instr):
+        keys_w = self.space.fixedview(self.popvalue())
+        w_dict = self.space.newdict()
+        for i in range(itemcount):
+            w_value = self.peekvalue(itemcount - 1 - i)
+            w_key = keys_w[i]
+            self.space.setitem(w_dict, w_key, w_value)
+        self.dropvalues(itemcount)
         self.pushvalue(w_dict)
 
     @jit.unroll_safe
@@ -1440,7 +1446,7 @@
         for i in range(itemcount-1, -1, -1):
             w_item = self.peekvalue(i)
             self.space.call_method(w_set, 'add', w_item)
-        self.popvalues(itemcount)
+        self.dropvalues(itemcount)
         self.pushvalue(w_set)
 
     @jit.unroll_safe
diff --git a/pypy/tool/opcode3.py b/pypy/tool/opcode3.py
--- a/pypy/tool/opcode3.py
+++ b/pypy/tool/opcode3.py
@@ -216,6 +216,7 @@
 def_op('BUILD_SET_UNPACK', 153)
 
 def_op('FORMAT_VALUE', 155)   # in CPython 3.6, but available in PyPy from 3.5
+def_op("BUILD_CONST_KEY_MAP", 156)
 def_op('BUILD_STRING', 157)   # in CPython 3.6, but available in PyPy from 3.5
 
 # pypy modification, experimental bytecode

From pypy.commits at gmail.com  Mon May 21 10:03:57 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:03:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: try to see whether it's plausible to
 forbid moving methods up the class
Message-ID: <5b02d1cd.1c69fb81.2a879.1909@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94626:b51d40dbb01d
Date: 2018-05-17 13:56 +0200
http://bitbucket.org/pypy/pypy/changeset/b51d40dbb01d/

Log:	try to see whether it's plausible to forbid moving methods up the
	class hierarchy in rpython

diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -102,13 +102,20 @@
 
     def validate(self, homedef):
         s_newvalue = self.s_value
-        # check for after-the-fact method additions
+        homedesc = homedef.classdesc
+        # check for method demotion and after-the-fact method additions
         if isinstance(s_newvalue, SomePBC):
             attr = self.name
             if s_newvalue.getKind() == MethodDesc:
                 # is method
                 if homedef.classdesc.read_attribute(attr, None) is None:
-                    homedef.check_missing_attribute_update(attr)
+                    if not homedef.check_missing_attribute_update(attr):
+                        for desc in s_newvalue.descriptions:
+                            if desc.selfclassdef is None:
+                                raise AnnotatorError(
+                                    "demoting method %s from %s to class "
+                                    "%s not allowed" % (self.name, desc.originclassdef, homedef)
+                                )
 
         # check for attributes forbidden by slots or _attrs_
         if homedef.classdesc.all_enforced_attrs is not None:
@@ -489,6 +496,7 @@
     knowntype = type
     instance_level = False
     all_enforced_attrs = None   # or a set
+    settled = False
     _detect_invalid_attrs = None
 
     def __init__(self, bookkeeper, cls,
@@ -559,6 +567,9 @@
         if base is not object:
             self.basedesc = bookkeeper.getdesc(base)
 
+        if '_settled_' in cls.__dict__:
+            self.settled = bool(cls.__dict__['_settled_'])
+
         if '__slots__' in cls.__dict__ or '_attrs_' in cls.__dict__:
             attrs = {}
             for decl in ('__slots__', '_attrs_'):
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -2374,7 +2374,8 @@
     def test_stored_bound_method_2(self):
         # issue 129
         class H:
-            pass
+            def h(self):
+                raise NotImplementedError("abstract method")
         class H1(H):
             def h(self):
                 return 42
@@ -3049,6 +3050,7 @@
     def test_slots_check(self):
         class Base(object):
             __slots__ = 'x'
+            def m(self): raise NotImplementedError("abstract")
         class A(Base):
             __slots__ = 'y'
             def m(self):
@@ -3098,6 +3100,7 @@
     def test_enforced_attrs_check(self):
         class Base(object):
             _attrs_ = 'x'
+            def m(self): raise NotImplementedError("abstract")
         class A(Base):
             _attrs_ = 'y'
             def m(self):
@@ -3167,6 +3170,45 @@
         a = self.RPythonAnnotator()
         a.build_types(f, [bool])
 
+    def test_enforce_settled(self):
+        class A(object):
+            _settled_ = True
+
+            def m(self):
+                raise NotImplementedError
+
+        class B(A):
+
+            def m(self):
+                return 1
+
+            def n(self):
+                return 1
+
+        def fun(x):
+            if x:
+                a = A()
+            else:
+                a = B()
+
+            return a.m()
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(fun, [bool])
+        assert s.knowntype == int
+
+        def fun(x):
+            if x:
+                a = A()
+            else:
+                a = B()
+
+            return a.n()
+
+        a = self.RPythonAnnotator()
+        with py.test.raises(AnnotatorError):
+            a.build_types(fun, [bool])
+
     def test_float_cmp(self):
         def fun(x, y):
             return (x < y,
diff --git a/rpython/translator/test/snippet.py b/rpython/translator/test/snippet.py
--- a/rpython/translator/test/snippet.py
+++ b/rpython/translator/test/snippet.py
@@ -375,7 +375,9 @@
     return _getstuff(d), _getstuff(e)
 
 class F:
-    pass
+    def m(self, x):
+        raise NotImplementedError("abstract base")
+
 class G(F):
     def m(self, x):
         return self.m2(x)

From pypy.commits at gmail.com  Mon May 21 10:04:00 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:04:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: start adding missing abstract methods
Message-ID: <5b02d1d0.1c69fb81.649e4.e7b1@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94627:bf0a01ba0389
Date: 2018-05-17 23:05 +0200
http://bitbucket.org/pypy/pypy/changeset/bf0a01ba0389/

Log:	start adding missing abstract methods

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -39,6 +39,9 @@
     def mutate_over(self, visitor):
         raise AssertionError("mutate_over() implementation not provided")
 
+    def to_object(self, space):
+        raise NotImplementedError("abstract base class")
+
 
 class NodeVisitorNotImplemented(Exception):
     pass
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -435,6 +435,9 @@
     def mutate_over(self, visitor):
         raise AssertionError("mutate_over() implementation not provided")
 
+    def to_object(self, space):
+        raise NotImplementedError("abstract base class")
+
 
 class NodeVisitorNotImplemented(Exception):
     pass
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -33,6 +33,7 @@
     __slots__ = ('__weakref__',)
     _must_be_light_finalizer_ = True
     user_overridden_class = False
+    _settled_ = True
 
     def getdict(self, space):
         return None
@@ -197,6 +198,8 @@
     # hooks that the mapdict implementations needs:
     def _get_mapdict_map(self):
         return None
+    def _mapdict_init_empty(self, terminator):
+        return None
     def _set_mapdict_map(self, map):
         raise NotImplementedError
     def _mapdict_read_storage(self, index):
@@ -913,9 +916,11 @@
         """Unpack an iterable into a real (interpreter-level) list.
 
         Raise an OperationError(w_ValueError) if the length is wrong."""
+        from pypy.interpreter.generator import GeneratorIterator
         w_iterator = self.iter(w_iterable)
         if expected_length == -1:
             if self.is_generator(w_iterator):
+                assert isinstance(w_iterator, GeneratorIterator)
                 # special hack for speed
                 lst_w = []
                 w_iterator.unpack_into(lst_w)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -301,6 +301,7 @@
         """
         strategy = self.get_strategy()
         strategy.move_to_end(self, w_key, last_flag)
+            # XXX this should be in the default move_to_end method!
 
     def nondescr_popitem_first(self, space):
         """Not exposed directly to app-level, but via __pypy__.popitem_first().
@@ -498,6 +499,9 @@
     def get_empty_storage(self):
         raise NotImplementedError
 
+    def switch_to_object_strategy(self, w_dict):
+        raise NotImplementedError
+
     @jit.look_inside_iff(lambda self, w_dict:
                          w_dict_unrolling_heuristic(w_dict))
     def w_keys(self, w_dict):
@@ -584,6 +588,36 @@
     def prepare_update(self, w_dict, num_extra):
         pass
 
+    def length(self, w_dict):
+        raise NotImplementedError
+
+    def getitem(self, w_dict, w_key):
+        raise NotImplementedError
+
+    def getitem_str(self, w_dict, key):
+        raise NotImplementedError
+
+    def setitem(self, w_dict, w_key, w_value):
+        raise NotImplementedError
+
+    def setitem_str(self, w_dict, key, w_value):
+        raise NotImplementedError
+
+    def delitem(self, w_dict, w_key):
+        raise NotImplementedError
+
+    def setdefault(self, w_dict, w_key, w_default):
+        raise NotImplementedError
+
+    def iterkeys(self, w_dict):
+        raise NotImplementedError
+
+    def itervalues(self, w_dict):
+        raise NotImplementedError
+
+    def iteritems(self, w_dict):
+        raise NotImplementedError
+
     def move_to_end(self, w_dict, w_key, last_flag):
         # fall-back
         w_value = w_dict.getitem(w_key)
@@ -807,12 +841,22 @@
 class BaseKeyIterator(BaseIteratorImplementation):
     next_key = _new_next('key')
 
+    def next_key_entry(self):
+        raise NotImplementedError
+
+
 class BaseValueIterator(BaseIteratorImplementation):
     next_value = _new_next('value')
 
+    def next_value_entry(self):
+        raise NotImplementedError
+
 class BaseItemIterator(BaseIteratorImplementation):
     next_item = _new_next('item')
 
+    def next_item_entry(self):
+        raise NotImplementedError
+
 
 def create_iterator_classes(dictimpl):
     if not hasattr(dictimpl, 'wrapkey'):
@@ -1447,6 +1491,7 @@
 class W_DictMultiIterKeysObject(W_BaseDictMultiIterObject):
     def descr_next(self, space):
         iteratorimplementation = self.iteratorimplementation
+        assert isinstance(iteratorimplementation, BaseKeyIterator)
         w_key = iteratorimplementation.next_key()
         if w_key is not None:
             return w_key
@@ -1455,6 +1500,7 @@
 class W_DictMultiIterValuesObject(W_BaseDictMultiIterObject):
     def descr_next(self, space):
         iteratorimplementation = self.iteratorimplementation
+        assert isinstance(iteratorimplementation, BaseValueIterator)
         w_value = iteratorimplementation.next_value()
         if w_value is not None:
             return w_value
@@ -1463,6 +1509,7 @@
 class W_DictMultiIterItemsObject(W_BaseDictMultiIterObject):
     def descr_next(self, space):
         iteratorimplementation = self.iteratorimplementation
+        assert isinstance(iteratorimplementation, BaseItemIterator)
         w_key, w_value = iteratorimplementation.next_item()
         if w_key is not None:
             return space.newtuple([w_key, w_value])
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -808,6 +808,9 @@
     def getitems(self, w_list):
         return self.getitems_copy(w_list)
 
+    def getitems_fixedsize(self, w_list):
+        raise NotImplementedError
+
     def getitems_copy(self, w_list):
         raise NotImplementedError
 
@@ -856,11 +859,13 @@
         raise NotImplementedError
 
     def extend(self, w_list, w_any):
+        from pypy.interpreter.generator import GeneratorIterator
         space = self.space
         if type(w_any) is W_ListObject or (isinstance(w_any, W_ListObject) and
                                            space._uses_list_iter(w_any)):
             self._extend_from_list(w_list, w_any)
         elif space.is_generator(w_any):
+            assert isinstance(w_any, GeneratorIterator)
             w_any.unpack_into_w(w_list)
         else:
             self._extend_from_iterable(w_list, w_any)

From pypy.commits at gmail.com  Mon May 21 10:04:02 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:04:02 -0700 (PDT)
Subject: [pypy-commit] pypy default: more base methods
Message-ID: <5b02d1d2.1c69fb81.18faa.e3f6@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94628:f0f9cfefd069
Date: 2018-05-18 10:42 +0200
http://bitbucket.org/pypy/pypy/changeset/f0f9cfefd069/

Log:	more base methods

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -769,6 +769,9 @@
     def __init__(self, space):
         self.space = space
 
+    def get_empty_storage(self, sizehint):
+        raise NotImplementedError
+
     def get_sizehint(self):
         return -1
 
@@ -826,6 +829,9 @@
     def getitems_float(self, w_list):
         return None
 
+    def getitems_unroll(self, w_list):
+        raise NotImplementedError
+
     def getstorage_copy(self, w_list):
         raise NotImplementedError
 
@@ -1080,6 +1086,16 @@
         strategy = w_list.strategy = self.space.fromcache(IntegerListStrategy)
         w_list.lstorage = strategy.erase(items)
 
+    def step(self, w_list):
+        raise NotImplementedError
+
+    def _getitems_range(self, w_list, wrap_items):
+        raise NotImplementedError
+    _getitems_range_unroll = _getitems_range
+
+    def _getitem_range_unwrapped(self, w_list, i):
+        raise NotImplementedError
+
     def wrap(self, intval):
         return self.space.newint(intval)
 
@@ -1104,7 +1120,7 @@
         w_other.lstorage = w_list.lstorage
 
     def getitem(self, w_list, i):
-        return self.wrap(self._getitem_unwrapped(w_list, i))
+        return self.wrap(self._getitem_range_unwrapped(w_list, i))
 
     def getitems_int(self, w_list):
         return self._getitems_range(w_list, False)
@@ -1194,7 +1210,7 @@
     def step(self, w_list):
         return 1
 
-    def _getitem_unwrapped(self, w_list, i):
+    def _getitem_range_unwrapped(self, w_list, i):
         length = self.unerase(w_list.lstorage)[0]
         if i < 0:
             i += length
@@ -1272,7 +1288,7 @@
     def step(self, w_list):
         return self.unerase(w_list.lstorage)[1]
 
-    def _getitem_unwrapped(self, w_list, i):
+    def _getitem_range_unwrapped(self, w_list, i):
         v = self.unerase(w_list.lstorage)
         start = v[0]
         step = v[1]
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -31,6 +31,9 @@
         reprlist = [repr(w_item) for w_item in self.getkeys()]
         return "<%s(%s)(%s)>" % (self.__class__.__name__, self.strategy, ', '.join(reprlist))
 
+    def _newobj(self, space, w_iterable):
+        raise NotImplementedError
+
     def from_storage_and_strategy(self, storage, strategy):
         obj = self._newobj(self.space, None)
         assert isinstance(obj, W_BaseSetObject)
@@ -702,6 +705,12 @@
     #def unerase(self, storage):
     #    raise NotImplementedError
 
+    def may_contain_equal_elements(self, strategy):
+        return True
+
+    def _intersect_wrapped(self, w_set, w_other):
+        raise NotImplementedError
+
     # __________________ methods called on W_SetObject _________________
 
     def clear(self, w_set):
@@ -1622,8 +1631,8 @@
         if type(w_item) is not W_IntObject:
             break
     else:
-        w_set.strategy = space.fromcache(IntegerSetStrategy)
-        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        strategy = w_set.strategy = space.fromcache(IntegerSetStrategy)
+        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
         return
 
     # check for strings
@@ -1631,8 +1640,8 @@
         if type(w_item) is not W_BytesObject:
             break
     else:
-        w_set.strategy = space.fromcache(BytesSetStrategy)
-        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        strategy = w_set.strategy = space.fromcache(BytesSetStrategy)
+        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
         return
 
     # check for unicode
@@ -1640,8 +1649,8 @@
         if type(w_item) is not W_UnicodeObject:
             break
     else:
-        w_set.strategy = space.fromcache(UnicodeSetStrategy)
-        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        strategy = w_set.strategy = space.fromcache(UnicodeSetStrategy)
+        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
         return
 
     # check for compares by identity
@@ -1649,12 +1658,12 @@
         if not space.type(w_item).compares_by_identity():
             break
     else:
-        w_set.strategy = space.fromcache(IdentitySetStrategy)
-        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        strategy = w_set.strategy = space.fromcache(IdentitySetStrategy)
+        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
         return
 
-    w_set.strategy = space.fromcache(ObjectSetStrategy)
-    w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+    strategy = w_set.strategy = space.fromcache(ObjectSetStrategy)
+    w_set.sstorage = strategy.get_storage_from_list(iterable_w)
 
 
 create_set_driver = jit.JitDriver(name='create_set',
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -112,10 +112,14 @@
                     if not homedef.check_missing_attribute_update(attr):
                         for desc in s_newvalue.descriptions:
                             if desc.selfclassdef is None:
-                                raise AnnotatorError(
+                                print AnnotatorError(
                                     "demoting method %s from %s to class "
-                                    "%s not allowed" % (self.name, desc.originclassdef, homedef)
+                                    "%s not allowed\n" % (self.name, desc.originclassdef, homedef) +
+                                    "either you need to add an abstract method to the base class\n" +
+                                    "or you need an assert isinstance(...) to ensure the annotator " +
+                                    "that the instance is of the right class"
                                 )
+                                break
 
         # check for attributes forbidden by slots or _attrs_
         if homedef.classdesc.all_enforced_attrs is not None:
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -103,6 +103,9 @@
             self.seen = AddressDict()
         self.pending = AddressStack()
 
+    def processobj(self, obj):
+        raise NotImplementedError("abstract base class")
+
     def delete(self):
         if self.gcflag == 0:
             self.seen.delete()
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -57,6 +57,9 @@
         """Return the size in bytes."""
         raise NotImplementedError
 
+    def get_raw_address(self):
+        raise NotImplementedError
+
     def __len__(self):
         res = self.getlength()
         assert res >= 0
@@ -163,7 +166,7 @@
     def decorate(targetcls):
         """
         Create and attach specialized versions of typed_{read,write}. We need to
-        do this becase the JIT codewriters mandates that base_ofs is an
+        do this because the JIT codewriters mandates that base_ofs is an
         RPython constant.
         """
         if targetcls.__bases__ != (GCBuffer,):

From pypy.commits at gmail.com  Mon May 21 10:04:07 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:04:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge
Message-ID: <5b02d1d7.1c69fb81.a8e33.fbb0@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94630:622b5969fe73
Date: 2018-05-21 16:03 +0200
http://bitbucket.org/pypy/pypy/changeset/622b5969fe73/

Log:	merge

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -39,6 +39,9 @@
     def mutate_over(self, visitor):
         raise AssertionError("mutate_over() implementation not provided")
 
+    def to_object(self, space):
+        raise NotImplementedError("abstract base class")
+
 
 class NodeVisitorNotImplemented(Exception):
     pass
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -435,6 +435,9 @@
     def mutate_over(self, visitor):
         raise AssertionError("mutate_over() implementation not provided")
 
+    def to_object(self, space):
+        raise NotImplementedError("abstract base class")
+
 
 class NodeVisitorNotImplemented(Exception):
     pass
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -33,6 +33,7 @@
     __slots__ = ('__weakref__',)
     _must_be_light_finalizer_ = True
     user_overridden_class = False
+    _settled_ = True
 
     def getdict(self, space):
         return None
@@ -197,6 +198,8 @@
     # hooks that the mapdict implementations needs:
     def _get_mapdict_map(self):
         return None
+    def _mapdict_init_empty(self, terminator):
+        return None
     def _set_mapdict_map(self, map):
         raise NotImplementedError
     def _mapdict_read_storage(self, index):
@@ -913,9 +916,11 @@
         """Unpack an iterable into a real (interpreter-level) list.
 
         Raise an OperationError(w_ValueError) if the length is wrong."""
+        from pypy.interpreter.generator import GeneratorIterator
         w_iterator = self.iter(w_iterable)
         if expected_length == -1:
             if self.is_generator(w_iterator):
+                assert isinstance(w_iterator, GeneratorIterator)
                 # special hack for speed
                 lst_w = []
                 w_iterator.unpack_into(lst_w)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -301,6 +301,7 @@
         """
         strategy = self.get_strategy()
         strategy.move_to_end(self, w_key, last_flag)
+            # XXX this should be in the default move_to_end method!
 
     def nondescr_popitem_first(self, space):
         """Not exposed directly to app-level, but via __pypy__.popitem_first().
@@ -498,6 +499,9 @@
     def get_empty_storage(self):
         raise NotImplementedError
 
+    def switch_to_object_strategy(self, w_dict):
+        raise NotImplementedError
+
     @jit.look_inside_iff(lambda self, w_dict:
                          w_dict_unrolling_heuristic(w_dict))
     def w_keys(self, w_dict):
@@ -584,6 +588,36 @@
     def prepare_update(self, w_dict, num_extra):
         pass
 
+    def length(self, w_dict):
+        raise NotImplementedError
+
+    def getitem(self, w_dict, w_key):
+        raise NotImplementedError
+
+    def getitem_str(self, w_dict, key):
+        raise NotImplementedError
+
+    def setitem(self, w_dict, w_key, w_value):
+        raise NotImplementedError
+
+    def setitem_str(self, w_dict, key, w_value):
+        raise NotImplementedError
+
+    def delitem(self, w_dict, w_key):
+        raise NotImplementedError
+
+    def setdefault(self, w_dict, w_key, w_default):
+        raise NotImplementedError
+
+    def iterkeys(self, w_dict):
+        raise NotImplementedError
+
+    def itervalues(self, w_dict):
+        raise NotImplementedError
+
+    def iteritems(self, w_dict):
+        raise NotImplementedError
+
     def move_to_end(self, w_dict, w_key, last_flag):
         # fall-back
         w_value = w_dict.getitem(w_key)
@@ -807,12 +841,22 @@
 class BaseKeyIterator(BaseIteratorImplementation):
     next_key = _new_next('key')
 
+    def next_key_entry(self):
+        raise NotImplementedError
+
+
 class BaseValueIterator(BaseIteratorImplementation):
     next_value = _new_next('value')
 
+    def next_value_entry(self):
+        raise NotImplementedError
+
 class BaseItemIterator(BaseIteratorImplementation):
     next_item = _new_next('item')
 
+    def next_item_entry(self):
+        raise NotImplementedError
+
 
 def create_iterator_classes(dictimpl):
     if not hasattr(dictimpl, 'wrapkey'):
@@ -1447,6 +1491,7 @@
 class W_DictMultiIterKeysObject(W_BaseDictMultiIterObject):
     def descr_next(self, space):
         iteratorimplementation = self.iteratorimplementation
+        assert isinstance(iteratorimplementation, BaseKeyIterator)
         w_key = iteratorimplementation.next_key()
         if w_key is not None:
             return w_key
@@ -1455,6 +1500,7 @@
 class W_DictMultiIterValuesObject(W_BaseDictMultiIterObject):
     def descr_next(self, space):
         iteratorimplementation = self.iteratorimplementation
+        assert isinstance(iteratorimplementation, BaseValueIterator)
         w_value = iteratorimplementation.next_value()
         if w_value is not None:
             return w_value
@@ -1463,6 +1509,7 @@
 class W_DictMultiIterItemsObject(W_BaseDictMultiIterObject):
     def descr_next(self, space):
         iteratorimplementation = self.iteratorimplementation
+        assert isinstance(iteratorimplementation, BaseItemIterator)
         w_key, w_value = iteratorimplementation.next_item()
         if w_key is not None:
             return space.newtuple([w_key, w_value])
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -769,6 +769,9 @@
     def __init__(self, space):
         self.space = space
 
+    def get_empty_storage(self, sizehint):
+        raise NotImplementedError
+
     def get_sizehint(self):
         return -1
 
@@ -808,6 +811,9 @@
     def getitems(self, w_list):
         return self.getitems_copy(w_list)
 
+    def getitems_fixedsize(self, w_list):
+        raise NotImplementedError
+
     def getitems_copy(self, w_list):
         raise NotImplementedError
 
@@ -823,6 +829,9 @@
     def getitems_float(self, w_list):
         return None
 
+    def getitems_unroll(self, w_list):
+        raise NotImplementedError
+
     def getstorage_copy(self, w_list):
         raise NotImplementedError
 
@@ -856,11 +865,13 @@
         raise NotImplementedError
 
     def extend(self, w_list, w_any):
+        from pypy.interpreter.generator import GeneratorIterator
         space = self.space
         if type(w_any) is W_ListObject or (isinstance(w_any, W_ListObject) and
                                            space._uses_list_iter(w_any)):
             self._extend_from_list(w_list, w_any)
         elif space.is_generator(w_any):
+            assert isinstance(w_any, GeneratorIterator)
             w_any.unpack_into_w(w_list)
         else:
             self._extend_from_iterable(w_list, w_any)
@@ -1075,6 +1086,16 @@
         strategy = w_list.strategy = self.space.fromcache(IntegerListStrategy)
         w_list.lstorage = strategy.erase(items)
 
+    def step(self, w_list):
+        raise NotImplementedError
+
+    def _getitems_range(self, w_list, wrap_items):
+        raise NotImplementedError
+    _getitems_range_unroll = _getitems_range
+
+    def _getitem_range_unwrapped(self, w_list, i):
+        raise NotImplementedError
+
     def wrap(self, intval):
         return self.space.newint(intval)
 
@@ -1099,7 +1120,7 @@
         w_other.lstorage = w_list.lstorage
 
     def getitem(self, w_list, i):
-        return self.wrap(self._getitem_unwrapped(w_list, i))
+        return self.wrap(self._getitem_range_unwrapped(w_list, i))
 
     def getitems_int(self, w_list):
         return self._getitems_range(w_list, False)
@@ -1189,7 +1210,7 @@
     def step(self, w_list):
         return 1
 
-    def _getitem_unwrapped(self, w_list, i):
+    def _getitem_range_unwrapped(self, w_list, i):
         length = self.unerase(w_list.lstorage)[0]
         if i < 0:
             i += length
@@ -1267,7 +1288,7 @@
     def step(self, w_list):
         return self.unerase(w_list.lstorage)[1]
 
-    def _getitem_unwrapped(self, w_list, i):
+    def _getitem_range_unwrapped(self, w_list, i):
         v = self.unerase(w_list.lstorage)
         start = v[0]
         step = v[1]
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -31,6 +31,9 @@
         reprlist = [repr(w_item) for w_item in self.getkeys()]
         return "<%s(%s)(%s)>" % (self.__class__.__name__, self.strategy, ', '.join(reprlist))
 
+    def _newobj(self, space, w_iterable):
+        raise NotImplementedError
+
     def from_storage_and_strategy(self, storage, strategy):
         obj = self._newobj(self.space, None)
         assert isinstance(obj, W_BaseSetObject)
@@ -702,6 +705,12 @@
     #def unerase(self, storage):
     #    raise NotImplementedError
 
+    def may_contain_equal_elements(self, strategy):
+        return True
+
+    def _intersect_wrapped(self, w_set, w_other):
+        raise NotImplementedError
+
     # __________________ methods called on W_SetObject _________________
 
     def clear(self, w_set):
@@ -1622,8 +1631,8 @@
         if type(w_item) is not W_IntObject:
             break
     else:
-        w_set.strategy = space.fromcache(IntegerSetStrategy)
-        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        strategy = w_set.strategy = space.fromcache(IntegerSetStrategy)
+        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
         return
 
     # check for strings
@@ -1631,8 +1640,8 @@
         if type(w_item) is not W_BytesObject:
             break
     else:
-        w_set.strategy = space.fromcache(BytesSetStrategy)
-        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        strategy = w_set.strategy = space.fromcache(BytesSetStrategy)
+        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
         return
 
     # check for unicode
@@ -1640,8 +1649,8 @@
         if type(w_item) is not W_UnicodeObject:
             break
     else:
-        w_set.strategy = space.fromcache(UnicodeSetStrategy)
-        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        strategy = w_set.strategy = space.fromcache(UnicodeSetStrategy)
+        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
         return
 
     # check for compares by identity
@@ -1649,12 +1658,12 @@
         if not space.type(w_item).compares_by_identity():
             break
     else:
-        w_set.strategy = space.fromcache(IdentitySetStrategy)
-        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        strategy = w_set.strategy = space.fromcache(IdentitySetStrategy)
+        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
         return
 
-    w_set.strategy = space.fromcache(ObjectSetStrategy)
-    w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+    strategy = w_set.strategy = space.fromcache(ObjectSetStrategy)
+    w_set.sstorage = strategy.get_storage_from_list(iterable_w)
 
 
 create_set_driver = jit.JitDriver(name='create_set',
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -102,13 +102,24 @@
 
     def validate(self, homedef):
         s_newvalue = self.s_value
-        # check for after-the-fact method additions
+        homedesc = homedef.classdesc
+        # check for method demotion and after-the-fact method additions
         if isinstance(s_newvalue, SomePBC):
             attr = self.name
             if s_newvalue.getKind() == MethodDesc:
                 # is method
                 if homedef.classdesc.read_attribute(attr, None) is None:
-                    homedef.check_missing_attribute_update(attr)
+                    if not homedef.check_missing_attribute_update(attr):
+                        for desc in s_newvalue.descriptions:
+                            if desc.selfclassdef is None:
+                                print AnnotatorError(
+                                    "demoting method %s from %s to class "
+                                    "%s not allowed\n" % (self.name, desc.originclassdef, homedef) +
+                                    "either you need to add an abstract method to the base class\n" +
+                                    "or you need an assert isinstance(...) to ensure the annotator " +
+                                    "that the instance is of the right class"
+                                )
+                                break
 
         # check for attributes forbidden by slots or _attrs_
         if homedef.classdesc.all_enforced_attrs is not None:
@@ -489,6 +500,7 @@
     knowntype = type
     instance_level = False
     all_enforced_attrs = None   # or a set
+    settled = False
     _detect_invalid_attrs = None
 
     def __init__(self, bookkeeper, cls,
@@ -559,6 +571,9 @@
         if base is not object:
             self.basedesc = bookkeeper.getdesc(base)
 
+        if '_settled_' in cls.__dict__:
+            self.settled = bool(cls.__dict__['_settled_'])
+
         if '__slots__' in cls.__dict__ or '_attrs_' in cls.__dict__:
             attrs = {}
             for decl in ('__slots__', '_attrs_'):
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -2374,7 +2374,8 @@
     def test_stored_bound_method_2(self):
         # issue 129
         class H:
-            pass
+            def h(self):
+                raise NotImplementedError("abstract method")
         class H1(H):
             def h(self):
                 return 42
@@ -3049,6 +3050,7 @@
     def test_slots_check(self):
         class Base(object):
             __slots__ = 'x'
+            def m(self): raise NotImplementedError("abstract")
         class A(Base):
             __slots__ = 'y'
             def m(self):
@@ -3098,6 +3100,7 @@
     def test_enforced_attrs_check(self):
         class Base(object):
             _attrs_ = 'x'
+            def m(self): raise NotImplementedError("abstract")
         class A(Base):
             _attrs_ = 'y'
             def m(self):
@@ -3167,6 +3170,45 @@
         a = self.RPythonAnnotator()
         a.build_types(f, [bool])
 
+    def test_enforce_settled(self):
+        class A(object):
+            _settled_ = True
+
+            def m(self):
+                raise NotImplementedError
+
+        class B(A):
+
+            def m(self):
+                return 1
+
+            def n(self):
+                return 1
+
+        def fun(x):
+            if x:
+                a = A()
+            else:
+                a = B()
+
+            return a.m()
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(fun, [bool])
+        assert s.knowntype == int
+
+        def fun(x):
+            if x:
+                a = A()
+            else:
+                a = B()
+
+            return a.n()
+
+        a = self.RPythonAnnotator()
+        with py.test.raises(AnnotatorError):
+            a.build_types(fun, [bool])
+
     def test_float_cmp(self):
         def fun(x, y):
             return (x < y,
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -103,6 +103,9 @@
             self.seen = AddressDict()
         self.pending = AddressStack()
 
+    def processobj(self, obj):
+        raise NotImplementedError("abstract base class")
+
     def delete(self):
         if self.gcflag == 0:
             self.seen.delete()
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -57,6 +57,9 @@
         """Return the size in bytes."""
         raise NotImplementedError
 
+    def get_raw_address(self):
+        raise NotImplementedError
+
     def __len__(self):
         res = self.getlength()
         assert res >= 0
@@ -163,7 +166,7 @@
     def decorate(targetcls):
         """
         Create and attach specialized versions of typed_{read,write}. We need to
-        do this becase the JIT codewriters mandates that base_ofs is an
+        do this because the JIT codewriters mandates that base_ofs is an
         RPython constant.
         """
         if targetcls.__bases__ != (GCBuffer,):
diff --git a/rpython/translator/test/snippet.py b/rpython/translator/test/snippet.py
--- a/rpython/translator/test/snippet.py
+++ b/rpython/translator/test/snippet.py
@@ -375,7 +375,9 @@
     return _getstuff(d), _getstuff(e)
 
 class F:
-    pass
+    def m(self, x):
+        raise NotImplementedError("abstract base")
+
 class G(F):
     def m(self, x):
         return self.m2(x)

From pypy.commits at gmail.com  Mon May 21 10:04:04 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:04:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: don't fail when there is no JIT
Message-ID: <5b02d1d4.1c69fb81.c398f.17a2@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94629:340a8bb00e6c
Date: 2018-05-21 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/340a8bb00e6c/

Log:	don't fail when there is no JIT

diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -139,7 +139,10 @@
         cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
 
     def test_jit_backend_features(self):
-        from __pypy__ import jit_backend_features
+        try:
+            from __pypy__ import jit_backend_features
+        except ImportError:
+            skip("compiled without jit")
         supported_types = jit_backend_features
         assert isinstance(supported_types, list)
         for x in supported_types:

From pypy.commits at gmail.com  Mon May 21 10:07:46 2018
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 21 May 2018 07:07:46 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: experimental branch in which you can
 enable/disable/force major collections from applevel
Message-ID: <5b02d2b2.1c69fb81.59d1e.6c44@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94631:e5d3101ab418
Date: 2018-05-16 18:27 +0200
http://bitbucket.org/pypy/pypy/changeset/e5d3101ab418/

Log:	experimental branch in which you can enable/disable/force major
	collections from applevel


From pypy.commits at gmail.com  Mon May 21 10:07:48 2018
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 21 May 2018 07:07:48 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: introduce the debuglog fixture,
 so that tests can easily capture the content of debug_{start, stop, print}
Message-ID: <5b02d2b4.1c69fb81.1bfbd.4fda@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94632:c892bfb192d6
Date: 2018-05-18 18:35 +0200
http://bitbucket.org/pypy/pypy/changeset/c892bfb192d6/

Log:	introduce the debuglog fixture, so that tests can easily capture the
	content of debug_{start,stop,print}

diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -37,6 +37,10 @@
         assert False, ("nesting error: no start corresponding to stop %r" %
                        (category,))
 
+    def reset(self):
+        # only for tests: empty the log
+        self[:] = []
+
     def __repr__(self):
         import pprint
         return pprint.pformat(list(self))
diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py
--- a/rpython/rlib/test/test_debug.py
+++ b/rpython/rlib/test/test_debug.py
@@ -1,5 +1,5 @@
-
 import py
+import pytest
 from rpython.rlib.debug import (check_annotation, make_sure_not_resized,
                              debug_print, debug_start, debug_stop,
                              have_debug_prints, debug_offset, debug_flush,
@@ -10,6 +10,12 @@
 from rpython.rlib import debug
 from rpython.rtyper.test.test_llinterp import interpret, gengraph
 
+ at pytest.fixture
+def debuglog(monkeypatch):
+    dlog = debug.DebugLog()
+    monkeypatch.setattr(debug, '_log', dlog)
+    return dlog
+
 def test_check_annotation():
     class Error(Exception):
         pass
@@ -94,7 +100,7 @@
     py.test.raises(NotAListOfChars, "interpret(g, [3])")
 
 
-def test_debug_print_start_stop():
+def test_debug_print_start_stop(debuglog):
     def f(x):
         debug_start("mycat")
         debug_print("foo", 2, "bar", x)
@@ -103,21 +109,14 @@
         debug_offset()  # should not explode at least
         return have_debug_prints()
 
-    try:
-        debug._log = dlog = debug.DebugLog()
-        res = f(3)
-        assert res is True
-    finally:
-        debug._log = None
-    assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
+    res = f(3)
+    assert res is True
+    assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
+    debuglog.reset()
 
-    try:
-        debug._log = dlog = debug.DebugLog()
-        res = interpret(f, [3])
-        assert res is True
-    finally:
-        debug._log = None
-    assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
+    res = interpret(f, [3])
+    assert res is True
+    assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
 
 
 def test_debug_start_stop_timestamp():

From pypy.commits at gmail.com  Mon May 21 10:07:50 2018
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 21 May 2018 07:07:50 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: add debuglog.summary(),
 useful for tests
Message-ID: <5b02d2b6.1c69fb81.56a95.95bb@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94633:4eb5b98b9eb0
Date: 2018-05-19 00:30 +0200
http://bitbucket.org/pypy/pypy/changeset/4eb5b98b9eb0/

Log:	add debuglog.summary(), useful for tests

diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -1,5 +1,6 @@
 import sys
 import time
+from collections import Counter
 
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rlib.objectmodel import we_are_translated, always_inline
@@ -41,6 +42,19 @@
         # only for tests: empty the log
         self[:] = []
 
+    def summary(self, flatten=False):
+        res = Counter()
+        def visit(lst):
+            for section, sublist in lst:
+                if section == 'debug_print':
+                    continue
+                res[section] += 1
+                if flatten:
+                    visit(sublist)
+        #
+        visit(self)
+        return res
+
     def __repr__(self):
         import pprint
         return pprint.pformat(list(self))
diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py
--- a/rpython/rlib/test/test_debug.py
+++ b/rpython/rlib/test/test_debug.py
@@ -118,6 +118,18 @@
     assert res is True
     assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
 
+def test_debuglog_summary(debuglog):
+    debug_start('foo')
+    debug_start('bar') # this is nested, so not counted in the summary by default
+    debug_stop('bar')
+    debug_stop('foo')
+    debug_start('foo')
+    debug_stop('foo')
+    debug_start('bar')
+    debug_stop('bar')
+    #
+    assert debuglog.summary() == {'foo': 2, 'bar': 1}
+    assert debuglog.summary(flatten=True) == {'foo': 2, 'bar': 2}
 
 def test_debug_start_stop_timestamp():
     import time

From pypy.commits at gmail.com  Mon May 21 10:07:55 2018
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 21 May 2018 07:07:55 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: make sure that gc.collect() works
 even when the GC is disabled
Message-ID: <5b02d2bb.1c69fb81.5229b.bc34@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94635:5635dcf404cf
Date: 2018-05-19 11:37 +0200
http://bitbucket.org/pypy/pypy/changeset/5635dcf404cf/

Log:	make sure that gc.collect() works even when the GC is disabled

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -760,7 +760,7 @@
         if gen < 0:
             self._minor_collection()   # dangerous! no major GC cycle progress
         elif gen <= 1:
-            self.minor_collection_with_major_progress()
+            self.minor_collection_with_major_progress(force_enabled=True)
             if gen == 1 and self.gc_state == STATE_SCANNING:
                 self.major_collection_step()
         else:
@@ -768,14 +768,15 @@
         self.rrc_invoke_callback()
 
 
-    def minor_collection_with_major_progress(self, extrasize=0):
+    def minor_collection_with_major_progress(self, extrasize=0,
+                                             force_enabled=False):
         """Do a minor collection.  Then, if the GC is enabled and there
         is already a major GC in progress, run at least one major collection
         step.  If there is no major GC but the threshold is reached, start a
         major GC.
         """
         self._minor_collection()
-        if not self.enabled:
+        if not self.enabled and not force_enabled:
             return
 
         # If the gc_state is STATE_SCANNING, we're not in the middle
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -790,3 +790,28 @@
         self.gc.enable()
         summary = large_malloc()
         assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor']
+
+    def test_call_collect_when_disabled(self, debuglog):
+        # malloc an object and put it the old generation
+        s = self.malloc(S)
+        s.x = 42
+        self.stackroots.append(s)
+        self.gc.collect()
+        s = self.stackroots.pop()
+        #
+        self.gc.disable()
+        self.gc.collect(1) # start a major collect
+        assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor']
+        assert s.x == 42 # s is not freed yet
+        #
+        debuglog.reset()
+        self.gc.collect(1) # run one more step
+        assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor']
+        assert s.x == 42 # s is not freed yet
+        #
+        debuglog.reset()
+        self.gc.collect() # finish the major collection
+        summary = debuglog.summary()
+        assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor']
+        # s is freed
+        py.test.raises(RuntimeError, 's.x')

From pypy.commits at gmail.com  Mon May 21 10:07:53 2018
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 21 May 2018 07:07:53 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: WIP: introduce
 IncrementalMiniMarkGC.{enable,disable}(): when the GC is disabled, it runs
 only minor collections; major ones needs to be explictly triggered
Message-ID: <5b02d2b9.1c69fb81.a0222.d10c@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94634:48a6326917c6
Date: 2018-05-19 11:21 +0200
http://bitbucket.org/pypy/pypy/changeset/48a6326917c6/

Log:	WIP: introduce IncrementalMiniMarkGC.{enable,disable}(): when the GC
	is disabled, it runs only minor collections; major ones needs to be
	explictly triggered

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -376,6 +376,11 @@
         self.rawmalloced_peak_size = r_uint(0)
 
         self.gc_state = STATE_SCANNING
+
+        # if the GC is disabled, it runs only minor collections; major
+        # collections need to be manually triggered by explicitly calling
+        # collect()
+        self.enabled = True
         #
         # Two lists of all objects with finalizers.  Actually they are lists
         # of pairs (finalization_queue_nr, object).  "probably young objects"
@@ -511,6 +516,12 @@
             bigobj = self.nonlarge_max + 1
             self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2)
 
+    def enable(self):
+        self.enabled = True
+
+    def disable(self):
+        self.enabled = False
+
     def _nursery_memory_size(self):
         extra = self.nonlarge_max + 1
         return self.nursery_size + extra
@@ -758,11 +769,14 @@
 
 
     def minor_collection_with_major_progress(self, extrasize=0):
-        """Do a minor collection.  Then, if there is already a major GC
-        in progress, run at least one major collection step.  If there is
-        no major GC but the threshold is reached, start a major GC.
+        """Do a minor collection.  Then, if the GC is enabled and there
+        is already a major GC in progress, run at least one major collection
+        step.  If there is no major GC but the threshold is reached, start a
+        major GC.
         """
         self._minor_collection()
+        if not self.enabled:
+            return
 
         # If the gc_state is STATE_SCANNING, we're not in the middle
         # of an incremental major collection.  In that case, wait
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -13,6 +13,7 @@
 from rpython.memory.gc import minimark, incminimark
 from rpython.memory.gctypelayout import zero_gc_pointers_inside, zero_gc_pointers
 from rpython.rlib.debug import debug_print
+from rpython.rlib.test.test_debug import debuglog
 import pdb
 WORD = LONG_BIT // 8
 
@@ -770,4 +771,22 @@
                 assert elem.prev == lltype.nullptr(S)
                 assert elem.next == lltype.nullptr(S)
 
-            
+    def test_enable_disable(self, debuglog):
+        def large_malloc():
+            # malloc an object which is large enough to trigger a major collection
+            threshold = self.gc.next_major_collection_threshold
+            self.malloc(VAR, int(threshold/8))
+            summary = debuglog.summary()
+            debuglog.reset()
+            return summary
+        #
+        summary = large_malloc()
+        assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor']
+        #
+        self.gc.disable()
+        summary = large_malloc()
+        assert sorted(summary.keys()) == ['gc-minor']
+        #
+        self.gc.enable()
+        summary = large_malloc()
+        assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor']

From pypy.commits at gmail.com  Mon May 21 10:07:59 2018
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 21 May 2018 07:07:59 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: implement the gctransoform and C
 backend part of rgc.{enable, disable, isenabled}
Message-ID: <5b02d2bf.1c69fb81.a7fb8.5565@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94637:a9e0a0777f86
Date: 2018-05-21 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/a9e0a0777f86/

Log:	implement the gctransoform and C backend part of
	rgc.{enable,disable,isenabled}

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -149,6 +149,16 @@
     def get_size_incl_hash(self, obj):
         return self.get_size(obj)
 
+    # these can be overriden by subclasses, called by the GCTransformer
+    def enable(self):
+        pass
+
+    def disable(self):
+        pass
+
+    def isenabled(self):
+        return True
+
     def malloc(self, typeid, length=0, zero=False):
         """NOT_RPYTHON
         For testing.  The interface used by the gctransformer is
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -522,6 +522,9 @@
     def disable(self):
         self.enabled = False
 
+    def isenabled(self):
+        return self.enabled
+
     def _nursery_memory_size(self):
         extra = self.nonlarge_max + 1
         return self.nursery_size + extra
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -309,6 +309,10 @@
 
         self.collect_ptr = getfn(GCClass.collect.im_func,
             [s_gc, annmodel.SomeInteger()], annmodel.s_None)
+        self.enable_ptr = getfn(GCClass.enable.im_func, [s_gc], annmodel.s_None)
+        self.disable_ptr = getfn(GCClass.disable.im_func, [s_gc], annmodel.s_None)
+        self.isenabled_ptr = getfn(GCClass.isenabled.im_func, [s_gc],
+                                   annmodel.s_Bool)
         self.can_move_ptr = getfn(GCClass.can_move.im_func,
                                   [s_gc, SomeAddress()],
                                   annmodel.SomeBool())
@@ -884,6 +888,21 @@
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
+    def gct_gc__enable(self, hop):
+        op = hop.spaceop
+        hop.genop("direct_call", [self.enable_ptr, self.c_const_gc],
+                  resultvar=op.result)
+
+    def gct_gc__disable(self, hop):
+        op = hop.spaceop
+        hop.genop("direct_call", [self.disable_ptr, self.c_const_gc],
+                  resultvar=op.result)
+
+    def gct_gc__isenabled(self, hop):
+        op = hop.spaceop
+        hop.genop("direct_call", [self.isenabled_ptr, self.c_const_gc],
+                  resultvar=op.result)
+
     def gct_gc_can_move(self, hop):
         op = hop.spaceop
         v_addr = hop.genop('cast_ptr_to_adr',
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1812,6 +1812,45 @@
         res = self.run("ignore_finalizer")
         assert res == 1    # translated: x1 is removed from the list
 
+    def define_enable_disable(self):
+        class Counter(object):
+            val = 0
+        counter = Counter()
+        class X(object):
+            def __del__(self):
+                counter.val += 1
+        def f(should_disable):
+            x1 = X()
+            rgc.collect() # make x1 old
+            assert not rgc.can_move(x1)
+            x1 = None
+            #
+            if should_disable:
+                gc.disable()
+                assert not gc.isenabled()
+            # try to trigger a major collection
+            N = 100 # this should be enough, increase if not
+            lst = []
+            for i in range(N):
+                lst.append(chr(i%256) * (1024*1024))
+                #print i, counter.val
+            #
+            gc.enable()
+            assert gc.isenabled()
+            return counter.val
+        return f
+
+    def test_enable_disable(self):
+        # first, run with normal gc. If the assert fails it means that in the
+        # loop we don't allocate enough mem to trigger a major collection. Try
+        # to increase N
+        deleted = self.run("enable_disable", 0)
+        assert deleted == 1, 'This should not fail, try to increment N'
+        #
+        # now, run with gc.disable: this should NOT free x1
+        deleted = self.run("enable_disable", 1)
+        assert deleted == 0
+
 
 # ____________________________________________________________________
 

From pypy.commits at gmail.com  Mon May 21 10:07:57 2018
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 21 May 2018 07:07:57 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: implement the llinterp part of
 rgc.{enable, disable, isenabled}
Message-ID: <5b02d2bd.1c69fb81.d793e.e10a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94636:f7dbfd0f67ea
Date: 2018-05-21 11:46 +0200
http://bitbucket.org/pypy/pypy/changeset/f7dbfd0f67ea/

Log:	implement the llinterp part of rgc.{enable,disable,isenabled}

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -13,6 +13,9 @@
 # General GC features
 
 collect = gc.collect
+enable = gc.enable
+disable = gc.disable
+isenabled = gc.isenabled
 
 def set_max_heap_size(nbytes):
     """Limit the heap size to n bytes.
@@ -124,6 +127,32 @@
             args_v = hop.inputargs(lltype.Signed)
         return hop.genop('gc__collect', args_v, resulttype=hop.r_result)
 
+
+class EnableDisableEntry(ExtRegistryEntry):
+    _about_ = (gc.enable, gc.disable)
+
+    def compute_result_annotation(self):
+        from rpython.annotator import model as annmodel
+        return annmodel.s_None
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        opname = self.instance.__name__
+        return hop.genop('gc__%s' % opname, hop.args_v, resulttype=hop.r_result)
+
+
+class IsEnabledEntry(ExtRegistryEntry):
+    _about_ = gc.isenabled
+
+    def compute_result_annotation(self):
+        from rpython.annotator import model as annmodel
+        return annmodel.s_Bool
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        return hop.genop('gc__isenabled', hop.args_v, resulttype=hop.r_result)
+
+
 class SetMaxHeapSizeEntry(ExtRegistryEntry):
     _about_ = set_max_heap_size
 
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -39,6 +39,24 @@
 
     assert res is None
 
+def test_enable_disable():
+    def f():
+        gc.enable()
+        a = gc.isenabled()
+        gc.disable()
+        b = gc.isenabled()
+        return a and not b
+
+    t, typer, graph = gengraph(f, [])
+    blockops = list(graph.iterblockops())
+    opnames = [op.opname for block, op in blockops
+               if op.opname.startswith('gc__')]
+    assert opnames == ['gc__enable', 'gc__isenabled',
+                       'gc__disable', 'gc__isenabled']
+    res = interpret(f, [])
+    assert res
+
+
 def test_can_move():
     T0 = lltype.GcStruct('T')
     T1 = lltype.GcArray(lltype.Float)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -819,6 +819,15 @@
     def op_gc__collect(self, *gen):
         self.heap.collect(*gen)
 
+    def op_gc__enable(self):
+        self.heap.enable()
+
+    def op_gc__disable(self):
+        self.heap.disable()
+
+    def op_gc__isenabled(self):
+        return self.heap.isenabled()
+
     def op_gc_heap_stats(self):
         raise NotImplementedError
 
diff --git a/rpython/rtyper/lltypesystem/llheap.py b/rpython/rtyper/lltypesystem/llheap.py
--- a/rpython/rtyper/lltypesystem/llheap.py
+++ b/rpython/rtyper/lltypesystem/llheap.py
@@ -5,7 +5,7 @@
 
 setfield = setattr
 from operator import setitem as setarrayitem
-from rpython.rlib.rgc import can_move, collect, add_memory_pressure
+from rpython.rlib.rgc import can_move, collect, enable, disable, isenabled, add_memory_pressure
 
 def setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue,
                 offsets=None):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -450,6 +450,9 @@
     # __________ GC operations __________
 
     'gc__collect':          LLOp(canmallocgc=True),
+    'gc__enable':           LLOp(),
+    'gc__disable':          LLOp(),
+    'gc__isenabled':        LLOp(),
     'gc_free':              LLOp(),
     'gc_fetch_exception':   LLOp(),
     'gc_restore_exception': LLOp(),

From pypy.commits at gmail.com  Mon May 21 10:08:26 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:08:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backed out changeset f0f9cfefd069
Message-ID: <5b02d2da.1c69fb81.ad4cb.7a4e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94638:eef354dfdba8
Date: 2018-05-21 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/eef354dfdba8/

Log:	Backed out changeset f0f9cfefd069

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -769,9 +769,6 @@
     def __init__(self, space):
         self.space = space
 
-    def get_empty_storage(self, sizehint):
-        raise NotImplementedError
-
     def get_sizehint(self):
         return -1
 
@@ -829,9 +826,6 @@
     def getitems_float(self, w_list):
         return None
 
-    def getitems_unroll(self, w_list):
-        raise NotImplementedError
-
     def getstorage_copy(self, w_list):
         raise NotImplementedError
 
@@ -1086,16 +1080,6 @@
         strategy = w_list.strategy = self.space.fromcache(IntegerListStrategy)
         w_list.lstorage = strategy.erase(items)
 
-    def step(self, w_list):
-        raise NotImplementedError
-
-    def _getitems_range(self, w_list, wrap_items):
-        raise NotImplementedError
-    _getitems_range_unroll = _getitems_range
-
-    def _getitem_range_unwrapped(self, w_list, i):
-        raise NotImplementedError
-
     def wrap(self, intval):
         return self.space.newint(intval)
 
@@ -1120,7 +1104,7 @@
         w_other.lstorage = w_list.lstorage
 
     def getitem(self, w_list, i):
-        return self.wrap(self._getitem_range_unwrapped(w_list, i))
+        return self.wrap(self._getitem_unwrapped(w_list, i))
 
     def getitems_int(self, w_list):
         return self._getitems_range(w_list, False)
@@ -1210,7 +1194,7 @@
     def step(self, w_list):
         return 1
 
-    def _getitem_range_unwrapped(self, w_list, i):
+    def _getitem_unwrapped(self, w_list, i):
         length = self.unerase(w_list.lstorage)[0]
         if i < 0:
             i += length
@@ -1288,7 +1272,7 @@
     def step(self, w_list):
         return self.unerase(w_list.lstorage)[1]
 
-    def _getitem_range_unwrapped(self, w_list, i):
+    def _getitem_unwrapped(self, w_list, i):
         v = self.unerase(w_list.lstorage)
         start = v[0]
         step = v[1]
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -31,9 +31,6 @@
         reprlist = [repr(w_item) for w_item in self.getkeys()]
         return "<%s(%s)(%s)>" % (self.__class__.__name__, self.strategy, ', '.join(reprlist))
 
-    def _newobj(self, space, w_iterable):
-        raise NotImplementedError
-
     def from_storage_and_strategy(self, storage, strategy):
         obj = self._newobj(self.space, None)
         assert isinstance(obj, W_BaseSetObject)
@@ -705,12 +702,6 @@
     #def unerase(self, storage):
     #    raise NotImplementedError
 
-    def may_contain_equal_elements(self, strategy):
-        return True
-
-    def _intersect_wrapped(self, w_set, w_other):
-        raise NotImplementedError
-
     # __________________ methods called on W_SetObject _________________
 
     def clear(self, w_set):
@@ -1631,8 +1622,8 @@
         if type(w_item) is not W_IntObject:
             break
     else:
-        strategy = w_set.strategy = space.fromcache(IntegerSetStrategy)
-        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
+        w_set.strategy = space.fromcache(IntegerSetStrategy)
+        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
         return
 
     # check for strings
@@ -1640,8 +1631,8 @@
         if type(w_item) is not W_BytesObject:
             break
     else:
-        strategy = w_set.strategy = space.fromcache(BytesSetStrategy)
-        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
+        w_set.strategy = space.fromcache(BytesSetStrategy)
+        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
         return
 
     # check for unicode
@@ -1649,8 +1640,8 @@
         if type(w_item) is not W_UnicodeObject:
             break
     else:
-        strategy = w_set.strategy = space.fromcache(UnicodeSetStrategy)
-        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
+        w_set.strategy = space.fromcache(UnicodeSetStrategy)
+        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
         return
 
     # check for compares by identity
@@ -1658,12 +1649,12 @@
         if not space.type(w_item).compares_by_identity():
             break
     else:
-        strategy = w_set.strategy = space.fromcache(IdentitySetStrategy)
-        w_set.sstorage = strategy.get_storage_from_list(iterable_w)
+        w_set.strategy = space.fromcache(IdentitySetStrategy)
+        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
         return
 
-    strategy = w_set.strategy = space.fromcache(ObjectSetStrategy)
-    w_set.sstorage = strategy.get_storage_from_list(iterable_w)
+    w_set.strategy = space.fromcache(ObjectSetStrategy)
+    w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
 
 
 create_set_driver = jit.JitDriver(name='create_set',
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -112,14 +112,10 @@
                     if not homedef.check_missing_attribute_update(attr):
                         for desc in s_newvalue.descriptions:
                             if desc.selfclassdef is None:
-                                print AnnotatorError(
+                                raise AnnotatorError(
                                     "demoting method %s from %s to class "
-                                    "%s not allowed\n" % (self.name, desc.originclassdef, homedef) +
-                                    "either you need to add an abstract method to the base class\n" +
-                                    "or you need an assert isinstance(...) to ensure the annotator " +
-                                    "that the instance is of the right class"
+                                    "%s not allowed" % (self.name, desc.originclassdef, homedef)
                                 )
-                                break
 
         # check for attributes forbidden by slots or _attrs_
         if homedef.classdesc.all_enforced_attrs is not None:
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -103,9 +103,6 @@
             self.seen = AddressDict()
         self.pending = AddressStack()
 
-    def processobj(self, obj):
-        raise NotImplementedError("abstract base class")
-
     def delete(self):
         if self.gcflag == 0:
             self.seen.delete()
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -57,9 +57,6 @@
         """Return the size in bytes."""
         raise NotImplementedError
 
-    def get_raw_address(self):
-        raise NotImplementedError
-
     def __len__(self):
         res = self.getlength()
         assert res >= 0
@@ -166,7 +163,7 @@
     def decorate(targetcls):
         """
         Create and attach specialized versions of typed_{read,write}. We need to
-        do this because the JIT codewriters mandates that base_ofs is an
+        do this becase the JIT codewriters mandates that base_ofs is an
         RPython constant.
         """
         if targetcls.__bases__ != (GCBuffer,):

From pypy.commits at gmail.com  Mon May 21 10:08:28 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:08:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backed out changeset bf0a01ba0389
Message-ID: <5b02d2dc.1c69fb81.26ee2.f445@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94639:731e015261b7
Date: 2018-05-21 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/731e015261b7/

Log:	Backed out changeset bf0a01ba0389

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -39,9 +39,6 @@
     def mutate_over(self, visitor):
         raise AssertionError("mutate_over() implementation not provided")
 
-    def to_object(self, space):
-        raise NotImplementedError("abstract base class")
-
 
 class NodeVisitorNotImplemented(Exception):
     pass
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -435,9 +435,6 @@
     def mutate_over(self, visitor):
         raise AssertionError("mutate_over() implementation not provided")
 
-    def to_object(self, space):
-        raise NotImplementedError("abstract base class")
-
 
 class NodeVisitorNotImplemented(Exception):
     pass
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -33,7 +33,6 @@
     __slots__ = ('__weakref__',)
     _must_be_light_finalizer_ = True
     user_overridden_class = False
-    _settled_ = True
 
     def getdict(self, space):
         return None
@@ -198,8 +197,6 @@
     # hooks that the mapdict implementations needs:
     def _get_mapdict_map(self):
         return None
-    def _mapdict_init_empty(self, terminator):
-        return None
     def _set_mapdict_map(self, map):
         raise NotImplementedError
     def _mapdict_read_storage(self, index):
@@ -916,11 +913,9 @@
         """Unpack an iterable into a real (interpreter-level) list.
 
         Raise an OperationError(w_ValueError) if the length is wrong."""
-        from pypy.interpreter.generator import GeneratorIterator
         w_iterator = self.iter(w_iterable)
         if expected_length == -1:
             if self.is_generator(w_iterator):
-                assert isinstance(w_iterator, GeneratorIterator)
                 # special hack for speed
                 lst_w = []
                 w_iterator.unpack_into(lst_w)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -301,7 +301,6 @@
         """
         strategy = self.get_strategy()
         strategy.move_to_end(self, w_key, last_flag)
-            # XXX this should be in the default move_to_end method!
 
     def nondescr_popitem_first(self, space):
         """Not exposed directly to app-level, but via __pypy__.popitem_first().
@@ -499,9 +498,6 @@
     def get_empty_storage(self):
         raise NotImplementedError
 
-    def switch_to_object_strategy(self, w_dict):
-        raise NotImplementedError
-
     @jit.look_inside_iff(lambda self, w_dict:
                          w_dict_unrolling_heuristic(w_dict))
     def w_keys(self, w_dict):
@@ -588,36 +584,6 @@
     def prepare_update(self, w_dict, num_extra):
         pass
 
-    def length(self, w_dict):
-        raise NotImplementedError
-
-    def getitem(self, w_dict, w_key):
-        raise NotImplementedError
-
-    def getitem_str(self, w_dict, key):
-        raise NotImplementedError
-
-    def setitem(self, w_dict, w_key, w_value):
-        raise NotImplementedError
-
-    def setitem_str(self, w_dict, key, w_value):
-        raise NotImplementedError
-
-    def delitem(self, w_dict, w_key):
-        raise NotImplementedError
-
-    def setdefault(self, w_dict, w_key, w_default):
-        raise NotImplementedError
-
-    def iterkeys(self, w_dict):
-        raise NotImplementedError
-
-    def itervalues(self, w_dict):
-        raise NotImplementedError
-
-    def iteritems(self, w_dict):
-        raise NotImplementedError
-
     def move_to_end(self, w_dict, w_key, last_flag):
         # fall-back
         w_value = w_dict.getitem(w_key)
@@ -841,22 +807,12 @@
 class BaseKeyIterator(BaseIteratorImplementation):
     next_key = _new_next('key')
 
-    def next_key_entry(self):
-        raise NotImplementedError
-
-
 class BaseValueIterator(BaseIteratorImplementation):
     next_value = _new_next('value')
 
-    def next_value_entry(self):
-        raise NotImplementedError
-
 class BaseItemIterator(BaseIteratorImplementation):
     next_item = _new_next('item')
 
-    def next_item_entry(self):
-        raise NotImplementedError
-
 
 def create_iterator_classes(dictimpl):
     if not hasattr(dictimpl, 'wrapkey'):
@@ -1491,7 +1447,6 @@
 class W_DictMultiIterKeysObject(W_BaseDictMultiIterObject):
     def descr_next(self, space):
         iteratorimplementation = self.iteratorimplementation
-        assert isinstance(iteratorimplementation, BaseKeyIterator)
         w_key = iteratorimplementation.next_key()
         if w_key is not None:
             return w_key
@@ -1500,7 +1455,6 @@
 class W_DictMultiIterValuesObject(W_BaseDictMultiIterObject):
     def descr_next(self, space):
         iteratorimplementation = self.iteratorimplementation
-        assert isinstance(iteratorimplementation, BaseValueIterator)
         w_value = iteratorimplementation.next_value()
         if w_value is not None:
             return w_value
@@ -1509,7 +1463,6 @@
 class W_DictMultiIterItemsObject(W_BaseDictMultiIterObject):
     def descr_next(self, space):
         iteratorimplementation = self.iteratorimplementation
-        assert isinstance(iteratorimplementation, BaseItemIterator)
         w_key, w_value = iteratorimplementation.next_item()
         if w_key is not None:
             return space.newtuple([w_key, w_value])
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -808,9 +808,6 @@
     def getitems(self, w_list):
         return self.getitems_copy(w_list)
 
-    def getitems_fixedsize(self, w_list):
-        raise NotImplementedError
-
     def getitems_copy(self, w_list):
         raise NotImplementedError
 
@@ -859,13 +856,11 @@
         raise NotImplementedError
 
     def extend(self, w_list, w_any):
-        from pypy.interpreter.generator import GeneratorIterator
         space = self.space
         if type(w_any) is W_ListObject or (isinstance(w_any, W_ListObject) and
                                            space._uses_list_iter(w_any)):
             self._extend_from_list(w_list, w_any)
         elif space.is_generator(w_any):
-            assert isinstance(w_any, GeneratorIterator)
             w_any.unpack_into_w(w_list)
         else:
             self._extend_from_iterable(w_list, w_any)

From pypy.commits at gmail.com  Mon May 21 10:08:30 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:08:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backed out changeset b51d40dbb01d
Message-ID: <5b02d2de.1c69fb81.fbeed.8325@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94640:1a2536772b0e
Date: 2018-05-21 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/1a2536772b0e/

Log:	Backed out changeset b51d40dbb01d

diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -102,20 +102,13 @@
 
     def validate(self, homedef):
         s_newvalue = self.s_value
-        homedesc = homedef.classdesc
-        # check for method demotion and after-the-fact method additions
+        # check for after-the-fact method additions
         if isinstance(s_newvalue, SomePBC):
             attr = self.name
             if s_newvalue.getKind() == MethodDesc:
                 # is method
                 if homedef.classdesc.read_attribute(attr, None) is None:
-                    if not homedef.check_missing_attribute_update(attr):
-                        for desc in s_newvalue.descriptions:
-                            if desc.selfclassdef is None:
-                                raise AnnotatorError(
-                                    "demoting method %s from %s to class "
-                                    "%s not allowed" % (self.name, desc.originclassdef, homedef)
-                                )
+                    homedef.check_missing_attribute_update(attr)
 
         # check for attributes forbidden by slots or _attrs_
         if homedef.classdesc.all_enforced_attrs is not None:
@@ -496,7 +489,6 @@
     knowntype = type
     instance_level = False
     all_enforced_attrs = None   # or a set
-    settled = False
     _detect_invalid_attrs = None
 
     def __init__(self, bookkeeper, cls,
@@ -567,9 +559,6 @@
         if base is not object:
             self.basedesc = bookkeeper.getdesc(base)
 
-        if '_settled_' in cls.__dict__:
-            self.settled = bool(cls.__dict__['_settled_'])
-
         if '__slots__' in cls.__dict__ or '_attrs_' in cls.__dict__:
             attrs = {}
             for decl in ('__slots__', '_attrs_'):
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -2374,8 +2374,7 @@
     def test_stored_bound_method_2(self):
         # issue 129
         class H:
-            def h(self):
-                raise NotImplementedError("abstract method")
+            pass
         class H1(H):
             def h(self):
                 return 42
@@ -3050,7 +3049,6 @@
     def test_slots_check(self):
         class Base(object):
             __slots__ = 'x'
-            def m(self): raise NotImplementedError("abstract")
         class A(Base):
             __slots__ = 'y'
             def m(self):
@@ -3100,7 +3098,6 @@
     def test_enforced_attrs_check(self):
         class Base(object):
             _attrs_ = 'x'
-            def m(self): raise NotImplementedError("abstract")
         class A(Base):
             _attrs_ = 'y'
             def m(self):
@@ -3170,45 +3167,6 @@
         a = self.RPythonAnnotator()
         a.build_types(f, [bool])
 
-    def test_enforce_settled(self):
-        class A(object):
-            _settled_ = True
-
-            def m(self):
-                raise NotImplementedError
-
-        class B(A):
-
-            def m(self):
-                return 1
-
-            def n(self):
-                return 1
-
-        def fun(x):
-            if x:
-                a = A()
-            else:
-                a = B()
-
-            return a.m()
-
-        a = self.RPythonAnnotator()
-        s = a.build_types(fun, [bool])
-        assert s.knowntype == int
-
-        def fun(x):
-            if x:
-                a = A()
-            else:
-                a = B()
-
-            return a.n()
-
-        a = self.RPythonAnnotator()
-        with py.test.raises(AnnotatorError):
-            a.build_types(fun, [bool])
-
     def test_float_cmp(self):
         def fun(x, y):
             return (x < y,
diff --git a/rpython/translator/test/snippet.py b/rpython/translator/test/snippet.py
--- a/rpython/translator/test/snippet.py
+++ b/rpython/translator/test/snippet.py
@@ -375,9 +375,7 @@
     return _getstuff(d), _getstuff(e)
 
 class F:
-    def m(self, x):
-        raise NotImplementedError("abstract base")
-
+    pass
 class G(F):
     def m(self, x):
         return self.m2(x)

From pypy.commits at gmail.com  Mon May 21 10:20:45 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:20:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: fix test
Message-ID: <5b02d5bd.1c69fb81.9348a.fe57@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94641:f1df283e28af
Date: 2018-05-21 16:17 +0200
http://bitbucket.org/pypy/pypy/changeset/f1df283e28af/

Log:	fix test

diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -461,7 +461,7 @@
 
     def test_bug_null_in_objspace_type(self):
         import ast
-        code = ast.Expression(lineno=1, col_offset=1, body=ast.ListComp(lineno=1, col_offset=1, elt=ast.Call(lineno=1, col_offset=1, func=ast.Name(lineno=1, col_offset=1, id='str', ctx=ast.Load(lineno=1, col_offset=1)), args=[ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Load(lineno=1, col_offset=1))], keywords=[]), generators=[ast.comprehension(lineno=1, col_offset=1, target=ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Store(lineno=1, col_offset=1)), iter=ast.List(lineno=1, col_offset=1, elts=[ast.Num(lineno=1, col_offset=1, n=23)], ctx=ast.Load(lineno=1, col_offset=1, )), ifs=[])]))
+        code = ast.Expression(lineno=1, col_offset=1, body=ast.ListComp(lineno=1, col_offset=1, elt=ast.Call(lineno=1, col_offset=1, func=ast.Name(lineno=1, col_offset=1, id='str', ctx=ast.Load(lineno=1, col_offset=1)), args=[ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Load(lineno=1, col_offset=1))], keywords=[]), generators=[ast.comprehension(lineno=1, col_offset=1, target=ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Store(lineno=1, col_offset=1)), iter=ast.List(lineno=1, col_offset=1, elts=[ast.Num(lineno=1, col_offset=1, n=23)], ctx=ast.Load(lineno=1, col_offset=1, )), ifs=[], is_async=False)]))
         compile(code, '<template>', 'eval')
 
     def test_empty_yield_from(self):

From pypy.commits at gmail.com  Mon May 21 10:32:36 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:32:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: better error messages
Message-ID: <5b02d884.1c69fb81.a6674.3c0f@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94642:10f843fed291
Date: 2018-05-21 16:31 +0200
http://bitbucket.org/pypy/pypy/changeset/10f843fed291/

Log:	better error messages

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -1612,4 +1612,7 @@
         class Meta(type):
             pass
 
-        raises(TypeError, Meta, 5)
+        info = raises(TypeError, Meta, 5)
+        assert "takes exactly 3 arguments (1 given)" in str(info.value)
+        info = raises(TypeError, Meta, 5, 7)
+        assert "takes exactly 3 arguments (1 given)" in str(info.value)
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -752,8 +752,13 @@
 def descr__new__(space, w_typetype, __args__):
     """This is used to create user-defined classes only."""
     if len(__args__.arguments_w) not in (1, 3):
-        raise oefmt(space.w_TypeError,
-                    "type.__new__() takes 1 or 3 arguments")
+        if space.is_w(w_typetype, space.w_type):
+            raise oefmt(space.w_TypeError,
+                        "type.__new__() takes 1 or 3 arguments")
+        else:
+            raise oefmt(space.w_TypeError,
+                        "%N.__new__() takes exactly 3 arguments (1 given)",
+                        w_typetype)
 
     w_name = __args__.arguments_w[0]
 
@@ -765,7 +770,7 @@
             return space.type(w_name)
         else:
             raise oefmt(space.w_TypeError,
-                        "%N.__new__() takes 3 arguments (1 given)",
+                        "%N.__new__() takes exactly 3 arguments (1 given)",
                         w_typetype)
     w_bases = __args__.arguments_w[1]
     w_dict = __args__.arguments_w[2]

From pypy.commits at gmail.com  Mon May 21 10:35:57 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 07:35:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: be less strict about error message
Message-ID: <5b02d94d.1c69fb81.3c8f2.ad7e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94643:1f88315814dc
Date: 2018-05-21 16:34 +0200
http://bitbucket.org/pypy/pypy/changeset/1f88315814dc/

Log:	be less strict about error message

diff --git a/lib-python/3/test/test_types.py b/lib-python/3/test/test_types.py
--- a/lib-python/3/test/test_types.py
+++ b/lib-python/3/test/test_types.py
@@ -1004,7 +1004,7 @@
             X = types.new_class("X", (int(), C))
 
     def test_one_argument_type(self):
-        expected_message = 'type.__new__() takes exactly 3 arguments (1 given)'
+        expected_message = '.__new__() takes exactly 3 arguments (1 given)'
 
         # Only type itself can use the one-argument form (#27157)
         self.assertIs(type(5), int)
@@ -1013,13 +1013,13 @@
             pass
         with self.assertRaises(TypeError) as cm:
             M(5)
-        self.assertEqual(str(cm.exception), expected_message)
+        self.assertIn(expected_message, str(cm.exception))
 
         class N(type, metaclass=M):
             pass
         with self.assertRaises(TypeError) as cm:
             N(5)
-        self.assertEqual(str(cm.exception), expected_message)
+        self.assertIn(expected_message, str(cm.exception))
 
 
 class SimpleNamespaceTests(unittest.TestCase):

From pypy.commits at gmail.com  Mon May 21 12:25:14 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 21 May 2018 09:25:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: missing part of a8f64c4ae1ec
Message-ID: <5b02f2ea.1c69fb81.9f8a3.54b0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r94644:c59ecbebf6f8
Date: 2018-05-21 09:24 -0700
http://bitbucket.org/pypy/pypy/changeset/c59ecbebf6f8/

Log:	missing part of a8f64c4ae1ec

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1133,10 +1133,11 @@
 
 def attach_c_functions(space, eci, prefix):
     state = space.fromcache(State)
-    state.C._Py_Dealloc = rffi.llexternal('_Py_Dealloc',
-                                         [PyObject], lltype.Void,
-                                         compilation_info=eci,
-                                         _nowrapper=True)
+    state.C._Py_Dealloc = rffi.llexternal(
+        mangle_name(prefix, '_Py_Dealloc'),
+        [PyObject], lltype.Void,
+        compilation_info=eci,
+        _nowrapper=True)
     state.C.PyObject_Free = rffi.llexternal(
         mangle_name(prefix, 'PyObject_Free'),
         [rffi.VOIDP], lltype.Void,

From pypy.commits at gmail.com  Mon May 21 16:16:55 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 13:16:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix by calling fspath
Message-ID: <5b032937.1c69fb81.77fc6.a954@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94648:717023d5ef1c
Date: 2018-05-21 22:07 +0200
http://bitbucket.org/pypy/pypy/changeset/717023d5ef1c/

Log:	fix by calling fspath

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -185,7 +185,8 @@
                                 "Cannot use closefd=False with file name")
 
                 if space.is_none(w_opener):
-                    from pypy.module.posix.interp_posix import dispatch_filename
+                    from pypy.module.posix.interp_posix import dispatch_filename, fspath
+                    w_name = fspath(space, w_name)
                     while True:
                         try:
                             self.fd = dispatch_filename(rposix.open)(

From pypy.commits at gmail.com  Mon May 21 16:16:57 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 13:16:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: make sure error messages get the original
 argument
Message-ID: <5b032939.1c69fb81.5b674.41e7@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94649:d8ae94b9e568
Date: 2018-05-21 22:11 +0200
http://bitbucket.org/pypy/pypy/changeset/d8ae94b9e568/

Log:	make sure error messages get the original argument

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -186,11 +186,11 @@
 
                 if space.is_none(w_opener):
                     from pypy.module.posix.interp_posix import dispatch_filename, fspath
-                    w_name = fspath(space, w_name)
+                    w_path = fspath(space, w_name)
                     while True:
                         try:
                             self.fd = dispatch_filename(rposix.open)(
-                                space, w_name, flags, 0666)
+                                space, w_path, flags, 0666)
                             fd_is_own = True
                             break
                         except OSError as e:

From pypy.commits at gmail.com  Mon May 21 16:16:52 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 21 May 2018 13:16:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: failing test that shows the problem of
 test_compileall.py
Message-ID: <5b032934.1c69fb81.7ef92.bb5d@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94647:8daf0c2b6487
Date: 2018-05-21 20:35 +0200
http://bitbucket.org/pypy/pypy/changeset/8daf0c2b6487/

Log:	failing test that shows the problem of test_compileall.py

diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -87,6 +87,15 @@
         assert f.readline() == b''
         f.close()
 
+    def test_support_fspath(self):
+        import _io
+        class P(object):
+            def __fspath__(x):
+                return self.tmpfile
+        f = _io.FileIO(P(), 'rb')
+        assert f.readline() == b'a\n'
+        f.close()
+
     def test_readlines(self):
         import _io
         f = _io.FileIO(self.tmpfile, 'rb')

From pypy.commits at gmail.com  Mon May 21 23:15:56 2018
From: pypy.commits at gmail.com (william_ml_leslie)
Date: Mon, 21 May 2018 20:15:56 -0700 (PDT)
Subject: [pypy-commit] pypy real-mode-translator-driver: Close branch
 real-mode-translator-driver.
Message-ID: <5b038b6c.1c69fb81.9eb2d.2cea@mx.google.com>

Author: William Leslie <william.leslie.ttg at gmail.com>
Branch: real-mode-translator-driver
Changeset: r94650:6ca216fc3bce
Date: 2018-05-22 03:15 +0000
http://bitbucket.org/pypy/pypy/changeset/6ca216fc3bce/

Log:	Close branch real-mode-translator-driver.


From pypy.commits at gmail.com  Tue May 22 00:52:04 2018
From: pypy.commits at gmail.com (mattip)
Date: Mon, 21 May 2018 21:52:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge default into py3.5
Message-ID: <5b03a1f4.1c69fb81.73d10.5868@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r94651:39fd746993fe
Date: 2018-05-21 21:34 -0700
http://bitbucket.org/pypy/pypy/changeset/39fd746993fe/

Log:	merge default into py3.5

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -18,3 +18,7 @@
 .. branch: crypt_h
 
 Include crypt.h for crypt() on Linux
+
+.. branch: gc-more-logging
+
+Log additional gc-minor and gc-collect-step info in the PYPYLOG
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -135,7 +135,10 @@
         cls.w_runappdirect = cls.space.wrap(cls.runappdirect)
 
     def test_jit_backend_features(self):
-        from __pypy__ import jit_backend_features
+        try:
+            from __pypy__ import jit_backend_features
+        except ImportError:
+            skip("compiled without jit")
         supported_types = jit_backend_features
         assert isinstance(supported_types, list)
         for x in supported_types:
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -660,7 +660,7 @@
     '_PyObject_New', '_PyObject_NewVar',
     '_PyObject_GC_New', '_PyObject_GC_NewVar',
     'PyObject_Init', 'PyObject_InitVar',
-    'PyTuple_New',
+    'PyTuple_New', '_Py_Dealloc',
 ]
 TYPES = {}
 FORWARD_DECLS = []
@@ -1150,10 +1150,11 @@
 
 def attach_c_functions(space, eci, prefix):
     state = space.fromcache(State)
-    state.C._Py_Dealloc = rffi.llexternal('_Py_Dealloc',
-                                         [PyObject], lltype.Void,
-                                         compilation_info=eci,
-                                         _nowrapper=True)
+    state.C._Py_Dealloc = rffi.llexternal(
+        mangle_name(prefix, '_Py_Dealloc'),
+        [PyObject], lltype.Void,
+        compilation_info=eci,
+        _nowrapper=True)
     state.C.PyObject_Free = rffi.llexternal(
         mangle_name(prefix, 'PyObject_Free'),
         [rffi.VOIDP], lltype.Void,
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -226,13 +226,7 @@
         """Not exposed directly to app-level, but via __pypy__.reversed_dict().
         """
         strategy = self.get_strategy()
-        if strategy.has_iterreversed:
-            it = strategy.iterreversed(self)
-            return W_DictMultiIterKeysObject(space, it)
-        else:
-            # fall-back
-            w_keys = self.w_keys()
-            return space.call_method(w_keys, '__reversed__')
+        return strategy.iterreversed(self)
 
     def nondescr_delitem_if_value_is(self, space, w_key, w_value):
         """Not exposed directly to app-level, but used by
@@ -246,32 +240,7 @@
         """Not exposed directly to app-level, but via __pypy__.move_to_end().
         """
         strategy = self.get_strategy()
-        if strategy.has_move_to_end:
-            strategy.move_to_end(self, w_key, last_flag)
-        else:
-            # fall-back
-            w_value = self.getitem(w_key)
-            if w_value is None:
-                space.raise_key_error(w_key)
-            else:
-                self.internal_delitem(w_key)
-                if last_flag:
-                    self.setitem(w_key, w_value)
-                else:
-                    # *very slow* fall-back
-                    keys_w = []
-                    values_w = []
-                    iteratorimplementation = self.iteritems()
-                    while True:
-                        w_k, w_v = iteratorimplementation.next_item()
-                        if w_k is None:
-                            break
-                        keys_w.append(w_k)
-                        values_w.append(w_v)
-                    self.clear()
-                    self.setitem(w_key, w_value)
-                    for i in range(len(keys_w)):
-                        self.setitem(keys_w[i], values_w[i])
+        strategy.move_to_end(self, w_key, last_flag)
 
     def nondescr_popitem_first(self, space):
         """Not exposed directly to app-level, but via __pypy__.popitem_first().
@@ -298,21 +267,10 @@
         otherwise KeyError is raised
         """
         strategy = self.get_strategy()
-        if strategy.has_pop:
-            try:
-                return strategy.pop(self, w_key, w_default)
-            except KeyError:
-                raise space.raise_key_error(w_key)
-        # fall-back
-        w_item = self.getitem(w_key)
-        if w_item is None:
-            if w_default is not None:
-                return w_default
-            else:
-                space.raise_key_error(w_key)
-        else:
-            self.internal_delitem(w_key)
-            return w_item
+        try:
+            return strategy.pop(self, w_key, w_default)
+        except KeyError:
+            raise space.raise_key_error(w_key)
 
     def descr_popitem(self, space):
         """D.popitem() -> (k, v), remove and return some (key, value) pair as
@@ -547,7 +505,6 @@
         raise NotImplementedError
 
     has_iterreversed = False
-    has_move_to_end = False
     has_pop = False
     # ^^^ no default implementation available for these methods
 
@@ -562,6 +519,48 @@
     def prepare_update(self, w_dict, num_extra):
         pass
 
+    def move_to_end(self, w_dict, w_key, last_flag):
+        # fall-back
+        w_value = w_dict.getitem(w_key)
+        if w_value is None:
+            self.space.raise_key_error(w_key)
+        else:
+            w_dict.internal_delitem(w_key)
+            if last_flag:
+                w_dict.setitem(w_key, w_value)
+            else:
+                # *very slow* fall-back
+                keys_w = []
+                values_w = []
+                iteratorimplementation = w_dict.iteritems()
+                while True:
+                    w_k, w_v = iteratorimplementation.next_item()
+                    if w_k is None:
+                        break
+                    keys_w.append(w_k)
+                    values_w.append(w_v)
+                w_dict.clear()
+                w_dict.setitem(w_key, w_value)
+                for i in range(len(keys_w)):
+                    w_dict.setitem(keys_w[i], values_w[i])
+
+
+    def pop(self, w_dict, w_key, w_default):
+        # fall-back
+        w_item = w_dict.getitem(w_key)
+        if w_item is None:
+            if w_default is not None:
+                return w_default
+            else:
+                raise KeyError
+        else:
+            w_dict.internal_delitem(w_key)
+            return w_item
+
+    def iterreversed(self, w_dict):
+        # fall-back if getiterreversed is not present
+        w_keys = self.w_keys(w_dict)
+        return self.space.call_method(w_keys, '__reversed__')
 
 class EmptyDictStrategy(DictStrategy):
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -660,6 +659,15 @@
     def view_as_kwargs(self, w_dict):
         return ([], [])
 
+    def move_to_end(self, w_dict, w_key, last_flag):
+        self.space.raise_key_error(w_key)
+
+    def pop(self, w_dict, w_key, w_default):
+        if w_default is not None:
+            return w_default
+        else:
+            raise KeyError
+
     # ---------- iterator interface ----------------
 
     def getiterkeys(self, w_dict):
@@ -829,15 +837,10 @@
 
     if hasattr(dictimpl, 'getiterreversed'):
         def iterreversed(self, w_dict):
-            return IterClassReversed(self.space, self, w_dict)
+            return W_DictMultiIterKeysObject(
+                    self.space,
+                    IterClassReversed(self.space, self, w_dict))
         dictimpl.iterreversed = iterreversed
-        dictimpl.has_iterreversed = True
-
-    if hasattr(dictimpl, 'move_to_end'):
-        dictimpl.has_move_to_end = True
-
-    if hasattr(dictimpl, 'pop'):
-        dictimpl.has_pop = True
 
     @jit.look_inside_iff(lambda self, w_dict, w_updatedict:
                          w_dict_unrolling_heuristic(w_dict))
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -231,6 +231,33 @@
         raises(KeyError, d.pop, "abc")
         assert len(d) == 2
 
+    def test_pop_empty_bug(self):
+        d = {}
+        assert d.pop(1, 2) == 2
+        def f(**d): return d
+        d = f()
+        assert d.pop(1, 2) == 2
+
+    def test_pop_kwargs(self):
+        def kw(**d): return d
+        d = kw(o=2, t=4)
+        dd = d.copy()
+        result = dd.pop("o")
+        assert result == 2
+        assert len(dd) == 1
+        dd = d.copy()
+        result = dd.pop("o", 44)
+        assert result == 2
+        assert len(dd) == 1
+        result = dd.pop("o", 44)
+        assert result == 44
+        assert len(dd) == 1
+        raises(KeyError, dd.pop, "33")
+
+        assert d.pop("abc", None) is None
+        raises(KeyError, d.pop, "abc")
+        assert len(d) == 2
+
     def test_items(self):
         d = {1: 2, 3: 4}
         its = list(d.items())
@@ -261,8 +288,9 @@
 
     def test_reversed_dict(self):
         import __pypy__
-        for d in [{}, {1: 2, 3: 4, 5: 6}, {"a": 5, "b": 2, "c": 6}]:
-            assert list(__pypy__.reversed_dict(d)) == list(d.keys())[::-1]
+        def kw(**d): return d
+        for d in [{}, {1: 2, 3: 4, 5: 6}, {"a": 5, "b": 2, "c": 6}, kw(a=1, b=2)]:
+            assert list(__pypy__.reversed_dict(d)) == list(d.keys()[::-1])
         raises(TypeError, __pypy__.reversed_dict, 42)
 
     def test_reversed_dict_runtimeerror(self):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1836,6 +1836,7 @@
         debug_print("minor collect, total memory used:", total_memory_used)
         debug_print("number of pinned objects:",
                     self.pinned_objects_in_nursery)
+        debug_print("total size of surviving objects:", self.nursery_surviving_size)
         if self.DEBUG >= 2:
             self.debug_check_consistency()     # expensive!
         #
@@ -2401,7 +2402,9 @@
                 # a total object size of at least '3 * nursery_size' bytes
                 # is processed.
                 limit = 3 * self.nursery_size // self.small_request_threshold
-                self.free_unvisited_rawmalloc_objects_step(limit)
+                nobjects = self.free_unvisited_rawmalloc_objects_step(limit)
+                debug_print("freeing raw objects:", limit-nobjects,
+                            "freed, limit was", limit)
                 done = False    # the 2nd half below must still be done
             else:
                 # Ask the ArenaCollection to visit a fraction of the objects.
@@ -2411,6 +2414,8 @@
                 limit = 3 * self.nursery_size // self.ac.page_size
                 done = self.ac.mass_free_incremental(self._free_if_unvisited,
                                                      limit)
+                status = done and "No more pages left." or "More to do."
+                debug_print("freeing GC objects, up to", limit, "pages.", status)
             # XXX tweak the limits above
             #
             if done:

From pypy.commits at gmail.com  Tue May 22 06:19:46 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 22 May 2018 03:19:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: fix test
Message-ID: <5b03eec2.1c69fb81.d59ca.505e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94652:bac7bc4a6c4f
Date: 2018-05-22 12:19 +0200
http://bitbucket.org/pypy/pypy/changeset/bac7bc4a6c4f/

Log:	fix test

diff --git a/lib-python/3/test/test_opcodes.py b/lib-python/3/test/test_opcodes.py
--- a/lib-python/3/test/test_opcodes.py
+++ b/lib-python/3/test/test_opcodes.py
@@ -27,7 +27,9 @@
             with open(ann_module.__file__) as f:
                 txt = f.read()
             co = compile(txt, ann_module.__file__, 'exec')
-            self.assertEqual(co.co_firstlineno, 6)
+            # On PyPy, the lineno of multiline tokens is the *first* line, on
+            # CPython the last (CPython expects 6 here)
+            self.assertEqual(co.co_firstlineno, 3)
         except OSError:
             pass
 

From pypy.commits at gmail.com  Wed May 23 10:07:39 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 07:07:39 -0700 (PDT)
Subject: [pypy-commit] cffi default: Mention cppyy
Message-ID: <5b0575ab.1c69fb81.39cbb.9b43@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3120:93d9ae775fb2
Date: 2018-05-23 16:07 +0200
http://bitbucket.org/cffi/cffi/changeset/93d9ae775fb2/

Log:	Mention cppyy

diff --git a/doc/source/goals.rst b/doc/source/goals.rst
--- a/doc/source/goals.rst
+++ b/doc/source/goals.rst
@@ -31,6 +31,12 @@
   Python, unlike `Weave`_.  This is about calling existing C libraries
   from Python.
 
+* There is no C++ support.  Sometimes, it is reasonable to write a C
+  wrapper around the C++ code and then call this C API with CFFI.
+  Otherwise, look at other projects.  I would recommend cppyy_, which
+  has got some similarities (and also works efficiently on both CPython
+  and PyPy).
+
 .. _`LuaJIT's FFI`: http://luajit.org/ext_ffi.html
 .. _`Cython`: http://www.cython.org
 .. _`SWIG`: http://www.swig.org/
@@ -38,6 +44,7 @@
 .. _`native C extensions`: http://docs.python.org/extending/extending.html
 .. _`ctypes`: http://docs.python.org/library/ctypes.html
 .. _`Weave`: http://wiki.scipy.org/Weave
+.. _`cppyy`: http://cppyy.readthedocs.io/en/latest/
 .. _`manually wrap`: overview.html#abi-versus-api
 
 Get started by reading `the overview`__.

From pypy.commits at gmail.com  Wed May 23 10:13:15 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 07:13:15 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: hg merge default
Message-ID: <5b0576fb.1c69fb81.eefda.b2b8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r94653:dd287c100ac7
Date: 2018-05-23 16:12 +0200
http://bitbucket.org/pypy/pypy/changeset/dd287c100ac7/

Log:	hg merge default

diff too long, truncating to 2000 out of 4168 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -56,3 +56,5 @@
 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0
 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0
 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1
+ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0
+fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0
diff --git a/dotviewer/font/NOTICE b/dotviewer/font/COPYING.txt
rename from dotviewer/font/NOTICE
rename to dotviewer/font/COPYING.txt
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -82,8 +82,11 @@
     def _CData_output(self, resarray, base=None, index=-1):
         from _rawffi.alt import types
         # If a char_p or unichar_p is received, skip the string interpretation
-        if base._ffiargtype != types.Pointer(types.char_p) and \
-           base._ffiargtype != types.Pointer(types.unichar_p):
+        try:
+            deref = type(base)._deref_ffiargtype()
+        except AttributeError:
+            deref = None
+        if deref != types.char_p and deref != types.unichar_p:
             # this seems to be a string if we're array of char, surprise!
             from ctypes import c_char, c_wchar
             if self._type_ is c_char:
@@ -120,6 +123,12 @@
                 value = self(*value)
         return _CDataMeta.from_param(self, value)
 
+    def _build_ffiargtype(self):
+        return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+    def _deref_ffiargtype(self):
+        return self._type_.get_ffi_argtype()
+
 def array_get_slice_params(self, index):
     if hasattr(self, '_length_'):
         start, stop, step = index.indices(self._length_)
@@ -248,6 +257,5 @@
             _type_ = base
         )
         cls = ArrayMeta(name, (Array,), tpdict)
-        cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype())
         ARRAY_CACHE[key] = cls
         return cls
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -49,10 +49,13 @@
         else:
             return self.from_param(as_parameter)
 
+    def _build_ffiargtype(self):
+        return _shape_to_ffi_type(self._ffiargshape_)
+
     def get_ffi_argtype(self):
         if self._ffiargtype:
             return self._ffiargtype
-        self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_)
+        self._ffiargtype = self._build_ffiargtype()
         return self._ffiargtype
 
     def _CData_output(self, resbuffer, base=None, index=-1):
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -70,7 +70,12 @@
         self._ffiarray = ffiarray
         self.__init__ = __init__
         self._type_ = TP
-        self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype())
+
+    def _build_ffiargtype(self):
+        return _ffi.types.Pointer(self._type_.get_ffi_argtype())
+
+    def _deref_ffiargtype(self):
+        return self._type_.get_ffi_argtype()
 
     from_address = cdata_from_address
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -160,6 +160,10 @@
             raise AttributeError("_fields_ is final")
         if self in [f[1] for f in value]:
             raise AttributeError("Structure or union cannot contain itself")
+        if self._ffiargtype is not None:
+            raise NotImplementedError("Too late to set _fields_: we already "
+                        "said to libffi that the structure type %s is opaque"
+                        % (self,))
         names_and_fields(
             self,
             value, self.__bases__[0],
diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst
--- a/pypy/doc/architecture.rst
+++ b/pypy/doc/architecture.rst
@@ -73,3 +73,63 @@
 This division between bytecode evaluator and object space gives a lot of
 flexibility.  One can plug in different :doc:`object spaces <objspace>` to get
 different or enriched behaviours of the Python objects.
+
+Layers
+------
+
+RPython
+~~~~~~~
+:ref:`RPython <rpython:language>` is the language in which we write interpreters.
+Not the entire PyPy project is written in RPython, only the parts that are
+compiled in the translation process. The interesting point is that RPython
+has no parser, it's compiled from the live python objects, which makes it
+possible to do all kinds of metaprogramming during import time. In short,
+Python is a meta programming language for RPython.
+
+The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+Consult `Getting Started with RPython`_ for further reading
+
+Translation
+~~~~~~~~~~~
+The translation toolchain - this is the part that takes care of translating
+RPython to flow graphs and then to C. There is more in the
+:doc:`architecture <architecture>` document written about it.
+
+It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
+and ``rtyper``.
+
+PyPy Interpreter
+~~~~~~~~~~~~~~~~
+This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
+interpreter for Python written in RPython.  The fact that it is
+RPython is not apparent at first.  Built-in modules are written in
+``pypy/module/*``.  Some modules that CPython implements in C are
+simply written in pure Python; they are in the top-level ``lib_pypy``
+directory.  The standard library of Python (with a few changes to
+accomodate PyPy) is in ``lib-python``.
+
+JIT Compiler
+~~~~~~~~~~~~
+:ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
+interpreter written in RPython, rather than the user program that it
+interprets.  As a result it applies to any interpreter, i.e. any
+language.  But getting it to work correctly is not trivial: it
+requires a small number of precise "hints" and possibly some small
+refactorings of the interpreter.  The JIT itself also has several
+almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
+optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
+residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
+that turns it into machine code.  Writing a new backend is a
+traditional way to get into the project.
+
+Garbage Collectors
+~~~~~~~~~~~~~~~~~~
+Garbage Collectors (GC): as you may notice if you are used to CPython's
+C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+:ref:`rpython:garbage-collection` is inserted
+during translation.  Moreover, this is not reference counting; it is a real
+GC written as more RPython code.  The best one we have so far is in
+``rpython/memory/gc/incminimark.py``.
+
+.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -267,14 +267,14 @@
 * PyPy 2.5.1 or earlier: normal users would see permission errors.
   Installers need to run ``pypy -c "import gdbm"`` and other similar
   commands at install time; the exact list is in
-  :source:`pypy/tool/release/package.py <package.py>`.  Users
+  :source:`pypy/tool/release/package.py`.  Users
   seeing a broken installation of PyPy can fix it after-the-fact if they
   have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
 
 * PyPy 2.6 and later: anyone would get ``ImportError: no module named
   _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
   ``lib_pypy`` directory during the installation process (plus others;
-  see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+  see the exact list in :source:`pypy/tool/release/package.py`).
   Users seeing a broken
   installation of PyPy can fix it after-the-fact, by running ``pypy
   /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -539,7 +539,7 @@
 
     hg help branch
 
-.. _official wiki: http://mercurial.selenic.com/wiki/Branch
+.. _official wiki: https://www.mercurial-scm.org/wiki/
 
 
 .. _using-development-tracker:
@@ -547,15 +547,7 @@
 Using the development bug/feature tracker
 -----------------------------------------
 
-We have a `development tracker`_, based on Richard Jones'
-`roundup`_ application.  You can file bugs,
-feature requests or see what's going on
-for the next milestone, both from an E-Mail and from a
-web interface.
-
-.. _development tracker: https://bugs.pypy.org/
-.. _roundup: http://roundup.sourceforge.net/
-
+We use bitbucket for :source:`issues` tracking and :source:`pull-requests`.
 
 .. _testing:
 
diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst
--- a/pypy/doc/commandline_ref.rst
+++ b/pypy/doc/commandline_ref.rst
@@ -8,3 +8,4 @@
    :maxdepth: 1
 
    man/pypy.1.rst
+   man/pypy3.1.rst
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -66,9 +66,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '5.8'
+version = '6.0'
 # The full version, including alpha/beta/rc tags.
-release = '5.8.0'
+release = '6.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/config/objspace.usemodules._cppyy.txt b/pypy/doc/config/objspace.usemodules._cppyy.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules._cppyy.txt
@@ -0,0 +1,1 @@
+The internal backend for cppyy
diff --git a/pypy/doc/config/objspace.usemodules._rawffi.txt b/pypy/doc/config/objspace.usemodules._rawffi.txt
--- a/pypy/doc/config/objspace.usemodules._rawffi.txt
+++ b/pypy/doc/config/objspace.usemodules._rawffi.txt
@@ -1,3 +1,3 @@
-An experimental module providing very low-level interface to
+A module providing very low-level interface to
 C-level libraries, for use when implementing ctypes, not
-intended for a direct use at all.
\ No newline at end of file
+intended for a direct use at all.
diff --git a/pypy/doc/config/objspace.usemodules.cpyext.txt b/pypy/doc/config/objspace.usemodules.cpyext.txt
--- a/pypy/doc/config/objspace.usemodules.cpyext.txt
+++ b/pypy/doc/config/objspace.usemodules.cpyext.txt
@@ -1,1 +1,1 @@
-Use (experimental) cpyext module, that tries to load and run CPython extension modules
+Use cpyext module to load and run CPython extension modules
diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/contributing.rst
@@ -0,0 +1,472 @@
+Contributing Guidelines
+===========================
+
+.. contents::
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy's
+  approach is great if you do TDD, and not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below.
+
+The first thing to remember is that PyPy project is very different than most
+projects out there. It's also different from a classic compiler project,
+so academic courses about compilers often don't apply or lead in the wrong
+direction. However, if you want to understand how designing & building a runtime
+works in the real world then this is a great project!
+
+Getting involved
+^^^^^^^^^^^^^^^^
+
+PyPy employs a relatively standard open-source development process. You are
+encouraged as a first step to join our `pypy-dev mailing list`_ and IRC channel,
+details of which can be found in our :ref:`contact <contact>` section. The folks
+there are very friendly, and can point you in the right direction.
+
+We give out commit rights usually fairly liberally, so if you want to do something
+with PyPy, you can become a committer. We also run frequent coding sprints which
+are separately announced and often happen around Python conferences such as
+EuroPython or PyCon. Upcoming events are usually announced on `the blog`_.
+
+Further Reading: :ref:`Contact <contact>`
+
+.. _the blog: http://morepypy.blogspot.com
+.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
+
+
+Your first contribution
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The first and most important rule how **not** to contribute to PyPy is
+"just hacking a feature". This won't work, and you'll find your PR will typically
+require a lot of re-work. There are a few reasons why not:
+
+* build times are large
+* PyPy has very thick layer separation
+* context of the cPython runtime is often required
+
+Instead, reach out on the dev mailing list or the IRC channel, and we're more
+than happy to help! :)
+
+Some ideas for first contributions are:
+
+* Documentation - this will give you an understanding of the pypy architecture
+* Test failures - find a failing test in the `nightly builds`_, and fix it
+* Missing language features - these are listed in our `issue tracker`_
+
+.. _nightly builds: http://buildbot.pypy.org/nightly/
+.. _issue tracker: https://bitbucket.org/pypy/pypy/issues
+
+Source Control
+--------------
+
+PyPy development is based a typical fork/pull request based workflow, centered
+around Mercurial (hg), hosted on Bitbucket. If you have not used this workflow
+before, a good introduction can be found here:
+
+    https://www.atlassian.com/git/tutorials/comparing-workflows/forking-workflow
+
+The cycle for a new PyPy contributor goes typically like this:
+
+Fork & Clone
+------------
+
+* Make an account on bitbucket_.
+
+* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
+  icons).  You get a fork of the repository, e.g. in
+  `https://bitbucket.org/yourname/pypy/`.
+
+* Clone your new repo (i.e. the fork) to your local machine with the command
+  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
+  operation but only ever needs to be done once.  See also
+  http://pypy.org/download.html#building-from-source .
+  If you already cloned
+  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
+  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
+  your clone to contain the line ``default =
+  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
+  up``.  If you already have such a clone but don't want to change it,
+  you can clone that copy with ``hg clone /path/to/other/copy``, and
+  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
+
+* Now you have a complete copy of the PyPy repo.  Make a branch
+  with a command like ``hg branch name_of_your_branch``.
+
+Edit
+----
+
+* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
+  to make Mercurial aware of new files you added, e.g. new test files.
+  Use ``hg status`` to see if there are such files.  Write and run tests!
+  (See the rest of this page.)
+
+* Commit regularly with ``hg commit``.  A one-line commit message is
+  fine.  We love to have tons of commits; make one as soon as you have
+  some progress, even if it is only some new test that doesn't pass yet,
+  or fixing things even if not all tests pass.  Step by step, you are
+  building the history of your changes, which is the point of a version
+  control system.  (There are commands like ``hg log`` and ``hg up``
+  that you should read about later, to learn how to navigate this
+  history.)
+
+* The commits stay on your machine until you do ``hg push`` to "push"
+  them back to the repo named in the file ``.hg/hgrc``.  Repos are
+  basically just collections of commits (a commit is also called a
+  changeset): there is one repo per url, plus one for each local copy on
+  each local machine.  The commands ``hg push`` and ``hg pull`` copy
+  commits around, with the goal that all repos in question end up with
+  the exact same set of commits.  By opposition, ``hg up`` only updates
+  the "working copy" by reading the local repository, i.e. it makes the
+  files that you see correspond to the latest (or any other) commit
+  locally present.
+
+* You should push often; there is no real reason not to.  Remember that
+  even if they are pushed, with the setup above, the commits are (1)
+  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
+  named.  Yes, they are publicly visible, but don't worry about someone
+  walking around the thousands of repos on bitbucket saying "hah, look
+  at the bad coding style of that guy".  Try to get into the mindset
+  that your work is not secret and it's fine that way.  We might not
+  accept it as is for PyPy, asking you instead to improve some things,
+  but we are not going to judge you.
+
+Pull Request
+------------
+
+* The final step is to open a pull request, so that we know that you'd
+  like to merge that branch back to the original ``pypy/pypy`` repo.
+  This can also be done several times if you have interesting
+  intermediate states, but if you get there, then we're likely to
+  proceed to the next stage, which is...
+
+* Get a regular account for pushing directly to
+  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
+  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
+  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
+  then be pushed directly to the official repo, but (if you follow these
+  rules) they are still on a branch, and we can still review the
+  branches you want to merge.
+
+* If you get closer to the regular day-to-day development, you'll notice
+  that we generally push small changes as one or a few commits directly
+  to the branch ``default``.  Also, we often collaborate even if we are
+  on other branches, which do not really "belong" to anyone.  At this
+  point you'll need ``hg merge`` and learn how to resolve conflicts that
+  sometimes occur when two people try to push different commits in
+  parallel on the same branch.  But it is likely an issue for later ``:-)``
+
+.. _bitbucket: https://bitbucket.org/
+
+
+Architecture
+^^^^^^^^^^^^
+
+PyPy has layers. Just like ogres or onions. Those layers help us keep the
+respective parts separated enough to be worked on independently and make the
+complexity manageable. This is, again, just a sanity requirement for such
+a complex project. For example writing a new optimization for the JIT usually
+does **not** involve touching a Python interpreter at all or the JIT assembler
+backend or the garbage collector. Instead it requires writing small tests in
+``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
+After that, you can just compile PyPy and things should just work.
+
+Further Reading: :doc:`architecture <architecture>`
+
+Where to start?
+---------------
+
+PyPy is made from parts that are relatively independent of each other.
+You should start looking at the part that attracts you most (all paths are
+relative to the PyPy top level directory).  You may look at our
+:doc:`directory reference <dir-reference>` or start off at one of the following
+points:
+
+*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
+   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
+   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
+   function objects and argument passing in :source:`pypy/interpreter/function.py`
+   and :source:`pypy/interpreter/argument.py`, the object space interface
+   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
+   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
+   Core types supporting the bytecode interpreter are defined in
+   :source:`pypy/interpreter/typedef.py`.
+
+*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
+   and grammar files that allow it to parse the syntax of various Python
+   versions. Once the grammar has been processed, the parser can be
+   translated by the above machinery into efficient code.
+
+*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
+   contains a modified version of the compiler package from CPython
+   that fixes some bugs and is translatable.
+
+*  :source:`pypy/objspace/std` contains the
+   :ref:`Standard object space <standard-object-space>`.  The main file
+   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
+   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
+   as a first approximation.  (Some types have multiple implementations.)
+
+Building
+^^^^^^^^
+
+For building PyPy, we recommend installing a pre-built PyPy first (see
+:doc:`install`). It is possible to build PyPy with CPython, but it will take a
+lot longer to run -- depending on your architecture, between two and three
+times as long.
+
+Further Reading: :doc:`Build <build>`
+
+Coding Guide
+------------
+
+As well as the usual pep8 and formatting standards, there are a number of
+naming conventions and coding styles that are important to understand before
+browsing the source.
+
+Further Reading: :doc:`Coding Guide <coding-guide>`
+
+Testing
+^^^^^^^
+
+Test driven development
+-----------------------
+
+Instead, we practice a lot of test driven development. This is partly because
+of very high quality requirements for compilers and partly because there is
+simply no other way to get around such complex project, that will keep you sane.
+There are probably people out there who are smart enough not to need it, we're
+not one of those. You may consider familiarizing yourself with `pytest`_,
+since this is a tool we use for tests.
+This leads to the next issue:
+
+.. _pytest: http://pytest.org/
+
+py.test and the py lib
+----------------------
+
+The `py.test testing tool`_ drives all our testing needs.
+
+We use the `py library`_ for filesystem path manipulations, terminal
+writing, logging and some other support  functionality.
+
+You don't necessarily need to install these two libraries because
+we also ship them inlined in the PyPy source tree.
+
+.. _py library: http://pylib.readthedocs.org/
+
+Running PyPy's unit tests
+-------------------------
+
+PyPy development always was and is still thoroughly test-driven.
+We use the flexible `py.test testing tool`_ which you can `install independently
+<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
+
+The PyPy source tree comes with an inlined version of ``py.test``
+which you can invoke by typing::
+
+    python pytest.py -h
+
+This is usually equivalent to using an installed version::
+
+    py.test -h
+
+If you encounter problems with the installed version
+make sure you have the correct version installed which
+you can find out with the ``--version`` switch.
+
+You will need the `build requirements`_ to run tests successfully, since many of
+them compile little pieces of PyPy and then run the tests inside that minimal
+interpreter. The `cpyext` tests also require `pycparser`, and many tests build
+cases with `hypothesis`.
+
+Now on to running some tests.  PyPy has many different test directories
+and you can use shell completion to point at directories or files::
+
+    py.test pypy/interpreter/test/test_pyframe.py
+
+    # or for running tests of a whole subdirectory
+    py.test pypy/interpreter/
+
+See `py.test usage and invocations`_ for some more generic info
+on how you can run tests.
+
+Beware trying to run "all" pypy tests by pointing to the root
+directory or even the top level subdirectory ``pypy``.  It takes
+hours and uses huge amounts of RAM and is not recommended.
+
+To run CPython regression tests you can point to the ``lib-python``
+directory::
+
+    py.test lib-python/2.7/test/test_datetime.py
+
+This will usually take a long time because this will run
+the PyPy Python interpreter on top of CPython.  On the plus
+side, it's usually still faster than doing a full translation
+and running the regression test with the translated PyPy Python
+interpreter.
+
+.. _py.test testing tool: http://pytest.org
+.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
+.. _`build requirements`: build.html#install-build-time-dependencies
+
+Testing After Translation
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+While the usual invocation of `pytest` translates a piece of RPython code and
+runs it, we have a test extension to run tests without translation, directly
+on the host python. This is very convenient for modules such as `cpyext`, to
+compare and contrast test results between CPython and PyPy. Untranslated tests
+are invoked by using the `-A` or `--runappdirect` option to `pytest`::
+
+    python2 pytest.py -A pypy/module/cpyext/test
+
+where `python2` can be either `python2` or `pypy2`. On the `py3` branch, the
+collection phase must be run with `python2` so untranslated tests are run
+with::
+
+    cpython2 pytest.py -A pypy/module/cpyext/test --python=path/to/pypy3
+
+
+Tooling & Utilities
+^^^^^^^^^^^^^^^^^^^
+
+If you are interested in the inner workings of the PyPy Python interpreter,
+there are some features of the untranslated Python interpreter that allow you
+to introspect its internals.
+
+
+Interpreter-level console
+-------------------------
+
+To start interpreting Python with PyPy, install a C compiler that is
+supported by distutils and use Python 2.7 or greater to run PyPy::
+
+    cd pypy
+    python bin/pyinteractive.py
+
+After a few seconds (remember: this is running on top of CPython), you should
+be at the PyPy prompt, which is the same as the Python prompt, but with an
+extra ">".
+
+If you press
+<Ctrl-C> on the console you enter the interpreter-level console, a
+usual CPython console.  You can then access internal objects of PyPy
+(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
+prompt with the prefix ``w_``::
+
+    >>>> a = 123
+    >>>> <Ctrl-C>
+    *** Entering interpreter-level console ***
+    >>> w_a
+    W_IntObject(123)
+
+The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
+
+    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
+    >>> <Ctrl-D>
+    *** Leaving interpreter-level console ***
+
+    KeyboardInterrupt
+    >>>> l
+    [1, 'abc']
+
+Note that the prompt of the interpreter-level console is only '>>>' since
+it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
+Linux) or <Ctrl-Z>, <Enter> (under Windows).
+
+Also note that not all modules are available by default in this mode (for
+example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
+``--withmod-...`` command line options.
+
+You may be interested in reading more about the distinction between
+:ref:`interpreter-level and app-level <interpreter-level>`.
+
+pyinteractive.py options
+------------------------
+
+To list the PyPy interpreter command line options, type::
+
+    cd pypy
+    python bin/pyinteractive.py --help
+
+pyinteractive.py supports most of the options that CPython supports too (in addition to a
+large amount of options that can be used to customize pyinteractive.py).
+As an example of using PyPy from the command line, you could type::
+
+    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
+
+Alternatively, as with regular Python, you can simply give a
+script name on the command line::
+
+    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
+
+The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
+default almost none of them are, because initializing them takes time.
+If you want anyway to enable all built-in modules, you can use
+``--allworkingmodules``.
+
+See our :doc:`configuration sections <config/index>` for details about what all the commandline
+options do.
+
+
+.. _trace example:
+
+Tracing bytecode and operations on objects
+------------------------------------------
+
+You can use a simple tracing mode to monitor the interpretation of
+bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
+PyPy console::
+
+    >>>> __pytrace__ = 1
+    Tracing enabled
+    >>>> x = 5
+            <module>:           LOAD_CONST    0 (5)
+            <module>:           STORE_NAME    0 (x)
+            <module>:           LOAD_CONST    1 (None)
+            <module>:           RETURN_VALUE    0
+    >>>> x
+            <module>:           LOAD_NAME    0 (x)
+            <module>:           PRINT_EXPR    0
+    5
+            <module>:           LOAD_CONST    0 (None)
+            <module>:           RETURN_VALUE    0
+    >>>>
+
+
+Demos
+^^^^^
+
+The `example-interpreter`_ repository contains an example interpreter
+written using the RPython translation toolchain.
+
+.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
+
+
+graphviz & pygame for flow graph viewing (highly recommended)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+graphviz and pygame are both necessary if you want to look at generated flow
+graphs:
+
+    graphviz: http://www.graphviz.org/Download.php
+
+    pygame: http://www.pygame.org/download.shtml
+
diff --git a/pypy/doc/discussion/ctypes-implementation.rst b/pypy/doc/discussion/ctypes-implementation.rst
--- a/pypy/doc/discussion/ctypes-implementation.rst
+++ b/pypy/doc/discussion/ctypes-implementation.rst
@@ -141,28 +141,3 @@
 
 .. _pyglet: http://pyglet.org/
 
-
-ctypes configure
------------------
-
-We also released ``ctypes-configure``, which is an experimental package
-trying to approach the portability issues of ctypes-based code.
-
-idea
-~~~~
-
-One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes c
-compiler (via distutils) for various platform-dependent details like
-exact sizes of types (for example size_t), ``#defines``, exact outline of
-structures etc. It replaces in this regard code generator (h2py).
-
-installation
-~~~~~~~~~~~~
-
-``easy_install ctypes_configure``
-
-usage
-~~~~~
-
-:source:`ctypes_configure/doc/sample.py` explains in details how to use it.
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -1,5 +1,5 @@
-Embedding PyPy
-==============
+Embedding PyPy (DEPRECATED)
+===========================
 
 PyPy has a very minimal and a very strange embedding interface, based on
 the usage of `cffi`_ and the philosophy that Python is a better language than
diff --git a/pypy/doc/eventhistory.rst b/pypy/doc/eventhistory.rst
--- a/pypy/doc/eventhistory.rst
+++ b/pypy/doc/eventhistory.rst
@@ -40,11 +40,9 @@
 Main focus of the sprint will be on the goals of the upcoming June 0.9
 release.
 
-Read more in `the sprint announcement`__, see who is  planning to attend
-on the `people page`_.
+Read more about `the sprint`__
 
-__ https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/announce.html
-.. _people page: https://bitbucket.org/pypy/extradoc/raw/tip/sprintinfo/ddorf2006/people.txt
+__ https://bitbucket.org/pypy/extradoc/src/extradoc/sprintinfo/ddorf2006/
 
 
 PyPy sprint at Akihabara (Tokyo, Japan)
diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst
--- a/pypy/doc/extradoc.rst
+++ b/pypy/doc/extradoc.rst
@@ -75,12 +75,12 @@
 .. _A Way Forward in Parallelising Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2014/position-paper.pdf
 .. _Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf
 .. _Allocation Removal by Partial Evaluation in a Tracing JIT: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/pepm2011/bolz-allocation-removal.pdf
-.. _Towards a Jitting VM for Prolog Execution: http://www.stups.uni-duesseldorf.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
+.. _Towards a Jitting VM for Prolog Execution: http://stups.hhu.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
 .. _High performance implementation of Python for CLI/.NET with JIT compiler generation for dynamic languages: http://buildbot.pypy.org/misc/antocuni-thesis.pdf
 .. _How to *not* write Virtual Machines for Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/tip/talk/dyla2007/dyla.pdf
 .. _`Tracing the Meta-Level: PyPy's Tracing JIT Compiler`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009/bolz-tracing-jit.pdf
 .. _`Faster than C#: Efficient Implementation of Dynamic Languages on .NET`: https://bitbucket.org/pypy/extradoc/raw/tip/talk/icooolps2009-dotnet/cli-jit.pdf
-.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: http://stups.hhu.de/mediawiki/images/b/b9/Master_bolz.pdf
+.. _Automatic JIT Compiler Generation with Runtime Partial Evaluation: https://www.researchgate.net/profile/Davide_Ancona/publication/252023163_Automatic_generation_of_JIT_compilers_for_dynamic_languages_in_NET/links/53f2098e0cf2bc0c40e70023/Automatic-generation-of-JIT-compilers-for-dynamic-languages-in-NET.pdf
 .. _`RPython: A Step towards Reconciling Dynamically and Statically Typed OO Languages`: http://www.disi.unige.it/person/AnconaD/papers/DynamicLanguages_abstracts.html#AACM-DLS07
 .. _EU Reports: index-report.html
 .. _Hardware Transactional Memory Support for Lightweight Dynamic Language Evolution: http://sabi.net/nriley/pubs/dls6-riley.pdf
@@ -368,6 +368,6 @@
 .. _LLVM: http://llvm.org/
 .. _IronPython: http://ironpython.codeplex.com/
 .. _Dynamic Native Optimization of Native Interpreters: http://people.csail.mit.edu/gregs/dynamorio.html
-.. _JikesRVM: http://jikesrvm.org/
+.. _JikesRVM: http://www.jikesrvm.org/
 .. _Tunes: http://tunes.org
 .. _old Tunes Wiki: http://buildbot.pypy.org/misc/cliki.tunes.org/
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -67,7 +67,7 @@
 you may need to run the command with `sudo` for a global installation.
 The other commands of ``setup.py`` are available too, like ``build``.
 
-.. _PyPI: https://pypi.python.org/pypi
+.. _PyPI: https://pypi.org
 .. _`use virtualenv (as documented here)`: install.html#installing-using-virtualenv
 
 
@@ -360,7 +360,7 @@
 (produced during a sprint).  On the `PyPy bitbucket page`_ there is also a
 Scheme and an Io implementation; both of these are unfinished at the moment.
 
-.. _Topaz: http://topazruby.com/
+.. _Topaz: http://docs.topazruby.com/en/latest/
 .. _Hippy: http://morepypy.blogspot.ch/2012/07/hello-everyone.html
 .. _JavaScript interpreter: https://bitbucket.org/pypy/lang-js/
 .. _Prolog interpreter: https://bitbucket.org/cfbolz/pyrolog/
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -152,7 +152,7 @@
 to wait until it reaches a point in which the interpreter is in a known state
 and calling user-defined code is harmless.  It might happen that multiple
 events occur before the hook is invoked: in this case, you can inspect the
-value ``stats.count`` to know how many times the event occured since the last
+value ``stats.count`` to know how many times the event occurred since the last
 time the hook was called.  Similarly, ``stats.duration`` contains the
 **total** time spent by the GC for this specific event since the last time the
 hook was called.
@@ -163,7 +163,7 @@
 The attributes for ``GcMinorStats`` are:
 
 ``count``
-    The number of minor collections occured since the last hook call.
+    The number of minor collections occurred since the last hook call.
 
 ``duration``
     The total time spent inside minor collections since the last hook
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
deleted file mode 100644
--- a/pypy/doc/getting-started-dev.rst
+++ /dev/null
@@ -1,345 +0,0 @@
-Getting Started Developing With PyPy
-====================================
-
-.. contents::
-
-
-Using Mercurial
----------------
-
-PyPy development is based on Mercurial (hg).  If you are not used to
-version control, the cycle for a new PyPy contributor goes typically
-like this:
-
-* Make an account on bitbucket_.
-
-* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
-  icons).  You get a fork of the repository, e.g. in
-  https://bitbucket.org/yourname/pypy/.
-
-* Clone this new repo (i.e. the fork) to your local machine with the command 
-  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
-  operation but only ever needs to be done once.  See also 
-  http://pypy.org/download.html#building-from-source .
-  If you already cloned
-  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
-  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
-  your clone to contain the line ``default =
-  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
-  up``.  If you already have such a clone but don't want to change it,
-  you can clone that copy with ``hg clone /path/to/other/copy``, and
-  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
-
-* Now you have a complete copy of the PyPy repo.  Make a branch
-  with a command like ``hg branch name_of_your_branch``.
-
-* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
-  to make Mercurial aware of new files you added, e.g. new test files.
-  Use ``hg status`` to see if there are such files.  Write and run tests!
-  (See the rest of this page.)
-
-* Commit regularly with ``hg commit``.  A one-line commit message is
-  fine.  We love to have tons of commits; make one as soon as you have
-  some progress, even if it is only some new test that doesn't pass yet,
-  or fixing things even if not all tests pass.  Step by step, you are
-  building the history of your changes, which is the point of a version
-  control system.  (There are commands like ``hg log`` and ``hg up``
-  that you should read about later, to learn how to navigate this
-  history.)
-
-* The commits stay on your machine until you do ``hg push`` to "push"
-  them back to the repo named in the file ``.hg/hgrc``.  Repos are
-  basically just collections of commits (a commit is also called a
-  changeset): there is one repo per url, plus one for each local copy on
-  each local machine.  The commands ``hg push`` and ``hg pull`` copy
-  commits around, with the goal that all repos in question end up with
-  the exact same set of commits.  By opposition, ``hg up`` only updates
-  the "working copy" by reading the local repository, i.e. it makes the
-  files that you see correspond to the latest (or any other) commit
-  locally present.
-
-* You should push often; there is no real reason not to.  Remember that
-  even if they are pushed, with the setup above, the commits are (1)
-  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
-  named.  Yes, they are publicly visible, but don't worry about someone
-  walking around the thousands of repos on bitbucket saying "hah, look
-  at the bad coding style of that guy".  Try to get into the mindset
-  that your work is not secret and it's fine that way.  We might not
-  accept it as is for PyPy, asking you instead to improve some things,
-  but we are not going to judge you.
-
-* The final step is to open a pull request, so that we know that you'd
-  like to merge that branch back to the original ``pypy/pypy`` repo.
-  This can also be done several times if you have interesting
-  intermediate states, but if you get there, then we're likely to
-  proceed to the next stage, which is...
-
-* Get a regular account for pushing directly to
-  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
-  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
-  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
-  then be pushed directly to the official repo, but (if you follow these
-  rules) they are still on a branch, and we can still review the
-  branches you want to merge.
-
-* If you get closer to the regular day-to-day development, you'll notice
-  that we generally push small changes as one or a few commits directly
-  to the branch ``default``.  Also, we often collaborate even if we are
-  on other branches, which do not really "belong" to anyone.  At this
-  point you'll need ``hg merge`` and learn how to resolve conflicts that
-  sometimes occur when two people try to push different commits in
-  parallel on the same branch.  But it is likely an issue for later ``:-)``
-
-.. _bitbucket: https://bitbucket.org/
-
-
-Running PyPy's unit tests
--------------------------
-
-PyPy development always was and is still thoroughly test-driven.
-We use the flexible `py.test testing tool`_ which you can `install independently
-<http://pytest.org/latest/getting-started.html#getstarted>`_ and use for other projects.
-
-The PyPy source tree comes with an inlined version of ``py.test``
-which you can invoke by typing::
-
-    python pytest.py -h
-
-This is usually equivalent to using an installed version::
-
-    py.test -h
-
-If you encounter problems with the installed version
-make sure you have the correct version installed which
-you can find out with the ``--version`` switch.
-
-You will need the `build requirements`_ to run tests successfully, since many of
-them compile little pieces of PyPy and then run the tests inside that minimal
-interpreter
-
-Now on to running some tests.  PyPy has many different test directories
-and you can use shell completion to point at directories or files::
-
-    py.test pypy/interpreter/test/test_pyframe.py
-
-    # or for running tests of a whole subdirectory
-    py.test pypy/interpreter/
-
-See `py.test usage and invocations`_ for some more generic info
-on how you can run tests.
-
-Beware trying to run "all" pypy tests by pointing to the root
-directory or even the top level subdirectory ``pypy``.  It takes
-hours and uses huge amounts of RAM and is not recommended.
-
-To run CPython regression tests you can point to the ``lib-python``
-directory::
-
-    py.test lib-python/2.7/test/test_datetime.py
-
-This will usually take a long time because this will run
-the PyPy Python interpreter on top of CPython.  On the plus
-side, it's usually still faster than doing a full translation
-and running the regression test with the translated PyPy Python
-interpreter.
-
-.. _py.test testing tool: http://pytest.org
-.. _py.test usage and invocations: http://pytest.org/latest/usage.html#usage
-.. _`build requirements`: build.html#install-build-time-dependencies
-
-Special Introspection Features of the Untranslated Python Interpreter
----------------------------------------------------------------------
-
-If you are interested in the inner workings of the PyPy Python interpreter,
-there are some features of the untranslated Python interpreter that allow you
-to introspect its internals.
-
-
-Interpreter-level console
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To start interpreting Python with PyPy, install a C compiler that is
-supported by distutils and use Python 2.7 or greater to run PyPy::
-
-    cd pypy
-    python bin/pyinteractive.py
-
-After a few seconds (remember: this is running on top of CPython), you should
-be at the PyPy prompt, which is the same as the Python prompt, but with an
-extra ">".
-
-If you press
-<Ctrl-C> on the console you enter the interpreter-level console, a
-usual CPython console.  You can then access internal objects of PyPy
-(e.g. the :ref:`object space <objspace>`) and any variables you have created on the PyPy
-prompt with the prefix ``w_``::
-
-    >>>> a = 123
-    >>>> <Ctrl-C>
-    *** Entering interpreter-level console ***
-    >>> w_a
-    W_IntObject(123)
-
-The mechanism works in both directions. If you define a variable with the ``w_`` prefix on the interpreter-level, you will see it on the app-level::
-
-    >>> w_l = space.newlist([space.wrap(1), space.wrap("abc")])
-    >>> <Ctrl-D>
-    *** Leaving interpreter-level console ***
-
-    KeyboardInterrupt
-    >>>> l
-    [1, 'abc']
-
-Note that the prompt of the interpreter-level console is only '>>>' since
-it runs on CPython level. If you want to return to PyPy, press <Ctrl-D> (under
-Linux) or <Ctrl-Z>, <Enter> (under Windows).
-
-Also note that not all modules are available by default in this mode (for
-example: ``_continuation`` needed by ``greenlet``) , you may need to use one of
-``--withmod-...`` command line options.
-
-You may be interested in reading more about the distinction between
-:ref:`interpreter-level and app-level <interpreter-level>`.
-
-pyinteractive.py options
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-To list the PyPy interpreter command line options, type::
-
-    cd pypy
-    python bin/pyinteractive.py --help
-
-pyinteractive.py supports most of the options that CPython supports too (in addition to a
-large amount of options that can be used to customize pyinteractive.py).
-As an example of using PyPy from the command line, you could type::
-
-    python pyinteractive.py --withmod-time -c "from test import pystone; pystone.main(10)"
-
-Alternatively, as with regular Python, you can simply give a
-script name on the command line::
-
-    python pyinteractive.py --withmod-time ../../lib-python/2.7/test/pystone.py 10
-
-The ``--withmod-xxx`` option enables the built-in module ``xxx``.  By
-default almost none of them are, because initializing them takes time.
-If you want anyway to enable all built-in modules, you can use
-``--allworkingmodules``.
-
-See our :doc:`configuration sections <config/index>` for details about what all the commandline
-options do.
-
-
-.. _trace example:
-
-Tracing bytecode and operations on objects
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can use a simple tracing mode to monitor the interpretation of
-bytecodes.  To enable it, set ``__pytrace__ = 1`` on the interactive
-PyPy console::
-
-    >>>> __pytrace__ = 1
-    Tracing enabled
-    >>>> x = 5
-            <module>:           LOAD_CONST    0 (5)
-            <module>:           STORE_NAME    0 (x)
-            <module>:           LOAD_CONST    1 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>> x
-            <module>:           LOAD_NAME    0 (x)
-            <module>:           PRINT_EXPR    0 
-    5
-            <module>:           LOAD_CONST    0 (None)
-            <module>:           RETURN_VALUE    0 
-    >>>>
-
-
-Demos
------
-
-The `example-interpreter`_ repository contains an example interpreter
-written using the RPython translation toolchain.
-
-.. _example-interpreter: https://bitbucket.org/pypy/example-interpreter
-
-
-Additional Tools for running (and hacking) PyPy
------------------------------------------------
-
-We use some optional tools for developing PyPy. They are not required to run
-the basic tests or to get an interactive PyPy prompt but they help to
-understand  and debug PyPy especially for the translation process.
-
-
-graphviz & pygame for flow graph viewing (highly recommended)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-graphviz and pygame are both necessary if you
-want to look at generated flow graphs:
-
-	graphviz: http://www.graphviz.org/Download.php
-
-	pygame: http://www.pygame.org/download.shtml
-
-
-py.test and the py lib
-~~~~~~~~~~~~~~~~~~~~~~
-
-The `py.test testing tool`_ drives all our testing needs.
-
-We use the `py library`_ for filesystem path manipulations, terminal
-writing, logging and some other support  functionality.
-
-You don't necessarily need to install these two libraries because
-we also ship them inlined in the PyPy source tree.
-
-.. _py library: http://pylib.readthedocs.org/
-
-
-Getting involved
-----------------
-
-PyPy employs an open development process.  You are invited to join our
-`pypy-dev mailing list`_ or look at the other :ref:`contact
-possibilities <contact>`.  Usually we give out commit rights fairly liberally, so if you
-want to do something with PyPy, you can become a committer. We also run frequent
-coding sprints which are separately announced and often happen around Python
-conferences such as EuroPython or PyCon. Upcoming events are usually announced
-on `the blog`_.
-
-.. _the blog: http://morepypy.blogspot.com
-.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
-
-
-.. _start-reading-sources:
-
-Where to start reading the sources
-----------------------------------
-
-PyPy is made from parts that are relatively independent of each other.
-You should start looking at the part that attracts you most (all paths are
-relative to the PyPy top level directory).  You may look at our :doc:`directory reference <dir-reference>`
-or start off at one of the following points:
-
-*  :source:`pypy/interpreter` contains the bytecode interpreter: bytecode dispatcher
-   in :source:`pypy/interpreter/pyopcode.py`, frame and code objects in
-   :source:`pypy/interpreter/eval.py` and :source:`pypy/interpreter/pyframe.py`,
-   function objects and argument passing in :source:`pypy/interpreter/function.py`
-   and :source:`pypy/interpreter/argument.py`, the object space interface
-   definition in :source:`pypy/interpreter/baseobjspace.py`, modules in
-   :source:`pypy/interpreter/module.py` and :source:`pypy/interpreter/mixedmodule.py`.
-   Core types supporting the bytecode interpreter are defined in :source:`pypy/interpreter/typedef.py`.
-
-*  :source:`pypy/interpreter/pyparser` contains a recursive descent parser,
-   and grammar files that allow it to parse the syntax of various Python
-   versions. Once the grammar has been processed, the parser can be
-   translated by the above machinery into efficient code.
-
-*  :source:`pypy/interpreter/astcompiler` contains the compiler.  This
-   contains a modified version of the compiler package from CPython
-   that fixes some bugs and is translatable.
-
-*  :source:`pypy/objspace/std` contains the :ref:`Standard object space <standard-object-space>`.  The main file
-   is :source:`pypy/objspace/std/objspace.py`.  For each type, the file
-   ``xxxobject.py`` contains the implementation for objects of type ``xxx``,
-   as a first approximation.  (Some types have multiple implementations.)
diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst
deleted file mode 100644
--- a/pypy/doc/how-to-contribute.rst
+++ /dev/null
@@ -1,93 +0,0 @@
-How to contribute to PyPy
-=========================
-
-This page describes how to contribute to the PyPy project. The first thing
-to remember is that PyPy project is very different than most projects out there.
-It's also different from a classic compiler project, so academic courses
-about compilers often don't apply or lead in the wrong direction.
-
-
-Don't just hack
----------------
-
-The first and most important rule how not to contribute to PyPy is
-"just hacking". This won't work. There are two major reasons why not
--- build times are large and PyPy has very thick layer separation which
-make it harder to "just hack a feature".
-
-
-Test driven development
------------------------
-
-Instead, we practice a lot of test driven development. This is partly because
-of very high quality requirements for compilers and partly because there is
-simply no other way to get around such complex project, that will keep you sane.
-There are probably people out there who are smart enough not to need it, we're
-not one of those. You may consider familiarizing yourself with `pytest`_,
-since this is a tool we use for tests.
-This leads to the next issue:
-
-.. _pytest: http://pytest.org/
-
-
-Layers
-------
-
-PyPy has layers. Just like Ogres or onions.
-Those layers help us keep the respective parts separated enough
-to be worked on independently and make the complexity manageable. This is,
-again, just a sanity requirement for such a complex project. For example writing
-a new optimization for the JIT usually does **not** involve touching a Python
-interpreter at all or the JIT assembler backend or the garbage collector.
-Instead it requires writing small tests in
-``rpython/jit/metainterp/optimizeopt/test/test_*`` and fixing files there.
-After that, you can just compile PyPy and things should just work.
-
-The short list of layers for further reading. For each of those layers, a good
-entry point is a test subdirectory in respective directories. It usually
-describes (better or worse) the interfaces between the submodules. For the
-``pypy`` subdirectory, most tests are small snippets of python programs that
-check for correctness (calls ``AppTestXxx``) that will call the appropriate
-part of the interpreter. For the ``rpython`` directory, most tests are small
-RPython interpreters that perform certain tasks. To see how they translate
-to low-level graphs, run them with ``--view``. To see small interpreters
-with a JIT compiler, use ``--viewloops`` option.
-
-* **python interpreter** - it's the part implemented in the ``pypy/`` directory.
-  It's implemented in RPython, which is a high level static language with
-  classes, garbage collection, just-in-time compiler generation and the ability
-  to call C. A cool part about it is that it can be run untranslated, so all
-  the tests are runnable without translating PyPy.
-
-  **interpreter** contains the interpreter core
-
-  **objspace** contains implementations of various objects exported to
-  the Python layer
-
-  **module** directory contains extension modules written in RPython
-
-* **rpython compiler** that resides in ``rpython/annotator`` and
-  ``rpython/rtyper`` directories. Consult `Getting Started with RPython`_
-  for further reading
-
-* **JIT generator** lives in ``rpython/jit`` directory. optimizations live
-  in ``rpython/jit/metainterp/optimizeopt``, the main JIT in
-  ``rpython/jit/metainterp`` (runtime part) and
-  ``rpython/jit/codewriter`` (translation-time part). Backends live in
-  ``rpython/jit/backend``.
-
-* **garbage collection** lives in ``rpython/memory``
-
-The rest of directories serve specific niche goal and are unlikely a good
-entry point.
-
-
-More documentation
-------------------
-
-* `Getting Started Developing With PyPy`_
-
-* `Getting Started with RPython`_
-
-.. _`Getting Started Developing With PyPy`: getting-started-dev.html
-.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -34,6 +34,7 @@
    whatsnew-2.0.0-beta1.rst
    whatsnew-1.9.rst
 
+
 CPython 3.5 compatible versions
 -------------------------------
 
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -9,7 +9,7 @@
 
 * If you're interested in trying PyPy out, check out the :doc:`installation instructions <install>`.
 
-* If you want to help develop PyPy, please have a look at :doc:`how to contribute <how-to-contribute>`
+* If you want to help develop PyPy, please have a look at :doc:`contributing <contributing>`
   and get in touch (:ref:`contact`)!
 
 All of the documentation and source code is available under the MIT license,
@@ -31,6 +31,7 @@
   introduction
   install
   build
+  windows
   faq
 
 
@@ -40,43 +41,30 @@
 ----------
 
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
   cpython_differences
   extending
-  embedding
   gc_info
   jit-hooks
   stackless
   __pypy__-module
-  objspace-proxies
   sandbox
   stm
-  windows
 
-
-.. _developing-pypy:
-
-Development documentation
--------------------------
+Development
+-----------
 
 .. toctree::
-  :maxdepth: 1
+  :maxdepth: 2
 
-  getting-started-dev
-  how-to-contribute
-  you-want-to-help
+  contributing
   architecture
   configuration
   project-ideas
   project-documentation
   how-to-release
 
-.. TODO: audit ^^
-
-
-.. TODO: Fill this in
-
 
 Further resources
 -----------------
@@ -84,13 +72,10 @@
 .. toctree::
   :maxdepth: 1
 
-  extradoc
-  eventhistory
-  discussions
   index-of-release-notes
   index-of-whatsnew
   contributor
-
+  glossary
 
 .. _contact:
 
@@ -118,7 +103,7 @@
     the `development mailing list`_.
 
 .. _#pypy on irc.freenode.net: irc://irc.freenode.net/pypy
-.. _here: http://www.tismer.com/pypy/irc-logs/pypy/
+.. _here: https://botbot.me/freenode/pypy/
 .. _Development mailing list: http://mail.python.org/mailman/listinfo/pypy-dev
 .. _Commit mailing list: http://mail.python.org/mailman/listinfo/pypy-commit
 .. _Development bug/feature tracker: https://bitbucket.org/pypy/pypy/issues
diff --git a/pypy/doc/interpreter.rst b/pypy/doc/interpreter.rst
--- a/pypy/doc/interpreter.rst
+++ b/pypy/doc/interpreter.rst
@@ -102,7 +102,7 @@
 program flows with homogeneous name-value assignments on
 function invocations.
 
-.. _how-to guide for descriptors: http://users.rcn.com/python/download/Descriptor.htm
+.. _how-to guide for descriptors: https://docs.python.org/3/howto/descriptor.html
 
 
 Bytecode Interpreter Implementation Classes
diff --git a/pypy/doc/man/pypy3.1.rst b/pypy/doc/man/pypy3.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/man/pypy3.1.rst
@@ -0,0 +1,135 @@
+=======
+ pypy3
+=======
+
+.. note: this is turned into a regular man page "pypy3.1" by
+   doing "make man" in pypy/doc/
+
+SYNOPSIS
+========
+
+``pypy3`` [*options*]
+[``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ]
+[*arg*\ ...]
+
+OPTIONS
+=======
+
+-i
+    Inspect interactively after running script.
+
+-O
+    Skip assert statements.
+
+-OO
+    Remove docstrings when importing modules in addition to ``-O``.
+
+-c CMD
+    Program passed in as ``CMD`` (terminates option list).
+
+-S
+    Do not ``import site`` on initialization.
+
+-s
+    Don't add the user site directory to `sys.path`.
+
+-u
+    Unbuffered binary ``stdout`` and ``stderr``.
+
+-h, --help
+    Show a help message and exit.
+
+-m MOD
+    Library module to be run as a script (terminates option list).
+
+-W ARG
+    Warning control (*arg* is *action*:*message*:*category*:*module*:*lineno*).
+
+-E
+    Ignore environment variables (such as ``PYTHONPATH``).
+
+-B
+    Disable writing bytecode (``.pyc``) files.
+
+-X track-resources
+    Produce a ``ResourceWarning`` whenever a file or socket is closed by the
+    garbage collector.
+
+--version
+    Print the PyPy version.
+
+--info
+    Print translation information about this PyPy executable.
+
+--jit ARG
+    Low level JIT parameters.  Mostly internal.  Run ``--jit help``
+    for more information.
+
+ENVIRONMENT
+===========
+
+``PYTHONPATH``
+    Add directories to pypy3's module search path.
+    The format is the same as shell's ``PATH``.
+
+``PYTHONSTARTUP``
+    A script referenced by this variable will be executed before the
+    first prompt is displayed, in interactive mode.
+
+``PYTHONDONTWRITEBYTECODE``
+    If set to a non-empty value, equivalent to the ``-B`` option.
+    Disable writing ``.pyc`` files.
+
+``PYTHONINSPECT``
+    If set to a non-empty value, equivalent to the ``-i`` option.
+    Inspect interactively after running the specified script.
+
+``PYTHONIOENCODING``
+    If this is set, it overrides the encoding used for
+    *stdin*/*stdout*/*stderr*.
+    The syntax is *encodingname*:*errorhandler*
+    The *errorhandler* part is optional and has the same meaning as in
+    `str.encode`.
+
+``PYTHONNOUSERSITE``
+    If set to a non-empty value, equivalent to the ``-s`` option.
+    Don't add the user site directory to `sys.path`.
+
+``PYTHONWARNINGS``
+    If set, equivalent to the ``-W`` option (warning control).
+    The value should be a comma-separated list of ``-W`` parameters.
+
+``PYPYLOG``
+    If set to a non-empty value, enable logging, the format is:
+
+    *fname* or *+fname*
+        logging for profiling: includes all
+        ``debug_start``/``debug_stop`` but not any nested
+        ``debug_print``.
+        *fname* can be ``-`` to log to *stderr*.
+        The *+fname* form can be used if there is a *:* in fname
+
+    ``:``\ *fname*
+        Full logging, including ``debug_print``.
+
+    *prefix*\ ``:``\ *fname*
+        Conditional logging.
+        Multiple prefixes can be specified, comma-separated.
+        Only sections whose name match the prefix will be logged.
+
+    ``PYPYLOG=jit-log-opt,jit-backend:logfile`` will
+    generate a log suitable for *jitviewer*, a tool for debugging
+    performance issues under PyPy.
+
+``PYPY_IRC_TOPIC``
+    If set to a non-empty value, print a random #pypy IRC
+    topic at startup of interactive mode.
+
+
+.. include:: ../gc_info.rst
+   :start-line: 7
+
+SEE ALSO
+========
+
+**python3**\ (1)
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -1,28 +1,7 @@
-What PyPy can do for your objects
-=================================
-
-.. contents::
-
-
-Thanks to the :doc:`Object Space <objspace>` architecture, any feature that is
-based on proxying, extending, changing or otherwise controlling the
-behavior of objects in a running program is easy to implement on top of PyPy.
-
-Here is what we have implemented so far, in historical order:
-
-* *Dump Object Space*: dumps all operations performed on all the objects
-  into a large log file.  For debugging your applications.
-
-* *Transparent Proxies Extension*: adds new proxy objects to
-  the Standard Object Space that enable applications to
-  control operations on application and builtin objects,
-  e.g lists, dictionaries, tracebacks.
-
-
 .. _tproxy:
 
-Transparent Proxies
--------------------
+Transparent Proxies (DEPRECATED)
+--------------------------------
 
 .. warning::
 
@@ -194,7 +173,7 @@
 application-level code.
 
 Transparent proxies are implemented on top of the :ref:`standard object
-space <standard-object-space>`, in :source:`pypy/objspace/std/proxy_helpers.py`,
+space <standard-object-space>`, in :source:`pypy/objspace/std/proxyobject.py`,
 :source:`pypy/objspace/std/proxyobject.py` and :source:`pypy/objspace/std/transparent.py`.
 To use them you will need to pass a `--objspace-std-withtproxy`_ option to ``pypy``
 or ``translate.py``.  This registers implementations named :py:class:`W_TransparentXxx`
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -474,8 +474,8 @@
 :source:`pypy/objspace/std/bytesobject.py` defines ``W_AbstractBytesObject``,
 which contains everything needed to build the ``str`` app-level type;
 and there are subclasses ``W_BytesObject`` (the usual string) and
-``W_StringBufferObject`` (a special implementation tweaked for repeated
-additions, in :source:`pypy/objspace/std/strbufobject.py`).  For mutable data
+``W_Buffer`` (a special implementation tweaked for repeated
+additions, in :source:`pypy/objspace/std/bufferobject.py`).  For mutable data
 types like lists and dictionaries, we have a single class
 ``W_ListObject`` or ``W_DictMultiObject`` which has an indirection to
 the real data and a strategy; the strategy can change as the content of
diff --git a/pypy/doc/project-documentation.rst b/pypy/doc/project-documentation.rst
--- a/pypy/doc/project-documentation.rst
+++ b/pypy/doc/project-documentation.rst
@@ -32,10 +32,13 @@
    coding-guide
    sprint-reports
    extradoc
+   eventhistory
    video-index
    index-report
+   discussions
    dev_method
-   glossary
+   embedding
+   objspace-proxies
 
 
 Source Code Documentation
diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst
--- a/pypy/doc/release-v6.0.0.rst
+++ b/pypy/doc/release-v6.0.0.rst
@@ -8,13 +8,18 @@
 the dual release.
 
 This release is a feature release following our previous 5.10 incremental
-release in late December 2017. Our C-API compatability layer ``cpyext`` is
+release in late December 2017. Our C-API compatibility layer ``cpyext`` is
 now much faster (see the `blog post`_) as well as more complete. We have made
 many other improvements in speed and CPython compatibility. Since the changes
 affect the included python development header files, all c-extension modules must
 be recompiled for this version.
 
-First-time python users are often stumped by silly typos and emissions when
+Until we can work with downstream providers to distribute builds with PyPy, we
+have made packages for some common packages `available as wheels`_. You may
+compile yourself using ``pip install --no-build-isolation <package>``, the
+``no-build-isolation`` is currently needed for pip v10.
+
+First-time python users are often stumped by silly typos and omissions when
 getting started writing code. We have improved our parser to emit more friendly
 `syntax errors`_,  making PyPy not only faster but more friendly.
 
@@ -60,6 +65,7 @@
 .. _`hooks`: gc_info.html#gc-hooks
 .. _`cffi`: http://cffi.readthedocs.io
 .. _`cppyy`: https://cppyy.readthedocs.io
+.. _`available as wheels`: https://github.com/antocuni/pypy-wheels
 
 What is PyPy?
 =============
@@ -110,7 +116,7 @@
 * Fix JIT bugs exposed in the sre module
 * Improve speed of Python parser, improve ParseError messages and SyntaxError
 * Handle JIT hooks more efficiently
-* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s
+* Fix a rare GC bug exposed by intensive use of cpyext ``Buffer`` s
 
 We also refactored many parts of the JIT bridge optimizations, as well as cpyext
 internals, and together with new contributors fixed issues, added new
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,5 +5,20 @@
 .. this is a revision shortly after release-pypy-6.0.0
 .. startrev: e50e11af23f1
 
+.. branch: cppyy-packaging
 
+Upgrade to backend 0.6.0, support exception handling from wrapped functions,
+update enum handling, const correctness for data members and associated tests,
+support anonymous enums, support for function pointer arguments
 
+.. branch: socket_default_timeout_blockingness
+
+Make sure 'blocking-ness' of socket is set along with default timeout
+
+.. branch: crypt_h
+
+Include crypt.h for crypt() on Linux
+
+.. branch: gc-more-logging
+
+Log additional gc-minor and gc-collect-step info in the PYPYLOG
diff --git a/pypy/doc/whatsnew-pypy2-5.10.0.rst b/pypy/doc/whatsnew-pypy2-5.10.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.10.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst
@@ -32,6 +32,7 @@
 
 .. branch: fix-vmprof-stacklet-switch
 .. branch: fix-vmprof-stacklet-switch-2
+
 Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...)
 
 .. branch: win32-vcvars
@@ -39,4 +40,3 @@
 .. branch: rdict-fast-hash
 
 Make it possible to declare that the hash function of an r_dict is fast in RPython.
-
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -25,7 +25,7 @@
 
 This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has
 made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
-was checked in Nov 2016). Note that the compiler suite may be installed in
+was checked in May 2018). Note that the compiler suite may be installed in
 ``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``
 or in
 ``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``.
@@ -37,7 +37,7 @@
 by running the mt.exe command by hand from a DOS window (that is how the author
 discovered the problem).
 
-.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266
+.. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/EN-US/DOWNLOAD/DETAILS.ASPX?ID=44266
 
 Installing "Build Tools for Visual Studio 2017" (for Python 3)
 --------------------------------------------------------------
@@ -150,7 +150,7 @@
 build dependencies for windows. As part of the `rpython` setup stage, environment
 variables will be set to use these dependencies. The repository has a README
 file on how to replicate, and a branch for each supported platform. You may run
- the `get_externals.py` utility to checkout the proper branch for your platform
+the `get_externals.py` utility to checkout the proper branch for your platform
 and PyPy version.
 
 .. _repository:  https://bitbucket.org/pypy/external
diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
deleted file mode 100644
--- a/pypy/doc/you-want-to-help.rst
+++ /dev/null
@@ -1,81 +0,0 @@
-You want to help with PyPy, now what?
-=====================================
-
-PyPy is a very large project that has a reputation of being hard to dive into.
-Some of this fame is warranted, some of it is purely accidental. There are three
-important lessons that everyone willing to contribute should learn:
-
-* PyPy has layers. There are many pieces of architecture that are very well
-  separated from each other. More about this below, but often the manifestation
-  of this is that things are at a different layer than you would expect them
-  to be. For example if you are looking for the JIT implementation, you will
-  not find it in the implementation of the Python programming language.
-
-* Because of the above, we are very serious about Test Driven Development.
-  It's not only what we believe in, but also that PyPy's architecture is
-  working very well with TDD in mind and not so well without it. Often
-  development means progressing in an unrelated corner, one unittest
-  at a time; and then flipping a giant switch, bringing it all together.
-  (It generally works out of the box.  If it doesn't, then we didn't
-  write enough unit tests.)  It's worth repeating - PyPy's
-  approach is great if you do TDD, and not so great otherwise.
-
-* PyPy uses an entirely different set of tools - most of them included
-  in the PyPy repository. There is no Makefile, nor autoconf. More below.
-
-
-Architecture
-------------
-
-PyPy has layers. The 100 miles view:
-
-* :ref:`RPython <rpython:language>` is the language in which we write interpreters. Not the entire
-  PyPy project is written in RPython, only the parts that are compiled in
-  the translation process. The interesting point is that RPython has no parser,
-  it's compiled from the live python objects, which makes it possible to do
-  all kinds of metaprogramming during import time. In short, Python is a meta
-  programming language for RPython.
-
-  The RPython standard library is to be found in the ``rlib`` subdirectory.
-
-* The translation toolchain - this is the part that takes care of translating
-  RPython to flow graphs and then to C. There is more in the :doc:`architecture <architecture>`
-  document written about it.
-
-  It lives in the ``rpython`` directory: ``flowspace``, ``annotator``
-  and ``rtyper``.
-
-* Python Interpreter and modules
-
-  This is in the ``pypy`` directory.  ``pypy/interpreter`` is a standard
-  interpreter for Python written in RPython.  The fact that it is
-  RPython is not apparent at first.  Built-in modules are written in
-  ``pypy/module/*``.  Some modules that CPython implements in C are
-  simply written in pure Python; they are in the top-level ``lib_pypy``
-  directory.  The standard library of Python (with a few changes to
-  accomodate PyPy) is in ``lib-python``.
-
-* :ref:`Just-in-Time Compiler (JIT) <rpython:jit>`: we have a tracing JIT that traces the
-  interpreter written in RPython, rather than the user program that it
-  interprets.  As a result it applies to any interpreter, i.e. any
-  language.  But getting it to work correctly is not trivial: it
-  requires a small number of precise "hints" and possibly some small
-  refactorings of the interpreter.  The JIT itself also has several
-  almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the
-  optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of
-  residual operations, and the backend in ``rpython/jit/backend/<machine-name>``
-  that turns it into machine code.  Writing a new backend is a
-  traditional way to get into the project.
-
-* Garbage Collectors (GC): as you may notice if you are used to CPython's
-  C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
-  :ref:`rpython:garbage-collection` is inserted
-  during translation.  Moreover, this is not reference counting; it is a real
-  GC written as more RPython code.  The best one we have so far is in
-  ``rpython/memory/gc/incminimark.py``.
-
-
-Toolset
--------
-
-xxx
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -499,6 +499,7 @@
     def visit_truncatedint_w(self, typ):
         self.unwrap.append("space.truncatedint_w(%s)" % (self.nextarg(),))
 
+    @staticmethod
     def make_fastfunc(unwrap_spec, func):
         unwrap_info = UnwrapSpec_FastFunc_Unwrap()
         unwrap_info.apply_over(unwrap_spec)
@@ -529,7 +530,6 @@
             exec compile2(source) in unwrap_info.miniglobals, d
             fastfunc = d['fastfunc_%s_%d' % (func.__name__.replace('-', '_'), narg)]
         return narg, fastfunc
-    make_fastfunc = staticmethod(make_fastfunc)
 
 
 def int_unwrapping_space_method(typ):
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -78,6 +78,216 @@
         allow_surrogates=True)
 
 # ____________________________________________________________
+# utf-16
+
+def str_decode_utf_16(s, size, errors, final=True,
+                      errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "native")
+    return result, length
+
+def str_decode_utf_16_be(s, size, errors, final=True,
+                         errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "big")
+    return result, length
+
+def str_decode_utf_16_le(s, size, errors, final=True,
+                         errorhandler=None):
+    result, length, byteorder = str_decode_utf_16_helper(s, size, errors, final,
+                                                         errorhandler, "little")
+    return result, length
+
+def str_decode_utf_16_helper(s, size, errors, final=True,
+                             errorhandler=None,
+                             byteorder="native",
+                             public_encoding_name='utf16'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+    bo = 0
+
+    if BYTEORDER == 'little':
+        ihi = 1
+        ilo = 0
+    else:
+        ihi = 0
+        ilo = 1
+
+    #  Check for BOM marks (U+FEFF) in the input and adjust current
+    #  byte order setting accordingly. In native mode, the leading BOM
+    #  mark is skipped, in all other modes, it is copied to the output
+    #  stream as-is (giving a ZWNBSP character).
+    pos = 0
+    if byteorder == 'native':
+        if size >= 2:
+            bom = (ord(s[ihi]) << 8) | ord(s[ilo])
+            if BYTEORDER == 'little':
+                if bom == 0xFEFF:
+                    pos += 2
+                    bo = -1
+                elif bom == 0xFFFE:
+                    pos += 2
+                    bo = 1
+            else:
+                if bom == 0xFEFF:
+                    pos += 2
+                    bo = 1
+                elif bom == 0xFFFE:
+                    pos += 2
+                    bo = -1
+    elif byteorder == 'little':
+        bo = -1
+    else:
+        bo = 1
+    if size == 0:
+        return u'', 0, bo
+    if bo == -1:
+        # force little endian
+        ihi = 1
+        ilo = 0
+
+    elif bo == 1:
+        # force big endian
+        ihi = 0
+        ilo = 1
+
+    result = UnicodeBuilder(size // 2)
+
+    #XXX I think the errors are not correctly handled here
+    while pos < size:
+        # remaining bytes at the end? (size should be even)
+        if len(s) - pos < 2:
+            if not final:
+                break
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "truncated data",
+                                  s, pos, len(s))
+            result.append(r)
+            if len(s) - pos < 2:
+                break
+        ch = (ord(s[pos + ihi]) << 8) | ord(s[pos + ilo])
+        pos += 2
+        if ch < 0xD800 or ch > 0xDFFF:
+            result.append(unichr(ch))
+            continue
+        # UTF-16 code pair:
+        if len(s) - pos < 2:
+            pos -= 2
+            if not final:
+                break
+            errmsg = "unexpected end of data"
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  errmsg, s, pos, len(s))
+            result.append(r)
+            if len(s) - pos < 2:
+                break
+        elif 0xD800 <= ch <= 0xDBFF:
+            ch2 = (ord(s[pos+ihi]) << 8) | ord(s[pos+ilo])
+            pos += 2
+            if 0xDC00 <= ch2 <= 0xDFFF:
+                if MAXUNICODE < 65536:
+                    result.append(unichr(ch))
+                    result.append(unichr(ch2))
+                else:
+                    result.append(UNICHR((((ch & 0x3FF)<<10) |
+                                           (ch2 & 0x3FF)) + 0x10000))
+                continue
+            else:
+                r, pos = errorhandler(errors, public_encoding_name,
+                                      "illegal UTF-16 surrogate",
+                                      s, pos - 4, pos - 2)
+                result.append(r)
+        else:
+            r, pos = errorhandler(errors, public_encoding_name,
+                                  "illegal encoding",
+                                  s, pos - 2, pos)
+            result.append(r)
+    return result.build(), pos, bo
+
+def _STORECHAR(result, CH, byteorder):
+    hi = chr(((CH) >> 8) & 0xff)
+    lo = chr((CH) & 0xff)
+    if byteorder == 'little':
+        result.append(lo)
+        result.append(hi)
+    else:
+        result.append(hi)

From pypy.commits at gmail.com  Wed May 23 11:20:48 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 08:20:48 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: fix for test_app_main.py
Message-ID: <5b0586d0.1c69fb81.dfdad.45f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r94654:c2f8c612bd01
Date: 2018-05-23 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/c2f8c612bd01/

Log:	fix for test_app_main.py

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -83,7 +83,10 @@
     sys.excepthook(), catching SystemExit, printing a newline after
     sys.stdout if needed, etc.
     """
-    from __pypy__ import revdb_stop
+    if '__pypy__' in sys.builtin_module_names:
+        from __pypy__ import revdb_stop
+    else:
+        revdb_stop = None
     try:
         # run it
         try:

From pypy.commits at gmail.com  Wed May 23 13:26:55 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 10:26:55 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Explain and tweak
Message-ID: <5b05a45f.1c69fb81.73844.ba0b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r94655:40fff8a8e2e0
Date: 2018-05-23 19:26 +0200
http://bitbucket.org/pypy/pypy/changeset/40fff8a8e2e0/

Log:	Explain and tweak

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -83,17 +83,24 @@
     sys.excepthook(), catching SystemExit, printing a newline after
     sys.stdout if needed, etc.
     """
+    # don't use try:except: here, otherwise the exception remains
+    # visible in user code.  Make sure revdb_stop is a callable, so
+    # that we can call it immediately after finally: below.  Doing
+    # so minimizes the number of "blind" lines that we need to go
+    # back from, with "bstep", after we do "continue" in revdb.
     if '__pypy__' in sys.builtin_module_names:
         from __pypy__ import revdb_stop
     else:
         revdb_stop = None
+    if revdb_stop is None:
+        revdb_stop = lambda: None
+
     try:
         # run it
         try:
             f(*fargs, **fkwds)
         finally:
-            if revdb_stop:
-                revdb_stop()
+            revdb_stop()
             sys.settrace(None)
             sys.setprofile(None)
 

From pypy.commits at gmail.com  Wed May 23 13:31:36 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 10:31:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-reverse-debugger: merge branch
Message-ID: <5b05a578.1c69fb81.e3d05.755f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-reverse-debugger
Changeset: r94656:5b31e0272c45
Date: 2018-05-23 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/5b31e0272c45/

Log:	merge branch


From pypy.commits at gmail.com  Wed May 23 13:31:39 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 10:31:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-reverse-debugger: merge the two branches
Message-ID: <5b05a57b.1c69fb81.6088d.5463@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-reverse-debugger
Changeset: r94657:3d3ad98443d6
Date: 2018-05-23 19:30 +0200
http://bitbucket.org/pypy/pypy/changeset/3d3ad98443d6/

Log:	merge the two branches

diff too long, truncating to 2000 out of 7458 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,7 +33,12 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
+e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1
+a24d6c7000c8099c73d3660857f7e3cee5ac045c RevDB-pypy2.7-v5.6.2
 fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
 b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -223,5 +223,6 @@
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
 jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
+def_op('LOAD_REVDB_VAR', 205)         # reverse debugger (syntax example: $5)
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -61,6 +61,11 @@
     "termios", "_minimal_curses",
 ])
 
+reverse_debugger_disable_modules = set([
+    "_continuation", "_vmprof", "_multiprocessing",
+    "micronumpy",
+    ])
+
 # XXX this should move somewhere else, maybe to platform ("is this posixish"
 #     check or something)
 if sys.platform == "win32":
@@ -297,6 +302,9 @@
     modules = working_modules.copy()
     if config.translation.sandbox:
         modules = default_modules
+    if config.translation.reverse_debugger:
+        for mod in reverse_debugger_disable_modules:
+            setattr(config.objspace.usemodules, mod, False)
     # ignore names from 'essential_modules', notably 'exceptions', which
     # may not be present in config.objspace.usemodules at all
     modules = [name for name in modules if name not in essential_modules]
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -88,11 +88,17 @@
     run_protected() handles details like forwarding exceptions to
     sys.excepthook(), catching SystemExit, etc.
     """
+    if '__pypy__' in sys.builtin_module_names:
+        from __pypy__ import revdb_stop
+    else:
+        revdb_stop = None
     try:
         # run it
         try:
             f(*fargs, **fkwds)
         finally:
+            if revdb_stop:
+                revdb_stop()
             sys.settrace(None)
             sys.setprofile(None)
     except SystemExit as e:
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -692,6 +692,7 @@
 
     # TODO
     ops.BUILD_LIST_FROM_ARG: 1,
+    ops.LOAD_REVDB_VAR: 1,
 
     ops.LOAD_CLASSDEREF: 1,
 }
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1733,6 +1733,8 @@
             return Num.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Str):
             return Str.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_RevDBMetaVar):
+            return RevDBMetaVar.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_FormattedValue):
             return FormattedValue.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_JoinedStr):
@@ -2643,6 +2645,41 @@
 State.ast_type('Str', 'expr', ['s'])
 
 
+class RevDBMetaVar(expr):
+
+    def __init__(self, metavar, lineno, col_offset):
+        self.metavar = metavar
+        expr.__init__(self, lineno, col_offset)
+
+    def walkabout(self, visitor):
+        visitor.visit_RevDBMetaVar(self)
+
+    def mutate_over(self, visitor):
+        return visitor.visit_RevDBMetaVar(self)
+
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_RevDBMetaVar)
+        w_metavar = space.newint(self.metavar)  # int
+        space.setattr(w_node, space.newtext('metavar'), w_metavar)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
+        return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        w_metavar = get_field(space, w_node, 'metavar', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _metavar = space.int_w(w_metavar)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
+        return RevDBMetaVar(_metavar, _lineno, _col_offset)
+
+State.ast_type('RevDBMetaVar', 'expr', ['metavar'])
+
+
 class FormattedValue(expr):
 
     def __init__(self, value, conversion, format_spec, lineno, col_offset):
@@ -4130,6 +4167,8 @@
         return self.default_visitor(node)
     def visit_Str(self, node):
         return self.default_visitor(node)
+    def visit_RevDBMetaVar(self, node):
+        return self.default_visitor(node)
     def visit_FormattedValue(self, node):
         return self.default_visitor(node)
     def visit_JoinedStr(self, node):
@@ -4363,6 +4402,9 @@
     def visit_Str(self, node):
         pass
 
+    def visit_RevDBMetaVar(self, node):
+        pass
+
     def visit_FormattedValue(self, node):
         node.value.walkabout(self)
         if node.format_spec:
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1264,6 +1264,11 @@
                 else:
                     # a dictionary display
                     return self.handle_dictdisplay(maker, atom_node)
+        elif first_child_type == tokens.REVDBMETAVAR:
+            string = atom_node.get_child(0).get_value()
+            return ast.RevDBMetaVar(int(string[1:]),
+                                    atom_node.get_lineno(),
+                                    atom_node.get_column())
         else:
             raise AssertionError("unknown atom")
 
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1519,6 +1519,20 @@
             fmt.format_spec.walkabout(self)
         self.emit_op_arg(ops.FORMAT_VALUE, arg)
 
+    def _revdb_metavar(self, node):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import dbstate
+        if not dbstate.standard_code:
+            self.emit_op_arg(ops.LOAD_REVDB_VAR, node.metavar)
+            return True
+        return False
+
+    def visit_RevDBMetaVar(self, node):
+        if self.space.reverse_debugging and self._revdb_metavar(node):
+            return
+        self.error("Unknown character ('$NUM' is only valid in the "
+                   "reverse-debugger)", node)
+
 
 class TopLevelCodeGenerator(PythonCodeGenerator):
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1240,6 +1240,20 @@
         src = """# -*- coding: utf-8 -*-\nz=ord(fr'\xc3\x98')\n"""
         yield self.st, src, 'z', 0xd8
 
+    def test_revdb_metavar(self):
+        from pypy.interpreter.reverse_debugging import dbstate, setup_revdb
+        self.space.config.translation.reverse_debugger = True
+        self.space.reverse_debugging = True
+        try:
+            setup_revdb(self.space)
+            dbstate.standard_code = False
+            dbstate.metavars = [self.space.wrap(6)]
+            self.simple_test("x = 7*$0", "x", 42)
+            dbstate.standard_code = True
+            self.error_test("x = 7*$0", SyntaxError)
+        finally:
+            self.space.reverse_debugging = False
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -70,6 +70,7 @@
          | Call(expr func, expr* args, keyword* keywords)
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
+         | RevDBMetaVar(int metavar)
          | FormattedValue(expr value, int? conversion, expr? format_spec)
          | JoinedStr(expr* values)
          | Bytes(bytes s)
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -425,6 +425,9 @@
         node.slice.walkabout(self)
         self._validate_expr(node.value)
 
+    def visit_RevDBMetaVar(self, node):
+        pass
+
     # Subscripts
     def visit_Slice(self, node):
         if node.lower:
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -402,6 +402,8 @@
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
 
+    reverse_debugging = False
+
     @not_rpython
     def __init__(self, config=None):
         "Basic initialization of objects."
@@ -413,6 +415,7 @@
             from pypy.config.pypyoption import get_pypy_config
             config = get_pypy_config(translating=False)
         self.config = config
+        self.reverse_debugging = config.translation.reverse_debugger
 
         self.builtin_modules = {}
         self.reloading_modules = {}
@@ -430,6 +433,9 @@
 
     def startup(self):
         # To be called before using the space
+        if self.reverse_debugging:
+            self._revdb_startup()
+
         self.threadlocals.enter_thread(self)
 
         # Initialize already imported builtin modules
@@ -820,7 +826,8 @@
         w_u1 = self.interned_strings.get(u)
         if w_u1 is None:
             w_u1 = w_u
-            self.interned_strings.set(u, w_u1)
+            if self._side_effects_ok():
+                self.interned_strings.set(u, w_u1)
         return w_u1
 
     def new_interned_str(self, s):
@@ -832,9 +839,39 @@
         w_s1 = self.interned_strings.get(u)
         if w_s1 is None:
             w_s1 = self.newunicode(u)
-            self.interned_strings.set(u, w_s1)
+            if self._side_effects_ok():
+                self.interned_strings.set(u, w_s1)
         return w_s1
 
+    def _revdb_startup(self):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import setup_revdb
+        setup_revdb(self)
+
+    def _revdb_standard_code(self):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import dbstate
+        return dbstate.standard_code
+
+    def _side_effects_ok(self):
+        # For the reverse debugger: we run compiled watchpoint
+        # expressions in a fast way that will crash if they have
+        # side-effects.  The obvious Python code with side-effects is
+        # documented "don't do that"; but some non-obvious side
+        # effects are also common, like interning strings (from
+        # unmarshalling the code object containing the watchpoint
+        # expression) to the two attribute caches in mapdict.py and
+        # typeobject.py.  For now, we have to identify such places
+        # that are not acceptable for "reasonable" read-only
+        # watchpoint expressions, and write:
+        #
+        #     if not space._side_effects_ok():
+        #         don't cache.
+        #
+        if self.reverse_debugging:
+            return self._revdb_standard_code()
+        return True
+
     def get_interned_str(self, s):
         """Assumes an identifier (utf-8 encoded str).  Returns None if
         the identifier is not interned, or not a valid utf-8 string at all.
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -71,6 +71,8 @@
         return frame
 
     def enter(self, frame):
+        if self.space.reverse_debugging:
+            self._revdb_enter(frame)
         frame.f_backref = self.topframeref
         self.topframeref = jit.virtual_ref(frame)
 
@@ -91,6 +93,8 @@
                 # be accessed also later
                 frame_vref()
             jit.virtual_ref_finish(frame_vref, frame)
+            if self.space.reverse_debugging:
+                self._revdb_leave(got_exception)
 
     # ________________________________________________________________
 
@@ -160,6 +164,8 @@
         Like bytecode_trace() but doesn't invoke any other events besides the
         trace function.
         """
+        if self.space.reverse_debugging:
+            self._revdb_potential_stop_point(frame)
         if (frame.get_w_f_trace() is None or self.is_tracing or
             self.gettrace() is None):
             return
@@ -373,6 +379,21 @@
         if self.space.check_signal_action is not None:
             self.space.check_signal_action.perform(self, None)
 
+    def _revdb_enter(self, frame):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import enter_call
+        enter_call(self.topframeref(), frame)
+
+    def _revdb_leave(self, got_exception):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import leave_call
+        leave_call(self.topframeref(), got_exception)
+
+    def _revdb_potential_stop_point(self, frame):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import potential_stop_point
+        potential_stop_point(frame)
+
     def _freeze_(self):
         raise Exception("ExecutionContext instances should not be seen during"
                         " translation.  Now is a good time to inspect the"
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -774,9 +774,11 @@
     def fget_f_builtins(self, space):
         return self.get_builtin().getdict(space)
 
+    def get_f_back(self):
+        return ExecutionContext.getnextframe_nohidden(self)
+
     def fget_f_back(self, space):
-        f_back = ExecutionContext.getnextframe_nohidden(self)
-        return f_back
+        return self.get_f_back()
 
     def fget_f_lasti(self, space):
         return self.space.newint(self.last_instr)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -438,6 +438,8 @@
                 self.FORMAT_VALUE(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_STRING.index:
                 self.BUILD_STRING(oparg, next_instr)
+            elif opcode == opcodedesc.LOAD_REVDB_VAR.index:
+                self.LOAD_REVDB_VAR(oparg, next_instr)
             else:
                 self.MISSING_OPCODE(oparg, next_instr)
 
@@ -1114,9 +1116,16 @@
         # final result and returns.  In that case, we can just continue
         # with the next bytecode.
 
+    def _revdb_jump_backward(self, jumpto):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import jump_backward
+        jump_backward(self, jumpto)
+
     def jump_absolute(self, jumpto, ec):
         # this function is overridden by pypy.module.pypyjit.interp_jit
         check_nonneg(jumpto)
+        if self.space.reverse_debugging:
+            self._revdb_jump_backward(jumpto)
         return jumpto
 
     def JUMP_FORWARD(self, jumpby, next_instr):
@@ -1656,6 +1665,19 @@
         w_res = space.newunicode(u''.join(lst))
         self.pushvalue(w_res)
 
+    def _revdb_load_var(self, oparg):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import load_metavar
+        w_var = load_metavar(oparg)
+        self.pushvalue(w_var)
+
+    def LOAD_REVDB_VAR(self, oparg, next_instr):
+        if self.space.reverse_debugging:
+            self._revdb_load_var(oparg)
+        else:
+            self.MISSING_OPCODE(oparg, next_instr)
+
+
 ### ____________________________________________________________ ###
 
 class ExitFrame(Exception):
diff --git a/pypy/interpreter/pyparser/data/Grammar2.7 b/pypy/interpreter/pyparser/data/Grammar2.7
--- a/pypy/interpreter/pyparser/data/Grammar2.7
+++ b/pypy/interpreter/pyparser/data/Grammar2.7
@@ -104,7 +104,7 @@
        '[' [listmaker] ']' |
        '{' [dictorsetmaker] '}' |
        '`' testlist1 '`' |
-       NAME | NUMBER | STRING+)
+       NAME | NUMBER | STRING+ | '$NUM')
 listmaker: test ( list_for | (',' test)* [','] )
 testlist_comp: test ( comp_for | (',' test)* [','] )
 lambdef: 'lambda' [varargslist] ':' test
diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -103,7 +103,7 @@
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
-       NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
+       NAME | NUMBER | STRING+ | '$NUM' | '...' | 'None' | 'True' | 'False')
 testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
diff --git a/pypy/interpreter/pyparser/data/Grammar3.3 b/pypy/interpreter/pyparser/data/Grammar3.3
--- a/pypy/interpreter/pyparser/data/Grammar3.3
+++ b/pypy/interpreter/pyparser/data/Grammar3.3
@@ -103,7 +103,7 @@
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
-       NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
+       NAME | NUMBER | STRING+ | '$NUM' | '...' | 'None' | 'True' | 'False')
 testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
diff --git a/pypy/interpreter/pyparser/data/Grammar3.5 b/pypy/interpreter/pyparser/data/Grammar3.5
--- a/pypy/interpreter/pyparser/data/Grammar3.5
+++ b/pypy/interpreter/pyparser/data/Grammar3.5
@@ -108,7 +108,7 @@
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
-       NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
+       NAME | NUMBER | STRING+ | '$NUM' | '...' | 'None' | 'True' | 'False')
 testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py
--- a/pypy/interpreter/pyparser/dfa_generated.py
+++ b/pypy/interpreter/pyparser/dfa_generated.py
@@ -6,38 +6,39 @@
 from pypy.interpreter.pyparser import automata
 accepts = [True, True, True, True, True, True, True, True,
            True, True, True, False, True, True, True, True,
-           True, False, False, False, True, False, False,
-           False, True, False, True, False, True, False,
-           False, True, False, False, True, False, False,
-           True, True, True, False, False, True, False,
-           False, False, True]
+           True, False, False, False, False, True, False,
+           False, False, True, False, True, False, True,
+           False, True, False, True, False, False, True,
+           False, False, True, True, True, False, False,
+           True, False, False, False, True]
 states = [
     # 0
     {'\t': 0, '\n': 15, '\x0c': 0,
-     '\r': 16, ' ': 0, '!': 11, '"': 18,
-     '#': 20, '%': 14, '&': 14, "'": 17,
-     '(': 15, ')': 15, '*': 8, '+': 14,
-     ',': 15, '-': 12, '.': 7, '/': 13,
-     '0': 5, '1': 6, '2': 6, '3': 6,
-     '4': 6, '5': 6, '6': 6, '7': 6,
-     '8': 6, '9': 6, ':': 15, ';': 15,
-     '<': 10, '=': 14, '>': 9, '@': 14,
-     'A': 1, 'B': 2, 'C': 1, 'D': 1,
-     'E': 1, 'F': 2, 'G': 1, 'H': 1,
-     'I': 1, 'J': 1, 'K': 1, 'L': 1,
-     'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
-     'U': 4, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '[': 15, '\\': 19,
-     ']': 15, '^': 14, '_': 1, '`': 15,
-     'a': 1, 'b': 2, 'c': 1, 'd': 1,
-     'e': 1, 'f': 2, 'g': 1, 'h': 1,
-     'i': 1, 'j': 1, 'k': 1, 'l': 1,
-     'm': 1, 'n': 1, 'o': 1, 'p': 1,
-     'q': 1, 'r': 3, 's': 1, 't': 1,
-     'u': 4, 'v': 1, 'w': 1, 'x': 1,
-     'y': 1, 'z': 1, '{': 15, '|': 14,
-     '}': 15, '~': 15, '\x80': 1},
+     '\r': 16, ' ': 0, '!': 11, '"': 19,
+     '#': 21, '$': 17, '%': 14, '&': 14,
+     "'": 18, '(': 15, ')': 15, '*': 8,
+     '+': 14, ',': 15, '-': 12, '.': 7,
+     '/': 13, '0': 5, '1': 6, '2': 6,
+     '3': 6, '4': 6, '5': 6, '6': 6,
+     '7': 6, '8': 6, '9': 6, ':': 15,
+     ';': 15, '<': 10, '=': 14, '>': 9,
+     '@': 14, 'A': 1, 'B': 2, 'C': 1,
+     'D': 1, 'E': 1, 'F': 2, 'G': 1,
+     'H': 1, 'I': 1, 'J': 1, 'K': 1,
+     'L': 1, 'M': 1, 'N': 1, 'O': 1,
+     'P': 1, 'Q': 1, 'R': 3, 'S': 1,
+     'T': 1, 'U': 4, 'V': 1, 'W': 1,
+     'X': 1, 'Y': 1, 'Z': 1, '[': 15,
+     '\\': 20, ']': 15, '^': 14, '_': 1,
+     '`': 15, 'a': 1, 'b': 2, 'c': 1,
+     'd': 1, 'e': 1, 'f': 2, 'g': 1,
+     'h': 1, 'i': 1, 'j': 1, 'k': 1,
+     'l': 1, 'm': 1, 'n': 1, 'o': 1,
+     'p': 1, 'q': 1, 'r': 3, 's': 1,
+     't': 1, 'u': 4, 'v': 1, 'w': 1,
+     'x': 1, 'y': 1, 'z': 1, '{': 15,
+     '|': 14, '}': 15, '~': 15,
+     '\x80': 1},
     # 1
     {'0': 1, '1': 1, '2': 1, '3': 1,
      '4': 1, '5': 1, '6': 1, '7': 1,
@@ -56,7 +57,7 @@
      't': 1, 'u': 1, 'v': 1, 'w': 1,
      'x': 1, 'y': 1, 'z': 1, '\x80': 1},
     # 2
-    {'"': 18, "'": 17, '0': 1, '1': 1,
+    {'"': 19, "'": 18, '0': 1, '1': 1,
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 1, 'C': 1, 'D': 1,
@@ -74,7 +75,7 @@
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1, '\x80': 1},
     # 3
-    {'"': 18, "'": 17, '0': 1, '1': 1,
+    {'"': 19, "'": 18, '0': 1, '1': 1,
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 4, 'C': 1, 'D': 1,
@@ -92,7 +93,7 @@
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1, '\x80': 1},
     # 4
-    {'"': 18, "'": 17, '0': 1, '1': 1,
+    {'"': 19, "'": 18, '0': 1, '1': 1,
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 1, 'C': 1, 'D': 1,
@@ -110,21 +111,21 @@
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1, '\x80': 1},
     # 5
-    {'.': 26, '0': 24, '1': 25, '2': 25,
-     '3': 25, '4': 25, '5': 25, '6': 25,
-     '7': 25, '8': 25, '9': 25, 'B': 23,
-     'E': 27, 'J': 15, 'O': 22, 'X': 21,
-     'b': 23, 'e': 27, 'j': 15, 'o': 22,
-     'x': 21},
+    {'.': 27, '0': 25, '1': 26, '2': 26,
+     '3': 26, '4': 26, '5': 26, '6': 26,
+     '7': 26, '8': 26, '9': 26, 'B': 24,
+     'E': 28, 'J': 15, 'O': 23, 'X': 22,
+     'b': 24, 'e': 28, 'j': 15, 'o': 23,
+     'x': 22},
     # 6
-    {'.': 26, '0': 6, '1': 6, '2': 6,
+    {'.': 27, '0': 6, '1': 6, '2': 6,
      '3': 6, '4': 6, '5': 6, '6': 6,
-     '7': 6, '8': 6, '9': 6, 'E': 27,
-     'J': 15, 'e': 27, 'j': 15},
+     '7': 6, '8': 6, '9': 6, 'E': 28,
+     'J': 15, 'e': 28, 'j': 15},
     # 7
-    {'.': 29, '0': 28, '1': 28, '2': 28,
-     '3': 28, '4': 28, '5': 28, '6': 28,
-     '7': 28, '8': 28, '9': 28},
+    {'.': 30, '0': 29, '1': 29, '2': 29,
+     '3': 29, '4': 29, '5': 29, '6': 29,
+     '7': 29, '8': 29, '9': 29},
     # 8
     {'*': 14, '=': 15},
     # 9
@@ -144,107 +145,115 @@
     # 16
     {'\n': 15},
     # 17
-    {automata.DEFAULT: 33, '\n': 30,
-     '\r': 30, "'": 31, '\\': 32},
+    {'0': 31, '1': 31, '2': 31, '3': 31,
+     '4': 31, '5': 31, '6': 31, '7': 31,
+     '8': 31, '9': 31},
     # 18
-    {automata.DEFAULT: 36, '\n': 30,
-     '\r': 30, '"': 34, '\\': 35},
+    {automata.DEFAULT: 35, '\n': 32,
+     '\r': 32, "'": 33, '\\': 34},
     # 19
+    {automata.DEFAULT: 38, '\n': 32,
+     '\r': 32, '"': 36, '\\': 37},
+    # 20
     {'\n': 15, '\r': 16},
-    # 20
-    {automata.DEFAULT: 20, '\n': 30, '\r': 30},
     # 21
-    {'0': 37, '1': 37, '2': 37, '3': 37,
-     '4': 37, '5': 37, '6': 37, '7': 37,
-     '8': 37, '9': 37, 'A': 37, 'B': 37,
-     'C': 37, 'D': 37, 'E': 37, 'F': 37,
-     'a': 37, 'b': 37, 'c': 37, 'd': 37,
-     'e': 37, 'f': 37},
+    {automata.DEFAULT: 21, '\n': 32, '\r': 32},
     # 22
-    {'0': 38, '1': 38, '2': 38, '3': 38,
-     '4': 38, '5': 38, '6': 38, '7': 38},
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39, 'A': 39, 'B': 39,
+     'C': 39, 'D': 39, 'E': 39, 'F': 39,
+     'a': 39, 'b': 39, 'c': 39, 'd': 39,
+     'e': 39, 'f': 39},
     # 23
-    {'0': 39, '1': 39},
+    {'0': 40, '1': 40, '2': 40, '3': 40,
+     '4': 40, '5': 40, '6': 40, '7': 40},
     # 24
-    {'.': 26, '0': 24, '1': 25, '2': 25,
-     '3': 25, '4': 25, '5': 25, '6': 25,
-     '7': 25, '8': 25, '9': 25, 'E': 27,
-     'J': 15, 'e': 27, 'j': 15},
+    {'0': 41, '1': 41},
     # 25
-    {'.': 26, '0': 25, '1': 25, '2': 25,
-     '3': 25, '4': 25, '5': 25, '6': 25,
-     '7': 25, '8': 25, '9': 25, 'E': 27,
-     'J': 15, 'e': 27, 'j': 15},
+    {'.': 27, '0': 25, '1': 26, '2': 26,
+     '3': 26, '4': 26, '5': 26, '6': 26,
+     '7': 26, '8': 26, '9': 26, 'E': 28,
+     'J': 15, 'e': 28, 'j': 15},
     # 26
-    {'0': 26, '1': 26, '2': 26, '3': 26,
-     '4': 26, '5': 26, '6': 26, '7': 26,
-     '8': 26, '9': 26, 'E': 40, 'J': 15,
-     'e': 40, 'j': 15},
+    {'.': 27, '0': 26, '1': 26, '2': 26,
+     '3': 26, '4': 26, '5': 26, '6': 26,
+     '7': 26, '8': 26, '9': 26, 'E': 28,
+     'J': 15, 'e': 28, 'j': 15},
     # 27
-    {'+': 41, '-': 41, '0': 42, '1': 42,
-     '2': 42, '3': 42, '4': 42, '5': 42,
-     '6': 42, '7': 42, '8': 42, '9': 42},
+    {'0': 27, '1': 27, '2': 27, '3': 27,
+     '4': 27, '5': 27, '6': 27, '7': 27,
+     '8': 27, '9': 27, 'E': 42, 'J': 15,
+     'e': 42, 'j': 15},
     # 28
-    {'0': 28, '1': 28, '2': 28, '3': 28,
-     '4': 28, '5': 28, '6': 28, '7': 28,
-     '8': 28, '9': 28, 'E': 40, 'J': 15,
-     'e': 40, 'j': 15},
+    {'+': 43, '-': 43, '0': 44, '1': 44,
+     '2': 44, '3': 44, '4': 44, '5': 44,
+     '6': 44, '7': 44, '8': 44, '9': 44},
     # 29
+    {'0': 29, '1': 29, '2': 29, '3': 29,
+     '4': 29, '5': 29, '6': 29, '7': 29,
+     '8': 29, '9': 29, 'E': 42, 'J': 15,
+     'e': 42, 'j': 15},
+    # 30
     {'.': 15},
-    # 30
+    # 31
+    {'0': 31, '1': 31, '2': 31, '3': 31,
+     '4': 31, '5': 31, '6': 31, '7': 31,
+     '8': 31, '9': 31},
+    # 32
     {},
-    # 31
+    # 33
     {"'": 15},
-    # 32
-    {automata.DEFAULT: 43, '\n': 15, '\r': 16},
-    # 33
-    {automata.DEFAULT: 33, '\n': 30,
-     '\r': 30, "'": 15, '\\': 32},
     # 34
+    {automata.DEFAULT: 45, '\n': 15, '\r': 16},
+    # 35
+    {automata.DEFAULT: 35, '\n': 32,
+     '\r': 32, "'": 15, '\\': 34},
+    # 36
     {'"': 15},
-    # 35
-    {automata.DEFAULT: 44, '\n': 15, '\r': 16},
-    # 36
-    {automata.DEFAULT: 36, '\n': 30,
-     '\r': 30, '"': 15, '\\': 35},
     # 37
-    {'0': 37, '1': 37, '2': 37, '3': 37,
-     '4': 37, '5': 37, '6': 37, '7': 37,
-     '8': 37, '9': 37, 'A': 37, 'B': 37,
-     'C': 37, 'D': 37, 'E': 37, 'F': 37,
-     'a': 37, 'b': 37, 'c': 37, 'd': 37,
-     'e': 37, 'f': 37},
+    {automata.DEFAULT: 46, '\n': 15, '\r': 16},
     # 38
-    {'0': 38, '1': 38, '2': 38, '3': 38,
-     '4': 38, '5': 38, '6': 38, '7': 38},
+    {automata.DEFAULT: 38, '\n': 32,
+     '\r': 32, '"': 15, '\\': 37},
     # 39
-    {'0': 39, '1': 39},
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39, 'A': 39, 'B': 39,
+     'C': 39, 'D': 39, 'E': 39, 'F': 39,
+     'a': 39, 'b': 39, 'c': 39, 'd': 39,
+     'e': 39, 'f': 39},
     # 40
-    {'+': 45, '-': 45, '0': 46, '1': 46,
-     '2': 46, '3': 46, '4': 46, '5': 46,
-     '6': 46, '7': 46, '8': 46, '9': 46},
+    {'0': 40, '1': 40, '2': 40, '3': 40,
+     '4': 40, '5': 40, '6': 40, '7': 40},
     # 41
-    {'0': 42, '1': 42, '2': 42, '3': 42,
-     '4': 42, '5': 42, '6': 42, '7': 42,
-     '8': 42, '9': 42},
+    {'0': 41, '1': 41},
     # 42
-    {'0': 42, '1': 42, '2': 42, '3': 42,
-     '4': 42, '5': 42, '6': 42, '7': 42,
-     '8': 42, '9': 42, 'J': 15, 'j': 15},
+    {'+': 47, '-': 47, '0': 48, '1': 48,
+     '2': 48, '3': 48, '4': 48, '5': 48,
+     '6': 48, '7': 48, '8': 48, '9': 48},
     # 43
-    {automata.DEFAULT: 43, '\n': 30,
-     '\r': 30, "'": 15, '\\': 32},
+    {'0': 44, '1': 44, '2': 44, '3': 44,
+     '4': 44, '5': 44, '6': 44, '7': 44,
+     '8': 44, '9': 44},
     # 44
-    {automata.DEFAULT: 44, '\n': 30,
-     '\r': 30, '"': 15, '\\': 35},
+    {'0': 44, '1': 44, '2': 44, '3': 44,
+     '4': 44, '5': 44, '6': 44, '7': 44,
+     '8': 44, '9': 44, 'J': 15, 'j': 15},
     # 45
-    {'0': 46, '1': 46, '2': 46, '3': 46,
-     '4': 46, '5': 46, '6': 46, '7': 46,
-     '8': 46, '9': 46},
+    {automata.DEFAULT: 45, '\n': 32,
+     '\r': 32, "'": 15, '\\': 34},
     # 46
-    {'0': 46, '1': 46, '2': 46, '3': 46,
-     '4': 46, '5': 46, '6': 46, '7': 46,
-     '8': 46, '9': 46, 'J': 15, 'j': 15},
+    {automata.DEFAULT: 46, '\n': 32,
+     '\r': 32, '"': 15, '\\': 37},
+    # 47
+    {'0': 48, '1': 48, '2': 48, '3': 48,
+     '4': 48, '5': 48, '6': 48, '7': 48,
+     '8': 48, '9': 48},
+    # 48
+    {'0': 48, '1': 48, '2': 48, '3': 48,
+     '4': 48, '5': 48, '6': 48, '7': 48,
+     '8': 48, '9': 48, 'J': 15, 'j': 15},
     ]
 pseudoDFA = automata.DFA(states, accepts)
 
diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/gendfa.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -146,7 +146,10 @@
                     makeEOL(),
                     chainStr(states, "..."),
                     groupStr(states, "@:;.,`"))
-    funny = group(states, operator, bracket, special)
+    revdb_metavar = chain(states,
+                          groupStr(states, "$"),
+                          atleastonce(states, makeDigits()))
+    funny = group(states, operator, bracket, special, revdb_metavar)
     # ____________________________________________________________
     def makeStrPrefix ():
         return group(states,
diff --git a/pypy/interpreter/pyparser/pytoken.py b/pypy/interpreter/pyparser/pytoken.py
--- a/pypy/interpreter/pyparser/pytoken.py
+++ b/pypy/interpreter/pyparser/pytoken.py
@@ -72,5 +72,6 @@
 # extra PyPy-specific tokens
 _add_tok("COMMENT")
 _add_tok("NL")
+_add_tok("REVDBMETAVAR", "$NUM")
 
 del _add_tok
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -329,6 +329,10 @@
                     last_comment = ''
                 elif initial == '\\':                      # continued stmt
                     continued = 1
+                elif initial == '$':
+                    token_list.append((tokens.REVDBMETAVAR, token,
+                                       lnum, start, line))
+                    last_comment = ''
                 else:
                     if initial in '([{':
                         parenstack.append((initial, lnum, start, line))
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -147,6 +147,15 @@
             tree = self.parse(fmt % linefeed)
             assert expected_tree == tree
 
+    def test_revdb_dollar_num(self):
+        self.parse('$0')
+        self.parse('$5')
+        self.parse('$42')
+        self.parse('2+$42.attrname')
+        py.test.raises(SyntaxError, self.parse, '$')
+        py.test.raises(SyntaxError, self.parse, '$a')
+        py.test.raises(SyntaxError, self.parse, '$.5')
+
     def test_py3k_reject_old_binary_literal(self):
         py.test.raises(SyntaxError, self.parse, '0777')
 
diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/reverse_debugging.py
@@ -0,0 +1,852 @@
+import sys
+from rpython.rlib import revdb
+from rpython.rlib.debug import make_sure_not_resized
+from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter import gateway, typedef, pycode, pytraceback, pyframe
+from pypy.module.marshal import interp_marshal
+from pypy.interpreter.executioncontext import AbstractActionFlag, ActionFlag
+
+
+class DBState:
+    standard_code = True
+    breakpoint_stack_id = 0
+    breakpoint_funcnames = None
+    breakpoint_filelines = None
+    breakpoint_by_file = None
+    breakpoint_version = 0
+    printed_objects = {}
+    metavars = []
+    watch_progs = []
+    watch_futures = {}
+
+dbstate = DBState()
+
+
+pycode.PyCode.co_revdb_linestarts = None   # or a string: see below
+pycode.PyCode.co_revdb_bkpt_version = 0    # see check_and_trigger_bkpt()
+pycode.PyCode.co_revdb_bkpt_cache = None   # see check_and_trigger_bkpt()
+
+# invariant: "f_revdb_nextline_instr" is the bytecode offset of
+# the start of the line that follows "last_instr".
+pyframe.PyFrame.f_revdb_nextline_instr = -1
+
+
+# ____________________________________________________________
+
+
+def setup_revdb(space):
+    """Called at run-time, before the space is set up.
+
+    The various register_debug_command() lines attach functions
+    to some commands that 'revdb.py' can call, if we are running
+    in replay mode.
+    """
+    assert space.config.translation.reverse_debugger
+    dbstate.space = space
+
+    make_sure_not_resized(dbstate.watch_progs)
+    make_sure_not_resized(dbstate.metavars)
+
+    revdb.register_debug_command(revdb.CMD_PRINT, lambda_print)
+    revdb.register_debug_command(revdb.CMD_BACKTRACE, lambda_backtrace)
+    revdb.register_debug_command(revdb.CMD_LOCALS, lambda_locals)
+    revdb.register_debug_command(revdb.CMD_BREAKPOINTS, lambda_breakpoints)
+    revdb.register_debug_command(revdb.CMD_STACKID, lambda_stackid)
+    revdb.register_debug_command("ALLOCATING", lambda_allocating)
+    revdb.register_debug_command(revdb.CMD_ATTACHID, lambda_attachid)
+    revdb.register_debug_command(revdb.CMD_COMPILEWATCH, lambda_compilewatch)
+    revdb.register_debug_command(revdb.CMD_CHECKWATCH, lambda_checkwatch)
+    revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues)
+
+
+# ____________________________________________________________
+
+
+def enter_call(caller_frame, callee_frame):
+    if dbstate.breakpoint_funcnames is not None:
+        name = callee_frame.getcode().co_name
+        if name in dbstate.breakpoint_funcnames:
+            revdb.breakpoint(dbstate.breakpoint_funcnames[name])
+    if dbstate.breakpoint_stack_id != 0 and caller_frame is not None:
+        if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame):
+            revdb.breakpoint(-1)
+    #
+    code = callee_frame.pycode
+    if code.co_revdb_linestarts is None:
+        build_co_revdb_linestarts(code)
+
+def leave_call(caller_frame, got_exception):
+    if dbstate.breakpoint_stack_id != 0 and caller_frame is not None:
+        if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame):
+            revdb.breakpoint(-2)
+    if we_are_translated():
+        stop_point_activate(-2 + got_exception)
+
+def stop_point():
+    if we_are_translated():
+        revdb.breakpoint(-3)
+
+
+def jump_backward(frame, jumpto):
+    # When we see a jump backward, we set 'f_revdb_nextline_instr' in
+    # such a way that the next instruction, at 'jumpto', will trigger
+    # stop_point_activate().  We have to trigger it even if
+    # 'jumpto' is not actually a start of line.  For example, after a
+    # 'while foo:', the body ends with a JUMP_ABSOLUTE which
+    # jumps back to the *second* opcode of the while.
+    frame.f_revdb_nextline_instr = jumpto
+
+
+def potential_stop_point(frame):
+    if not we_are_translated():
+        return
+    #
+    # We only record a stop_point at every line, not every bytecode.
+    # Uses roughly the same algo as ExecutionContext.run_trace_func()
+    # to know where the line starts are, but tweaked for speed,
+    # avoiding the quadratic complexity when run N times with a large
+    # code object.
+    #
+    cur = frame.last_instr
+    if cur < frame.f_revdb_nextline_instr:
+        return    # fast path: we're still inside the same line as before
+    #
+    call_stop_point_at_line = True
+    co_revdb_linestarts = frame.pycode.co_revdb_linestarts
+    if cur > frame.f_revdb_nextline_instr:
+        #
+        # We jumped forward over the start of the next line.  We're
+        # inside a different line, but we will only trigger a stop
+        # point if we're at the starting bytecode of that line.  Fetch
+        # from co_revdb_linestarts the start of the line that is at or
+        # follows 'cur'.
+        ch = ord(co_revdb_linestarts[cur])
+        if ch == 0:
+            pass   # we are at the start of a line now
+        else:
+            # We are not, so don't call stop_point_activate().
+            # We still have to fill f_revdb_nextline_instr.
+            call_stop_point_at_line = False
+    #
+    if call_stop_point_at_line:
+        stop_point_activate(pycode=frame.pycode, opindex=cur)
+        cur += 1
+        ch = ord(co_revdb_linestarts[cur])
+    #
+    # Update f_revdb_nextline_instr.  Check if 'ch' was greater than
+    # 255, in which case it was rounded down to 255 and we have to
+    # continue looking
+    nextline_instr = cur + ch
+    while ch == 255:
+        ch = ord(co_revdb_linestarts[nextline_instr])
+        nextline_instr += ch
+    frame.f_revdb_nextline_instr = nextline_instr
+
+
+def build_co_revdb_linestarts(code):
+    # Inspired by findlinestarts() in the 'dis' standard module.
+    # Set up 'bits' so that it contains \x00 at line starts and \xff
+    # in-between.
+    bits = ['\xff'] * (len(code.co_code) + 1)
+    if not code.hidden_applevel:
+        lnotab = code.co_lnotab
+        addr = 0
+        p = 0
+        newline = 1
+        while p + 1 < len(lnotab):
+            byte_incr = ord(lnotab[p])
+            line_incr = ord(lnotab[p+1])
+            if byte_incr:
+                if newline != 0:
+                    bits[addr] = '\x00'
+                    newline = 0
+                addr += byte_incr
+            newline |= line_incr
+            p += 2
+        if newline:
+            bits[addr] = '\x00'
+    bits[len(code.co_code)] = '\x00'
+    #
+    # Change 'bits' so that the character at 'i', if not \x00, measures
+    # how far the next \x00 is
+    next_null = len(code.co_code)
+    p = next_null - 1
+    while p >= 0:
+        if bits[p] == '\x00':
+            next_null = p
+        else:
+            ch = next_null - p
+            if ch > 255: ch = 255
+            bits[p] = chr(ch)
+        p -= 1
+    lstart = ''.join(bits)
+    code.co_revdb_linestarts = lstart
+    return lstart
+
+def get_final_lineno(code):
+    lineno = code.co_firstlineno
+    lnotab = code.co_lnotab
+    p = 1
+    while p < len(lnotab):
+        line_incr = ord(lnotab[p])
+        lineno += line_incr
+        p += 2
+    return lineno
+
+def find_line_starts(code):
+    # RPython version of dis.findlinestarts()
+    lnotab = code.co_lnotab
+    lastlineno = -1
+    lineno = code.co_firstlineno
+    addr = 0
+    p = 0
+    result = []
+    while p < len(lnotab) - 1:
+        byte_incr = ord(lnotab[p + 0])
+        line_incr = ord(lnotab[p + 1])
+        if byte_incr:
+            if lineno != lastlineno:
+                result.append((addr, lineno))
+                lastlineno = lineno
+            addr += byte_incr
+        lineno += line_incr
+        p += 2
+    if lineno != lastlineno:
+        result.append((addr, lineno))
+    return result
+
+class NonStandardCode(object):
+    def __enter__(self):
+        dbstate.standard_code = False
+        self.t = dbstate.space.actionflag._ticker
+        self.c = dbstate.space.actionflag._ticker_revdb_count
+    def __exit__(self, *args):
+        dbstate.space.actionflag._ticker = self.t
+        dbstate.space.actionflag._ticker_revdb_count = self.c
+        dbstate.standard_code = True
+non_standard_code = NonStandardCode()
+
+
+def stop_point_activate(place=0, pycode=None, opindex=-1):
+    if revdb.watch_save_state():
+        any_watch_point = False
+        # ^^ this flag is set to True if we must continue to enter this
+        # block of code.  If it is still False for watch_restore_state()
+        # below, then future watch_save_state() will return False too---
+        # until the next time revdb.c:set_revdb_breakpoints() is called.
+        space = dbstate.space
+        with non_standard_code:
+            watch_id = -1
+            if dbstate.breakpoint_by_file is not None:
+                any_watch_point = True
+                if pycode is not None:
+                    watch_id = check_and_trigger_bkpt(pycode, opindex)
+            if watch_id == -1:
+                for prog, watch_id, expected in dbstate.watch_progs:
+                    any_watch_point = True
+                    try:
+                        got = _run_watch(space, prog)
+                    except OperationError as e:
+                        got = e.errorstr(space)
+                    except Exception:
+                        break
+                    if got != expected:
+                        break
+                else:
+                    watch_id = -1
+        revdb.watch_restore_state(any_watch_point)
+        if watch_id != -1:
+            revdb.breakpoint(watch_id)
+    revdb.stop_point(place)
+
+
+def future_object(space):
+    return space.w_Ellipsis    # a random prebuilt object
+
+def load_metavar(index):
+    assert index >= 0
+    space = dbstate.space
+    metavars = dbstate.metavars
+    w_var = metavars[index] if index < len(metavars) else None
+    if w_var is None:
+        raise oefmt(space.w_NameError, "no constant object '$%d'",
+                    index)
+    if w_var is future_object(space):
+        raise oefmt(space.w_RuntimeError,
+                    "'$%d' refers to an object created later in time",
+                    index)
+    return w_var
+
+def set_metavar(index, w_obj):
+    assert index >= 0
+    if index >= len(dbstate.metavars):
+        missing = index + 1 - len(dbstate.metavars)
+        dbstate.metavars = dbstate.metavars + [None] * missing
+    dbstate.metavars[index] = w_obj
+
+
+# ____________________________________________________________
+
+
+def fetch_cur_frame(silent=False):
+    ec = dbstate.space.threadlocals.get_ec()
+    if ec is None:
+        frame = None
+    else:
+        frame = ec.topframeref()
+    if frame is None and not silent:
+        revdb.send_output("No stack.\n")
+    return frame
+
+def compile(source, mode):
+    assert not dbstate.standard_code
+    space = dbstate.space
+    compiler = space.createcompiler()
+    code = compiler.compile(source, '<revdb>', mode, 0,
+                            hidden_applevel=True)
+    return code
+
+
+class W_RevDBOutput(W_Root):
+    softspace = 0
+
+    def __init__(self, space):
+        self.space = space
+
+    def descr_write(self, w_buffer):
+        assert not dbstate.standard_code
+        space = self.space
+        if space.isinstance_w(w_buffer, space.w_unicode):
+            w_buffer = space.call_method(w_buffer, 'encode',
+                                         space.newtext('utf-8'))   # safe?
+        revdb.send_output(space.bytes_w(w_buffer))
+
+def descr_get_softspace(space, revdb):
+    return space.newint(revdb.softspace)
+def descr_set_softspace(space, revdb, w_newvalue):
+    revdb.softspace = space.int_w(w_newvalue)
+
+W_RevDBOutput.typedef = typedef.TypeDef(
+    "revdb_output",
+    write = gateway.interp2app(W_RevDBOutput.descr_write),
+    softspace = typedef.GetSetProperty(descr_get_softspace,
+                                       descr_set_softspace,
+                                       cls=W_RevDBOutput),
+    )
+
+def revdb_displayhook(space, w_obj):
+    """Modified sys.displayhook() that also outputs '$NUM = ',
+    for non-prebuilt objects.  Such objects are then recorded in
+    'printed_objects'.
+    """
+    assert not dbstate.standard_code
+    if space.is_w(w_obj, space.w_None):
+        return
+    uid = revdb.get_unique_id(w_obj)
+    if uid > 0:
+        dbstate.printed_objects[uid] = w_obj
+        revdb.send_nextnid(uid)   # outputs '$NUM = '
+    space.setitem(space.builtin.w_dict, space.newtext('_'), w_obj)
+    # do repr() after setitem: if w_obj was produced successfully,
+    # but its repr crashes because it tries to do I/O, then we already
+    # have it recorded in '_' and in '$NUM ='.
+    w_repr = space.repr(w_obj)
+    if space.isinstance_w(w_repr, space.w_unicode):
+        w_repr = space.call_method(w_repr, 'encode',
+                                   space.newtext('utf-8'))   # safe?
+    revdb.send_output(space.bytes_w(w_repr))
+    revdb.send_output("\n")
+
+ at gateway.unwrap_spec(name='text0', level=int)
+def revdb_importhook(space, name, w_globals=None,
+                     w_locals=None, w_fromlist=None, level=-1):
+    # Incredibly simplified version of __import__, which only returns
+    # already-imported modules and doesn't call any custom import
+    # hooks.  Recognizes only absolute imports.  With a 'fromlist'
+    # argument that is a non-empty list, returns the module 'name3' if
+    # the 'name' argument is 'name1.name2.name3'.  With an empty or
+    # None 'fromlist' argument, returns the module 'name1' instead.
+    return space.appexec([space.newtext(name), w_fromlist or space.w_None,
+                          space.newint(level), space.sys],
+    """(name, fromlist, level, sys):
+        if level > 0:
+            raise ImportError("only absolute imports are "
+                               "supported in the debugger")
+        basename = name.split('.')[0]
+        try:
+            basemod = sys.modules[basename]
+            mod = sys.modules[name]
+        except KeyError:
+            raise ImportError("'%s' not found or not imported yet "
+                    "(the debugger can't import new modules, "
+                    "and only supports absolute imports)" % (name,))
+        if fromlist:
+            return mod
+        return basemod
+    """)
+
+ at specialize.memo()
+def get_revdb_displayhook(space):
+    return gateway.interp2app(revdb_displayhook).spacebind(space)
+
+ at specialize.memo()
+def get_revdb_importhook(space):
+    return gateway.interp2app(revdb_importhook).spacebind(space)
+
+
+def prepare_print_environment(space):
+    assert not dbstate.standard_code
+    w_revdb_output = W_RevDBOutput(space)
+    w_displayhook = get_revdb_displayhook(space)
+    w_import = get_revdb_importhook(space)
+    space.sys.setdictvalue(space, 'stdout', w_revdb_output)
+    space.sys.setdictvalue(space, 'stderr', w_revdb_output)
+    space.sys.setdictvalue(space, 'displayhook', w_displayhook)
+    space.builtin.setdictvalue(space, '__import__', w_import)
+
+def command_print(cmd, expression):
+    frame = fetch_cur_frame()
+    if frame is None:
+        return
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            prepare_print_environment(space)
+            code = compile(expression, 'single')
+            try:
+                code.exec_code(space,
+                               frame.get_w_globals(),
+                               frame.getdictscope())
+
+            except OperationError as operationerr:
+                # can't use sys.excepthook: it will likely try to do 'import
+                # traceback', which might not be doable without using I/O
+                tb = operationerr.get_traceback()
+                if tb is not None:
+                    revdb.send_output("Traceback (most recent call last):\n")
+                    while tb is not None:
+                        if not isinstance(tb, pytraceback.PyTraceback):
+                            revdb.send_output("  ??? %s\n" % tb)
+                            break
+                        show_frame(tb.frame, tb.get_lineno(), indent='  ')
+                        tb = tb.next
+                revdb.send_output('%s\n' % operationerr.errorstr(space))
+
+                # set the sys.last_xxx attributes
+                w_type = operationerr.w_type
+                w_value = operationerr.get_w_value(space)
+                w_tb = operationerr.get_traceback()
+                w_dict = space.sys.w_dict
+                space.setitem(w_dict, space.newtext('last_type'), w_type)
+                space.setitem(w_dict, space.newtext('last_value'), w_value)
+                space.setitem(w_dict, space.newtext('last_traceback'), w_tb)
+
+        except OperationError as e:
+            revdb.send_output('%s\n' % e.errorstr(space, use_repr=True))
+lambda_print = lambda: command_print
+
+
+def file_and_lineno(frame, lineno):
+    code = frame.getcode()
+    return 'File "%s", line %d in %s' % (
+        code.co_filename, lineno, code.co_name)
+
+def show_frame(frame, lineno=0, indent=''):
+    if lineno == 0:
+        lineno = frame.get_last_lineno()
+    revdb.send_output("%s%s\n%s  " % (
+        indent,
+        file_and_lineno(frame, lineno),
+        indent))
+    revdb.send_linecache(frame.getcode().co_filename, lineno)
+
+def display_function_part(frame, max_lines_before, max_lines_after):
+    if frame is None:
+        return
+    code = frame.getcode()
+    if code.co_filename.startswith('<builtin>'):
+        return
+    first_lineno = code.co_firstlineno
+    current_lineno = frame.get_last_lineno()
+    final_lineno = get_final_lineno(code)
+    #
+    ellipsis_after = False
+    if first_lineno < current_lineno - max_lines_before - 1:
+        first_lineno = current_lineno - max_lines_before
+        revdb.send_output("...\n")
+    if final_lineno > current_lineno + max_lines_after + 1:
+        final_lineno = current_lineno + max_lines_after
+        ellipsis_after = True
+    #
+    for i in range(first_lineno, final_lineno + 1):
+        if i == current_lineno:
+            if revdb.current_place() == -2: # <= this is the arg to stop_point()
+                prompt = "<< "     # return
+            elif revdb.current_place() == -1:
+                prompt = "!! "     # exceptional return
+            else:
+                prompt = " > "     # plain line
+            revdb.send_output(prompt)
+        else:
+            revdb.send_output("   ")
+        revdb.send_linecache(code.co_filename, i, strip=False)
+    #
+    if ellipsis_after:
+        revdb.send_output("...\n")
+
+def command_backtrace(cmd, extra):
+    frame = fetch_cur_frame(silent=True)
+    if cmd.c_arg1 == 0:
+        if frame is not None:
+            revdb.send_output("%s:\n" % (
+                file_and_lineno(frame, frame.get_last_lineno()),))
+        display_function_part(frame, max_lines_before=8, max_lines_after=5)
+    elif cmd.c_arg1 == 2:
+        display_function_part(frame, max_lines_before=1000,max_lines_after=1000)
+    else:
+        revdb.send_output("Current call stack (most recent call last):\n")
+        if frame is None:
+            revdb.send_output("  (empty)\n")
+        frames = []
+        while frame is not None:
+            frames.append(frame)
+            if len(frames) == 200:
+                revdb.send_output("  ...\n")
+                break
+            frame = frame.get_f_back()
+        while len(frames) > 0:
+            show_frame(frames.pop(), indent='  ')
+lambda_backtrace = lambda: command_backtrace
+
+
+def command_locals(cmd, extra):
+    frame = fetch_cur_frame()
+    if frame is None:
+        return
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            prepare_print_environment(space)
+            space.appexec([space.sys,
+                           frame.getdictscope()], """(sys, locals):
+                lst = locals.keys()
+                lst.sort()
+                print 'Locals:'
+                for key in lst:
+                    try:
+                        print '    %s =' % key,
+                        s = '%r' % locals[key]
+                        if len(s) > 140:
+                            s = s[:100] + '...' + s[-30:]
+                        print s
+                    except:
+                        exc, val, tb = sys.exc_info()
+                        print '!<%s: %r>' % (exc, val)
+            """)
+        except OperationError as e:
+            revdb.send_output('%s\n' % e.errorstr(space, use_repr=True))
+lambda_locals = lambda: command_locals
+
+# ____________________________________________________________
+
+
+def check_and_trigger_bkpt(pycode, opindex):
+    # We cache on 'pycode.co_revdb_bkpt_cache' either None or a dict
+    # mapping {opindex: bkpt_num}.  This cache is updated when the
+    # version in 'pycode.co_revdb_bkpt_version' does not match
+    # 'dbstate.breakpoint_version' any more.
+    if pycode.co_revdb_bkpt_version != dbstate.breakpoint_version:
+        update_bkpt_cache(pycode)
+    cache = pycode.co_revdb_bkpt_cache
+    if cache is not None and opindex in cache:
+        return cache[opindex]
+    else:
+        return -1
+
+def update_bkpt_cache(pycode):
+    # initialized by command_breakpoints():
+    #     dbstate.breakpoint_filelines == [('FILENAME', lineno, bkpt_num)]
+    # computed lazily (here, first half of the logic):
+    #     dbstate.breakpoint_by_file == {'co_filename': {lineno: bkpt_num}}
+    # the goal is to set:
+    #     pycode.co_revdb_bkpt_cache == {opindex: bkpt_num}
+
+    co_filename = pycode.co_filename
+    try:
+        linenos = dbstate.breakpoint_by_file[co_filename]
+    except KeyError:
+        linenos = None
+        match = co_filename.upper()    # ignore cAsE in filename matching
+        for filename, lineno, bkpt_num in dbstate.breakpoint_filelines:
+            if match.endswith(filename) and (
+                    len(match) == len(filename) or
+                    match[-len(filename)-1] in '/\\'):    # a valid prefix
+                if linenos is None:
+                    linenos = {}
+                linenos[lineno] = bkpt_num
+        dbstate.breakpoint_by_file[co_filename] = linenos
+
+    newcache = None
+    if linenos is not None:
+        # parse co_lnotab to figure out the opindexes that correspond
+        # to the marked line numbers.  here, linenos == {lineno: bkpt_num}
+        for addr, lineno in find_line_starts(pycode):
+            if lineno in linenos:
+                if newcache is None:
+                    newcache = {}
+                newcache[addr] = linenos[lineno]
+
+    pycode.co_revdb_bkpt_cache = newcache
+    pycode.co_revdb_bkpt_version = dbstate.breakpoint_version
+
+
+def valid_identifier(s):
+    if not s:
+        return False
+    if s[0].isdigit():
+        return False
+    for c in s:
+        if not (c.isalnum() or c == '_'):
+            return False
+    return True
+
+def add_breakpoint_funcname(name, i):
+    if dbstate.breakpoint_funcnames is None:
+        dbstate.breakpoint_funcnames = {}
+    dbstate.breakpoint_funcnames[name] = i
+
+def add_breakpoint_fileline(filename, lineno, i):
+    # dbstate.breakpoint_filelines is just a list of (FILENAME, lineno, i).
+    # dbstate.breakpoint_by_file is {co_filename: {lineno: i}}, but
+    # computed lazily when we encounter a code object with the given
+    # co_filename.  Any suffix 'filename' matches 'co_filename'.
+    if dbstate.breakpoint_filelines is None:
+        dbstate.breakpoint_filelines = []
+        dbstate.breakpoint_by_file = {}
+    dbstate.breakpoint_filelines.append((filename.upper(), lineno, i))
+
+def add_breakpoint(name, i):
+    # if it is empty, complain
+    if not name:
+        revdb.send_output("Empty breakpoint name\n")
+        revdb.send_change_breakpoint(i)
+        return
+    # if it is surrounded by < >, it is the name of a code object
+    if name.startswith('<') and name.endswith('>'):
+        add_breakpoint_funcname(name, i)
+        return
+    # if it has no ':', it can be a valid identifier (which we
+    # register as a function name), or a lineno
+    original_name = name
+    j = name.rfind(':')
+    if j < 0:
+        try:
+            lineno = int(name)
+        except ValueError:
+            if name.endswith('()'):
+                n = len(name) - 2
+                assert n >= 0
+                name = name[:n]
+            if not valid_identifier(name):
+                revdb.send_output(
+                    'Note: "%s()" doesn''t look like a function name. '
+                    'Setting breakpoint anyway\n' % name)
+            add_breakpoint_funcname(name, i)
+            name += '()'
+            if name != original_name:
+                revdb.send_change_breakpoint(i, name)
+            return
+        # "number" does the same as ":number"
+        filename = ''
+    else:
+        # if it has a ':', it must end in ':lineno'
+        try:
+            lineno = int(name[j+1:])
+        except ValueError:
+            revdb.send_output('expected a line number after colon\n')
+            revdb.send_change_breakpoint(i)
+            return
+        filename = name[:j]
+
+    # the text before must be a pathname, possibly a relative one,
+    # or be escaped by < >.  if it isn't, make it absolute and normalized
+    # and warn if it doesn't end in '.py'.
+    if filename == '':
+        frame = fetch_cur_frame()
+        if frame is None:
+            revdb.send_change_breakpoint(i)
+            return
+        filename = frame.getcode().co_filename
+    elif filename.startswith('<') and filename.endswith('>'):
+        pass    # use unmodified
+    elif not filename.lower().endswith('.py'):
+        # use unmodified, but warn
+        revdb.send_output(
+            'Note: "%s" doesn''t look like a Python filename. '
+            'Setting breakpoint anyway\n' % (filename,))
+
+    add_breakpoint_fileline(filename, lineno, i)
+    name = '%s:%d' % (filename, lineno)
+    if name != original_name:
+        revdb.send_change_breakpoint(i, name)
+
+def command_breakpoints(cmd, extra):
+    space = dbstate.space
+    dbstate.breakpoint_stack_id = cmd.c_arg1
+    revdb.set_thread_breakpoint(cmd.c_arg2)
+    dbstate.breakpoint_funcnames = None
+    dbstate.breakpoint_filelines = None
+    dbstate.breakpoint_by_file = None
+    dbstate.breakpoint_version += 1
+    watch_progs = []
+    with non_standard_code:
+        for i, kind, name in revdb.split_breakpoints_arg(extra):
+            if kind == 'B':
+                add_breakpoint(name, i)
+            elif kind == 'W':
+                code = interp_marshal.loads(space, space.newtext(name))
+                watch_progs.append((code, i, ''))
+    dbstate.watch_progs = watch_progs[:]
+lambda_breakpoints = lambda: command_breakpoints
+
+
+def command_watchvalues(cmd, extra):
+    expected = extra.split('\x00')
+    for j in range(len(dbstate.watch_progs)):
+        prog, i, _ = dbstate.watch_progs[j]
+        if i >= len(expected):
+            raise IndexError
+        dbstate.watch_progs[j] = prog, i, expected[i]
+lambda_watchvalues = lambda: command_watchvalues
+
+
+def command_stackid(cmd, extra):
+    frame = fetch_cur_frame(silent=True)
+    if frame is not None and cmd.c_arg1 != 0:     # parent_flag
+        frame = dbstate.space.getexecutioncontext().getnextframe_nohidden(frame)
+    if frame is None:
+        uid = 0
+    else:
+        uid = revdb.get_unique_id(frame)
+    if revdb.current_place() == -2:
+        hidden_level = 1      # hide the "<<" events from next/bnext commands
+    else:
+        hidden_level = 0
+    revdb.send_answer(revdb.ANSWER_STACKID, uid, hidden_level)
+lambda_stackid = lambda: command_stackid
+
+
+def command_allocating(uid, gcref):
+    w_obj = cast_gcref_to_instance(W_Root, gcref)
+    dbstate.printed_objects[uid] = w_obj
+    try:
+        index_metavar = dbstate.watch_futures.pop(uid)
+    except KeyError:
+        pass
+    else:
+        set_metavar(index_metavar, w_obj)
+lambda_allocating = lambda: command_allocating
+
+
+def command_attachid(cmd, extra):
+    space = dbstate.space
+    index_metavar = cmd.c_arg1
+    uid = cmd.c_arg2
+    try:
+        w_obj = dbstate.printed_objects[uid]
+    except KeyError:
+        # uid not found, probably a future object
+        dbstate.watch_futures[uid] = index_metavar
+        w_obj = future_object(space)
+    set_metavar(index_metavar, w_obj)
+lambda_attachid = lambda: command_attachid
+
+
+def command_compilewatch(cmd, expression):
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            code = compile(expression, 'eval')
+            marshalled_code = space.bytes_w(interp_marshal.dumps(
+                space, code,
+                space.newint(interp_marshal.Py_MARSHAL_VERSION)))
+        except OperationError as e:
+            revdb.send_watch(e.errorstr(space), ok_flag=0)
+        else:
+            revdb.send_watch(marshalled_code, ok_flag=1)
+lambda_compilewatch = lambda: command_compilewatch
+
+def command_checkwatch(cmd, marshalled_code):
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            code = interp_marshal.loads(space, space.newbytes(marshalled_code))
+            text = _run_watch(space, code)
+        except OperationError as e:
+            revdb.send_watch(e.errorstr(space), ok_flag=0)
+        else:
+            revdb.send_watch(text, ok_flag=1)
+lambda_checkwatch = lambda: command_checkwatch
+
+
+def _run_watch(space, prog):
+    # must be called from non_standard_code!
+    w_dict = space.builtin.w_dict
+    w_res = prog.exec_code(space, w_dict, w_dict)
+    return space.text_w(space.repr(w_res))
+
+
+# ____________________________________________________________
+
+
+ActionFlag._ticker_revdb_count = -1
+
+class RDBSignalActionFlag(AbstractActionFlag):
+    # Used instead of pypy.module.signal.interp_signal.SignalActionFlag
+    # when we have reverse-debugging.  That other class would work too,
+    # but inefficiently: it would generate two words of data per bytecode.
+    # This class is tweaked to generate one byte per _SIG_TICKER_COUNT
+    # bytecodes, at the expense of not reacting to signals instantly.
+
+    # Threads: after 10'000 calls to decrement_ticker(), it should
+    # return -1.  It should also return -1 if there was a signal.
+    # This is done by calling _update_ticker_from_signals() every 100
+    # calls, and invoking rsignal.pypysig_check_and_reset(); this in
+    # turn returns -1 if there was a signal or if it was called 100
+    # times.
+
+    _SIG_TICKER_COUNT = 100
+    _ticker = 0
+    _ticker_revdb_count = _SIG_TICKER_COUNT * 10
+
+    def get_ticker(self):
+        return self._ticker
+
+    def reset_ticker(self, value):
+        self._ticker = value
+
+    def rearm_ticker(self):
+        self._ticker = -1
+
+    def decrement_ticker(self, by):
+        if we_are_translated():
+            c = self._ticker_revdb_count - 1
+            if c < 0:
+                c = self._update_ticker_from_signals()
+            self._ticker_revdb_count = c
+        #if self.has_bytecode_counter:    # this 'if' is constant-folded
+        #    print ("RDBSignalActionFlag: has_bytecode_counter: "
+        #           "not supported for now")
+        #    raise NotImplementedError
+        return self._ticker
+
+    def _update_ticker_from_signals(self):
+        from rpython.rlib import rsignal
+        if dbstate.standard_code:
+            if rsignal.pypysig_check_and_reset():
+                self.rearm_ticker()
+        return self._SIG_TICKER_COUNT
+    _update_ticker_from_signals._dont_inline_ = True
diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/test_reverse_debugging.py
@@ -0,0 +1,121 @@
+import dis
+from pypy.interpreter.reverse_debugging import *
+from pypy.interpreter import reverse_debugging
+from rpython.rlib import revdb
+
+
+class FakeCode:
+    hidden_applevel = False
+    def __init__(self, co_code='', co_lnotab='', co_filename='?'):
+        self.co_firstlineno = 43
+        self.co_code = co_code
+        self.co_lnotab = co_lnotab
+        self.co_revdb_linestarts = None
+        self.co_filename = co_filename
+
+
+try:
+    from hypothesis import given, strategies, example
+except ImportError:
+    pass
+else:
+    @given(strategies.binary())
+    @example("\x01\x02\x03\x04"
+             "\x00\xFF\x20\x30\x00\xFF\x00\x40"
+             "\xFF\x00\x0A\x0B\xFF\x00\x0C\x00")
+    def test_build_co_revdb_linestarts(lnotab):
+        if len(lnotab) & 1:
+            lnotab = lnotab + '\x00'   # make the length even
+        code = FakeCode("?" * sum(map(ord, lnotab[0::2])), lnotab)
+        lstart = build_co_revdb_linestarts(code)
+        assert lstart is code.co_revdb_linestarts
+
+        expected_starts = set()
+        for addr, lineno in dis.findlinestarts(code):
+            expected_starts.add(addr)
+
+        next_start = len(code.co_code)
+        for index in range(len(code.co_code), -1, -1):
+            if index in expected_starts:
+                next_start = index
+            assert lstart[index] == chr(next_start - index
+                                        if next_start - index <= 255
+                                        else 255)
+
+
+class FakeFrame:
+    def __init__(self, code):
+        self.__code = code
+    def getcode(self):
+        return self.__code
+
+def check_add_breakpoint(input, curfilename=None,
+                         expected_funcname=None,
+                         expected_fileline=None,
+                         expected_output=None,
+                         expected_chbkpt=None):
+    dbstate.__dict__.clear()
+    prev = revdb.send_answer, reverse_debugging.fetch_cur_frame
+    try:
+        messages = []
+        def got_message(cmd, arg1=0, arg2=0, arg3=0, extra=""):
+            messages.append((cmd, arg1, arg2, arg3, extra))
+        def my_cur_frame():
+            assert curfilename is not None
+            return FakeFrame(FakeCode(co_filename=curfilename))
+        revdb.send_answer = got_message
+        reverse_debugging.fetch_cur_frame = my_cur_frame
+        add_breakpoint(input, 5)
+    finally:
+        revdb.send_answer, reverse_debugging.fetch_cur_frame = prev
+
+    if expected_funcname is None:
+        assert dbstate.breakpoint_funcnames is None
+    else:
+        assert dbstate.breakpoint_funcnames == {expected_funcname: 5}
+
+    if expected_fileline is None:
+        assert dbstate.breakpoint_filelines is None
+    else:
+        filename, lineno = expected_fileline
+        assert dbstate.breakpoint_filelines == [(filename.upper(), lineno, 5)]
+
+    got_output = None
+    got_chbkpt = None
+    for msg in messages:
+        if msg[0] == revdb.ANSWER_TEXT:
+            assert got_output is None
+            got_output = msg[-1]
+        elif msg[0] == revdb.ANSWER_CHBKPT:
+            assert got_chbkpt is None
+            assert msg[1] == 5
+            got_chbkpt = msg[-1]
+
+    assert got_output == expected_output
+    assert got_chbkpt == expected_chbkpt
+
+def test_add_breakpoint():
+    check_add_breakpoint('', expected_output="Empty breakpoint name\n",
+                         expected_chbkpt='')
+    check_add_breakpoint('foo42', expected_funcname="foo42",
+                         expected_chbkpt="foo42()")
+    check_add_breakpoint('foo42()', expected_funcname="foo42")
+    check_add_breakpoint('foo.bar', expected_funcname="foo.bar",
+        expected_output='Note: "foo.bar()" doesn''t look like a function name.'
+                        ' Setting breakpoint anyway\n',
+        expected_chbkpt="foo.bar()")
+    check_add_breakpoint('<foo.bar>', expected_funcname="<foo.bar>")
+    check_add_breakpoint('42', curfilename='abcd',
+                         expected_fileline=('abcd', 42),
+                         expected_chbkpt='abcd:42')
+    check_add_breakpoint(':42', curfilename='abcd',
+                         expected_fileline=('abcd', 42),
+                         expected_chbkpt='abcd:42')
+    check_add_breakpoint('abcd:42', expected_fileline=('abcd', 42),
+        expected_output='Note: "abcd" doesnt look like a Python filename.'
+                        ' Setting breakpoint anyway\n')
+    check_add_breakpoint('abcd.py:42',
+                         expected_fileline=('abcd.py', 42))
+    check_add_breakpoint('42:abc',
+        expected_output='expected a line number after colon\n',
+        expected_chbkpt='')
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -105,6 +105,7 @@
         '_promote'                   : 'interp_magic._promote',
         'normalize_exc'             : 'interp_magic.normalize_exc',
         'StdErrPrinter'             : 'interp_stderrprinter.W_StdErrPrinter',
+        'side_effects_ok'           : 'interp_magic.side_effects_ok',
         'stack_almost_full'         : 'interp_magic.stack_almost_full',
         'fsencode'                  : 'interp_magic.fsencode',
         'fsdecode'                  : 'interp_magic.fsdecode',
@@ -144,3 +145,7 @@
             features = detect_cpu.getcpufeatures(model)
             self.extra_interpdef('jit_backend_features',
                                     'space.wrap(%r)' % features)
+        if self.space.config.translation.reverse_debugger:
+            self.extra_interpdef('revdb_stop', 'interp_magic.revdb_stop')
+        else:
+            self.extra_interpdef('revdb_stop', 'space.w_None')
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -189,3 +189,23 @@
 def fsdecode(space, w_obj):
     """Direct access to the interp-level fsdecode()"""
     return space.fsdecode(w_obj)
+
+def side_effects_ok(space):
+    """For use with the reverse-debugger: this function normally returns
+    True, but will return False if we are evaluating a debugging command
+    like a watchpoint.  You are responsible for not doing any side effect
+    at all (including no caching) when evaluating watchpoints.  This
+    function is meant to help a bit---you can write:
+
+        if not __pypy__.side_effects_ok():
+            skip the caching logic
+
+    inside getter methods or properties, to make them usable from
+    watchpoints.  Note that you need to re-run ``REVDB=.. pypy''
+    after changing the Python code.
+    """
+    return space.newbool(space._side_effects_ok())
+
+def revdb_stop(space):
+    from pypy.interpreter.reverse_debugging import stop_point
+    stop_point()
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -3,9 +3,9 @@
 """
 import sys, os, py
 
-from rpython.rlib import clibffi, jit, rgc, objectmodel
+from rpython.rlib import clibffi, jit, objectmodel
 from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.module._cffi_backend import cerrno, misc, parse_c_type
@@ -13,6 +13,7 @@
 from pypy.module._cffi_backend.ctypefunc import SIZE_OF_FFI_ARG, W_CTypeFunc
 from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveSigned
 from pypy.module._cffi_backend.ctypevoid import W_CTypeVoid
+from pypy.module._cffi_backend.hide_reveal import hide_reveal1
 
 BIG_ENDIAN = sys.byteorder == 'big'
 
@@ -30,9 +31,7 @@
     return cdata
 
 def reveal_callback(raw_ptr):
-    addr = rffi.cast(llmemory.Address, raw_ptr)
-    gcref = rgc.reveal_gcref(addr)
-    return rgc.try_cast_gcref_to_instance(W_ExternPython, gcref)
+    return hide_reveal1().reveal_object(W_ExternPython, raw_ptr)
 
 
 class Closure(object):
@@ -92,9 +91,7 @@
         return ctype
 
     def hide_object(self):
-        gcref = rgc.cast_instance_to_gcref(self)
-        raw = rgc.hide_nonmovable_gcref(gcref)
-        return rffi.cast(rffi.VOIDP, raw)
+        return hide_reveal1().hide_object(rffi.VOIDP, self)
 
     def _repr_extra(self):
         space = self.space
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -168,7 +168,8 @@
     rawbytes = cache.wdict.get(w_x)
     if rawbytes is None:
         data = space.bytes_w(w_x)
-        if we_are_translated() and not rgc.can_move(data):
+        if (we_are_translated() and not rgc.can_move(data)
+                                and not rgc.must_split_gc_address_space()):
             lldata = llstr(data)
             data_start = (llmemory.cast_ptr_to_adr(lldata) +
                           rffi.offsetof(STR, 'chars') +
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -3,8 +3,9 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module._cffi_backend import ctypeobj, ctypeptr, cdataobj
+from pypy.module._cffi_backend.hide_reveal import hide_reveal2
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.rlib import rgc, objectmodel, jit
+from rpython.rlib import objectmodel, jit
 
 # ____________________________________________________________
 
@@ -15,9 +16,7 @@
     # we can cast the CCHARP back to a W_CDataHandle with reveal_gcref().
     new_cdataobj = objectmodel.instantiate(cdataobj.W_CDataHandle,
                                            nonmovable=True)
-    gcref = rgc.cast_instance_to_gcref(new_cdataobj)
-    _cdata = rgc.hide_nonmovable_gcref(gcref)
-    _cdata = rffi.cast(rffi.CCHARP, _cdata)
+    _cdata = hide_reveal2().hide_object(rffi.CCHARP, new_cdataobj)
     cdataobj.W_CDataHandle.__init__(new_cdataobj, space, _cdata, w_ctype, w_x)
     return new_cdataobj
 
@@ -43,11 +42,10 @@
 @jit.dont_look_inside
 def _reveal(space, ptr):
     addr = rffi.cast(llmemory.Address, ptr)
-    gcref = rgc.reveal_gcref(addr)
-    if not gcref:
+    if not addr:
         raise oefmt(space.w_RuntimeError,
                     "cannot use from_handle() on NULL pointer")
-    cd = rgc.try_cast_gcref_to_instance(cdataobj.W_CDataHandle, gcref)
+    cd = hide_reveal2().reveal_object(cdataobj.W_CDataHandle, addr)
     if cd is None:
         raise oefmt(space.w_SystemError,
                     "ffi.from_handle(): dead or bogus object handle")
diff --git a/pypy/module/_cffi_backend/hide_reveal.py b/pypy/module/_cffi_backend/hide_reveal.py
new file mode 100644
--- /dev/null

From pypy.commits at gmail.com  Wed May 23 13:34:16 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 10:34:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix test
Message-ID: <5b05a618.1c69fb81.c1bb2.e44b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94658:33ccfe808125
Date: 2018-05-23 19:33 +0200
http://bitbucket.org/pypy/pypy/changeset/33ccfe808125/

Log:	Fix test

diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -290,7 +290,7 @@
         import __pypy__
         def kw(**d): return d
         for d in [{}, {1: 2, 3: 4, 5: 6}, {"a": 5, "b": 2, "c": 6}, kw(a=1, b=2)]:
-            assert list(__pypy__.reversed_dict(d)) == list(d.keys()[::-1])
+            assert list(__pypy__.reversed_dict(d)) == list(d.keys())[::-1]
         raises(TypeError, __pypy__.reversed_dict, 42)
 
     def test_reversed_dict_runtimeerror(self):

From pypy.commits at gmail.com  Wed May 23 13:35:20 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 10:35:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-reverse-debugger: merge py3.5 again
Message-ID: <5b05a658.1c69fb81.46ce9.1ac8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-reverse-debugger
Changeset: r94659:6e57c08457f6
Date: 2018-05-23 19:34 +0200
http://bitbucket.org/pypy/pypy/changeset/6e57c08457f6/

Log:	merge py3.5 again

diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -290,7 +290,7 @@
         import __pypy__
         def kw(**d): return d
         for d in [{}, {1: 2, 3: 4, 5: 6}, {"a": 5, "b": 2, "c": 6}, kw(a=1, b=2)]:
-            assert list(__pypy__.reversed_dict(d)) == list(d.keys()[::-1])
+            assert list(__pypy__.reversed_dict(d)) == list(d.keys())[::-1]
         raises(TypeError, __pypy__.reversed_dict, 42)
 
     def test_reversed_dict_runtimeerror(self):

From pypy.commits at gmail.com  Wed May 23 14:40:41 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 11:40:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-reverse-debugger: manual 2to3
Message-ID: <5b05b5a9.1c69fb81.9885b.7896@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-reverse-debugger
Changeset: r94660:216549ff7482
Date: 2018-05-23 19:40 +0200
http://bitbucket.org/pypy/pypy/changeset/216549ff7482/

Log:	manual 2to3

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -532,19 +532,18 @@
             prepare_print_environment(space)
             space.appexec([space.sys,
                            frame.getdictscope()], """(sys, locals):
-                lst = locals.keys()
-                lst.sort()
-                print 'Locals:'
+                lst = sorted(locals.keys())
+                print('Locals:')
                 for key in lst:
                     try:
-                        print '    %s =' % key,
+                        print('    %s =' % key, end=' ', flush=True)
                         s = '%r' % locals[key]
                         if len(s) > 140:
                             s = s[:100] + '...' + s[-30:]
-                        print s
+                        print(s)
                     except:
                         exc, val, tb = sys.exc_info()
-                        print '!<%s: %r>' % (exc, val)
+                        print('!<%s: %r>' % (exc, val))
             """)
         except OperationError as e:
             revdb.send_output('%s\n' % e.errorstr(space, use_repr=True))

From pypy.commits at gmail.com  Wed May 23 14:40:46 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 11:40:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-reverse-debugger: add comment
Message-ID: <5b05b5ae.1c69fb81.36a3f.51b5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-reverse-debugger
Changeset: r94662:9239ac7f5456
Date: 2018-05-23 20:12 +0200
http://bitbucket.org/pypy/pypy/changeset/9239ac7f5456/

Log:	add comment

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -332,6 +332,7 @@
 W_RevDBOutput.typedef = typedef.TypeDef(
     "revdb_output",
     write = gateway.interp2app(W_RevDBOutput.descr_write),
+    # XXX is 'softspace' still necessary in Python 3?
     softspace = typedef.GetSetProperty(descr_get_softspace,
                                        descr_set_softspace,
                                        cls=W_RevDBOutput),

From pypy.commits at gmail.com  Wed May 23 14:40:48 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 11:40:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-reverse-debugger: hg merge reverse-debugger
Message-ID: <5b05b5b0.1c69fb81.9e7a.0674@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-reverse-debugger
Changeset: r94663:9e3c05dbb3fd
Date: 2018-05-23 20:13 +0200
http://bitbucket.org/pypy/pypy/changeset/9e3c05dbb3fd/

Log:	hg merge reverse-debugger

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -88,17 +88,24 @@
     run_protected() handles details like forwarding exceptions to
     sys.excepthook(), catching SystemExit, etc.
     """
+    # don't use try:except: here, otherwise the exception remains
+    # visible in user code.  Make sure revdb_stop is a callable, so
+    # that we can call it immediately after finally: below.  Doing
+    # so minimizes the number of "blind" lines that we need to go
+    # back from, with "bstep", after we do "continue" in revdb.
     if '__pypy__' in sys.builtin_module_names:
         from __pypy__ import revdb_stop
     else:
         revdb_stop = None
+    if revdb_stop is None:
+        revdb_stop = lambda: None
+
     try:
         # run it
         try:
             f(*fargs, **fkwds)
         finally:
-            if revdb_stop:
-                revdb_stop()
+            revdb_stop()
             sys.settrace(None)
             sys.setprofile(None)
     except SystemExit as e:
diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -123,6 +123,8 @@
 
     def compute_result_annotation(self):
         translator = self.bookkeeper.annotator.translator
+        if translator.config.translation.reverse_debugger:
+            return    # ignore and use the regular hash, with reverse-debugger
         if hasattr(translator, 'll_hash_string'):
             assert translator.ll_hash_string == ll_hash_string_siphash24
         else:
@@ -134,6 +136,9 @@
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+        translator = hop.rtyper.annotator.translator
+        if translator.config.translation.reverse_debugger:
+            return    # ignore and use the regular hash, with reverse-debugger
         bk = hop.rtyper.annotator.bookkeeper
         s_callable = bk.immutablevalue(initialize_from_env)
         r_callable = hop.rtyper.getrepr(s_callable)

From pypy.commits at gmail.com  Wed May 23 14:40:44 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 11:40:44 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Disable rsiphash if we're
 using the --revdb option
Message-ID: <5b05b5ac.1c69fb81.9a234.2f02@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r94661:f2c1d407e8d7
Date: 2018-05-23 20:12 +0200
http://bitbucket.org/pypy/pypy/changeset/f2c1d407e8d7/

Log:	Disable rsiphash if we're using the --revdb option

diff --git a/rpython/rlib/rsiphash.py b/rpython/rlib/rsiphash.py
--- a/rpython/rlib/rsiphash.py
+++ b/rpython/rlib/rsiphash.py
@@ -123,6 +123,8 @@
 
     def compute_result_annotation(self):
         translator = self.bookkeeper.annotator.translator
+        if translator.config.translation.reverse_debugger:
+            return    # ignore and use the regular hash, with reverse-debugger
         if hasattr(translator, 'll_hash_string'):
             assert translator.ll_hash_string == ll_hash_string_siphash24
         else:
@@ -134,6 +136,9 @@
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
+        translator = hop.rtyper.annotator.translator
+        if translator.config.translation.reverse_debugger:
+            return    # ignore and use the regular hash, with reverse-debugger
         bk = hop.rtyper.annotator.bookkeeper
         s_callable = bk.immutablevalue(initialize_from_env)
         r_callable = hop.rtyper.getrepr(s_callable)

From pypy.commits at gmail.com  Wed May 23 14:48:45 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 11:48:45 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: oops
Message-ID: <5b05b78d.1c69fb81.680a9.8623@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r94664:0242387b5ccf
Date: 2018-05-23 20:47 +0200
http://bitbucket.org/pypy/pypy/changeset/0242387b5ccf/

Log:	oops

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -707,7 +707,7 @@
             if kind == 'B':
                 add_breakpoint(name, i)
             elif kind == 'W':
-                code = interp_marshal.loads(space, space.newtext(name))
+                code = interp_marshal.loads(space, space.newbytes(name))
                 watch_progs.append((code, i, ''))
     dbstate.watch_progs = watch_progs[:]
 lambda_breakpoints = lambda: command_breakpoints

From pypy.commits at gmail.com  Wed May 23 14:48:47 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 11:48:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-reverse-debugger: hg merge reverse-debugger
Message-ID: <5b05b78f.1c69fb81.c9e21.fbe1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-reverse-debugger
Changeset: r94665:f05df0d75e88
Date: 2018-05-23 20:48 +0200
http://bitbucket.org/pypy/pypy/changeset/f05df0d75e88/

Log:	hg merge reverse-debugger

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -707,7 +707,7 @@
             if kind == 'B':
                 add_breakpoint(name, i)
             elif kind == 'W':
-                code = interp_marshal.loads(space, space.newtext(name))
+                code = interp_marshal.loads(space, space.newbytes(name))
                 watch_progs.append((code, i, ''))
     dbstate.watch_progs = watch_progs[:]
 lambda_breakpoints = lambda: command_breakpoints

From pypy.commits at gmail.com  Wed May 23 15:04:59 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 12:04:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Back-port from py3.5: this makes it
 easier to regenerate the DFA. See
Message-ID: <5b05bb5b.1c69fb81.dbc44.03df@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94666:61041045408c
Date: 2018-05-23 21:02 +0200
http://bitbucket.org/pypy/pypy/changeset/61041045408c/

Log:	Back-port from py3.5: this makes it easier to regenerate the DFA.
	See comments in dfa_generated.py

diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/pyparser/dfa_generated.py
@@ -0,0 +1,290 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY gendfa.py
+# DO NOT EDIT
+# TO REGENERATE THE FILE, RUN:
+#     python gendfa.py > dfa_generated.py
+
+from pypy.interpreter.pyparser import automata
+accepts = [True, True, True, True, True, True, True, True,
+           True, True, False, True, True, True, True, False,
+           False, False, True, False, False, True, False,
+           False, True, False, True, False, True, False,
+           False, True, False, False, True, True, True,
+           False, False, True, False, False, False, True]
+states = [
+    # 0
+    {'\t': 0, '\n': 13, '\x0c': 0,
+     '\r': 14, ' ': 0, '!': 10, '"': 16,
+     '#': 18, '%': 12, '&': 12, "'": 15,
+     '(': 13, ')': 13, '*': 7, '+': 12,
+     ',': 13, '-': 12, '.': 6, '/': 11,
+     '0': 4, '1': 5, '2': 5, '3': 5,
+     '4': 5, '5': 5, '6': 5, '7': 5,
+     '8': 5, '9': 5, ':': 13, ';': 13,
+     '<': 9, '=': 12, '>': 8, '@': 13,
+     'A': 1, 'B': 2, 'C': 1, 'D': 1,
+     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'I': 1, 'J': 1, 'K': 1, 'L': 1,
+     'M': 1, 'N': 1, 'O': 1, 'P': 1,
+     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
+     'U': 2, 'V': 1, 'W': 1, 'X': 1,
+     'Y': 1, 'Z': 1, '[': 13, '\\': 17,
+     ']': 13, '^': 12, '_': 1, '`': 13,
+     'a': 1, 'b': 2, 'c': 1, 'd': 1,
+     'e': 1, 'f': 1, 'g': 1, 'h': 1,
+     'i': 1, 'j': 1, 'k': 1, 'l': 1,
+     'm': 1, 'n': 1, 'o': 1, 'p': 1,
+     'q': 1, 'r': 3, 's': 1, 't': 1,
+     'u': 2, 'v': 1, 'w': 1, 'x': 1,
+     'y': 1, 'z': 1, '{': 13, '|': 12,
+     '}': 13, '~': 13},
+    # 1
+    {'0': 1, '1': 1, '2': 1, '3': 1,
+     '4': 1, '5': 1, '6': 1, '7': 1,
+     '8': 1, '9': 1, 'A': 1, 'B': 1,
+     'C': 1, 'D': 1, 'E': 1, 'F': 1,
+     'G': 1, 'H': 1, 'I': 1, 'J': 1,
+     'K': 1, 'L': 1, 'M': 1, 'N': 1,
+     'O': 1, 'P': 1, 'Q': 1, 'R': 1,
+     'S': 1, 'T': 1, 'U': 1, 'V': 1,
+     'W': 1, 'X': 1, 'Y': 1, 'Z': 1,
+     '_': 1, 'a': 1, 'b': 1, 'c': 1,
+     'd': 1, 'e': 1, 'f': 1, 'g': 1,
+     'h': 1, 'i': 1, 'j': 1, 'k': 1,
+     'l': 1, 'm': 1, 'n': 1, 'o': 1,
+     'p': 1, 'q': 1, 'r': 1, 's': 1,
+     't': 1, 'u': 1, 'v': 1, 'w': 1,
+     'x': 1, 'y': 1, 'z': 1},
+    # 2
+    {'"': 16, "'": 15, '0': 1, '1': 1,
+     '2': 1, '3': 1, '4': 1, '5': 1,
+     '6': 1, '7': 1, '8': 1, '9': 1,
+     'A': 1, 'B': 1, 'C': 1, 'D': 1,
+     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'I': 1, 'J': 1, 'K': 1, 'L': 1,
+     'M': 1, 'N': 1, 'O': 1, 'P': 1,
+     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
+     'U': 1, 'V': 1, 'W': 1, 'X': 1,
+     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
+     'b': 1, 'c': 1, 'd': 1, 'e': 1,
+     'f': 1, 'g': 1, 'h': 1, 'i': 1,
+     'j': 1, 'k': 1, 'l': 1, 'm': 1,
+     'n': 1, 'o': 1, 'p': 1, 'q': 1,
+     'r': 3, 's': 1, 't': 1, 'u': 1,
+     'v': 1, 'w': 1, 'x': 1, 'y': 1,
+     'z': 1},
+    # 3
+    {'"': 16, "'": 15, '0': 1, '1': 1,
+     '2': 1, '3': 1, '4': 1, '5': 1,
+     '6': 1, '7': 1, '8': 1, '9': 1,
+     'A': 1, 'B': 1, 'C': 1, 'D': 1,
+     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'I': 1, 'J': 1, 'K': 1, 'L': 1,
+     'M': 1, 'N': 1, 'O': 1, 'P': 1,
+     'Q': 1, 'R': 1, 'S': 1, 'T': 1,
+     'U': 1, 'V': 1, 'W': 1, 'X': 1,
+     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
+     'b': 1, 'c': 1, 'd': 1, 'e': 1,
+     'f': 1, 'g': 1, 'h': 1, 'i': 1,
+     'j': 1, 'k': 1, 'l': 1, 'm': 1,
+     'n': 1, 'o': 1, 'p': 1, 'q': 1,
+     'r': 1, 's': 1, 't': 1, 'u': 1,
+     'v': 1, 'w': 1, 'x': 1, 'y': 1,
+     'z': 1},
+    # 4
+    {'.': 24, '0': 21, '1': 21, '2': 21,
+     '3': 21, '4': 21, '5': 21, '6': 21,
+     '7': 21, '8': 23, '9': 23, 'B': 22,
+     'E': 25, 'J': 13, 'L': 13, 'O': 20,
+     'X': 19, 'b': 22, 'e': 25, 'j': 13,
+     'l': 13, 'o': 20, 'x': 19},
+    # 5
+    {'.': 24, '0': 5, '1': 5, '2': 5,
+     '3': 5, '4': 5, '5': 5, '6': 5,
+     '7': 5, '8': 5, '9': 5, 'E': 25,
+     'J': 13, 'L': 13, 'e': 25, 'j': 13,
+     'l': 13},
+    # 6
+    {'0': 26, '1': 26, '2': 26, '3': 26,
+     '4': 26, '5': 26, '6': 26, '7': 26,
+     '8': 26, '9': 26},
+    # 7
+    {'*': 12, '=': 13},
+    # 8
+    {'=': 13, '>': 12},
+    # 9
+    {'<': 12, '=': 13, '>': 13},
+    # 10
+    {'=': 13},
+    # 11
+    {'/': 12, '=': 13},
+    # 12
+    {'=': 13},
+    # 13
+    {},
+    # 14
+    {'\n': 13},
+    # 15
+    {automata.DEFAULT: 30, '\n': 27,
+     '\r': 27, "'": 28, '\\': 29},
+    # 16
+    {automata.DEFAULT: 33, '\n': 27,
+     '\r': 27, '"': 31, '\\': 32},
+    # 17
+    {'\n': 13, '\r': 14},
+    # 18
+    {automata.DEFAULT: 18, '\n': 27, '\r': 27},
+    # 19
+    {'0': 34, '1': 34, '2': 34, '3': 34,
+     '4': 34, '5': 34, '6': 34, '7': 34,
+     '8': 34, '9': 34, 'A': 34, 'B': 34,
+     'C': 34, 'D': 34, 'E': 34, 'F': 34,
+     'a': 34, 'b': 34, 'c': 34, 'd': 34,
+     'e': 34, 'f': 34},
+    # 20
+    {'0': 35, '1': 35, '2': 35, '3': 35,
+     '4': 35, '5': 35, '6': 35, '7': 35},
+    # 21
+    {'.': 24, '0': 21, '1': 21, '2': 21,
+     '3': 21, '4': 21, '5': 21, '6': 21,
+     '7': 21, '8': 23, '9': 23, 'E': 25,
+     'J': 13, 'L': 13, 'e': 25, 'j': 13,
+     'l': 13},
+    # 22
+    {'0': 36, '1': 36},
+    # 23
+    {'.': 24, '0': 23, '1': 23, '2': 23,
+     '3': 23, '4': 23, '5': 23, '6': 23,
+     '7': 23, '8': 23, '9': 23, 'E': 25,
+     'J': 13, 'e': 25, 'j': 13},
+    # 24
+    {'0': 24, '1': 24, '2': 24, '3': 24,
+     '4': 24, '5': 24, '6': 24, '7': 24,
+     '8': 24, '9': 24, 'E': 37, 'J': 13,
+     'e': 37, 'j': 13},
+    # 25
+    {'+': 38, '-': 38, '0': 39, '1': 39,
+     '2': 39, '3': 39, '4': 39, '5': 39,
+     '6': 39, '7': 39, '8': 39, '9': 39},
+    # 26
+    {'0': 26, '1': 26, '2': 26, '3': 26,
+     '4': 26, '5': 26, '6': 26, '7': 26,
+     '8': 26, '9': 26, 'E': 37, 'J': 13,
+     'e': 37, 'j': 13},
+    # 27
+    {},
+    # 28
+    {"'": 13},
+    # 29
+    {automata.DEFAULT: 40, '\n': 13, '\r': 14},
+    # 30
+    {automata.DEFAULT: 30, '\n': 27,
+     '\r': 27, "'": 13, '\\': 29},
+    # 31
+    {'"': 13},
+    # 32
+    {automata.DEFAULT: 41, '\n': 13, '\r': 14},
+    # 33
+    {automata.DEFAULT: 33, '\n': 27,
+     '\r': 27, '"': 13, '\\': 32},
+    # 34
+    {'0': 34, '1': 34, '2': 34, '3': 34,
+     '4': 34, '5': 34, '6': 34, '7': 34,
+     '8': 34, '9': 34, 'A': 34, 'B': 34,
+     'C': 34, 'D': 34, 'E': 34, 'F': 34,
+     'L': 13, 'a': 34, 'b': 34, 'c': 34,
+     'd': 34, 'e': 34, 'f': 34, 'l': 13},
+    # 35
+    {'0': 35, '1': 35, '2': 35, '3': 35,
+     '4': 35, '5': 35, '6': 35, '7': 35,
+     'L': 13, 'l': 13},
+    # 36
+    {'0': 36, '1': 36, 'L': 13, 'l': 13},
+    # 37
+    {'+': 42, '-': 42, '0': 43, '1': 43,
+     '2': 43, '3': 43, '4': 43, '5': 43,
+     '6': 43, '7': 43, '8': 43, '9': 43},
+    # 38
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39},
+    # 39
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39, 'J': 13, 'j': 13},
+    # 40
+    {automata.DEFAULT: 40, '\n': 27,
+     '\r': 27, "'": 13, '\\': 29},
+    # 41
+    {automata.DEFAULT: 41, '\n': 27,
+     '\r': 27, '"': 13, '\\': 32},
+    # 42
+    {'0': 43, '1': 43, '2': 43, '3': 43,
+     '4': 43, '5': 43, '6': 43, '7': 43,
+     '8': 43, '9': 43},
+    # 43
+    {'0': 43, '1': 43, '2': 43, '3': 43,
+     '4': 43, '5': 43, '6': 43, '7': 43,
+     '8': 43, '9': 43, 'J': 13, 'j': 13},
+    ]
+pseudoDFA = automata.DFA(states, accepts)
+
+accepts = [False, False, False, False, False, True]
+states = [
+    # 0
+    {automata.DEFAULT: 0, '"': 1, '\\': 2},
+    # 1
+    {automata.DEFAULT: 4, '"': 3, '\\': 2},
+    # 2
+    {automata.DEFAULT: 4},
+    # 3
+    {automata.DEFAULT: 4, '"': 5, '\\': 2},
+    # 4
+    {automata.DEFAULT: 4, '"': 1, '\\': 2},
+    # 5
+    {automata.DEFAULT: 4, '"': 5, '\\': 2},
+    ]
+double3DFA = automata.NonGreedyDFA(states, accepts)
+
+accepts = [False, False, False, False, False, True]
+states = [
+    # 0
+    {automata.DEFAULT: 0, "'": 1, '\\': 2},
+    # 1
+    {automata.DEFAULT: 4, "'": 3, '\\': 2},
+    # 2
+    {automata.DEFAULT: 4},
+    # 3
+    {automata.DEFAULT: 4, "'": 5, '\\': 2},
+    # 4
+    {automata.DEFAULT: 4, "'": 1, '\\': 2},
+    # 5
+    {automata.DEFAULT: 4, "'": 5, '\\': 2},
+    ]
+single3DFA = automata.NonGreedyDFA(states, accepts)
+
+accepts = [False, True, False, False]
+states = [
+    # 0
+    {automata.DEFAULT: 0, "'": 1, '\\': 2},
+    # 1
+    {},
+    # 2
+    {automata.DEFAULT: 3},
+    # 3
+    {automata.DEFAULT: 3, "'": 1, '\\': 2},
+    ]
+singleDFA = automata.DFA(states, accepts)
+
+accepts = [False, True, False, False]
+states = [
+    # 0
+    {automata.DEFAULT: 0, '"': 1, '\\': 2},
+    # 1
+    {},
+    # 2
+    {automata.DEFAULT: 3},
+    # 3
+    {automata.DEFAULT: 3, '"': 1, '\\': 2},
+    ]
+doubleDFA = automata.DFA(states, accepts)
+
diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/gendfa.py
rename from pypy/interpreter/pyparser/genpytokenize.py
rename to pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/genpytokenize.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-"""Module genPytokenize
+"""Module gendfa
 
 Generates finite state automata for recognizing Python tokens.  These are hand
 coded versions of the regular expressions originally appearing in Ping's
@@ -7,6 +7,10 @@
 
 When run from the command line, this should pretty print the DFA machinery.
 
+To regenerate the dfa, run::
+
+    $ python gendfa.py > dfa_generated.py
+
 $Id: genPytokenize.py,v 1.1 2003/10/02 17:37:17 jriehl Exp $
 """
 
@@ -322,6 +326,12 @@
     return ''.join(lines)
 
 def main ():
+    print "# THIS FILE IS AUTOMATICALLY GENERATED BY gendfa.py"
+    print "# DO NOT EDIT"
+    print "# TO REGENERATE THE FILE, RUN:"
+    print "#     python gendfa.py > dfa_generated.py"
+    print
+    print "from pypy.interpreter.pyparser import automata"
     pseudoDFA, states_pseudoDFA = makePyPseudoDFA()
     print output("pseudoDFA", "DFA", pseudoDFA, states_pseudoDFA)
     endDFAMap = makePyEndDFAMap()
@@ -331,7 +341,7 @@
     print output("single3DFA", "NonGreedyDFA", dfa, states)
     dfa, states = endDFAMap["'"]
     print output("singleDFA", "DFA", dfa, states)
-    dfa, states = endDFAMap["\""]
+    dfa, states = endDFAMap['"']
     print output("doubleDFA", "DFA", dfa, states)
 
 # ______________________________________________________________________
diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py
--- a/pypy/interpreter/pyparser/pytokenize.py
+++ b/pypy/interpreter/pyparser/pytokenize.py
@@ -17,299 +17,10 @@
 # ______________________________________________________________________
 
 from pypy.interpreter.pyparser import automata
+from pypy.interpreter.pyparser.dfa_generated import *
 
 __all__ = [ "tokenize" ]
 
-# ______________________________________________________________________
-# Automatically generated DFA's
-
-accepts = [True, True, True, True, True, True, True, True,
-           True, True, False, True, True, True, True, False,
-           False, False, True, False, False, True, False,
-           False, True, False, True, False, True, False,
-           False, True, False, False, True, True, True,
-           False, False, True, False, False, False, True]
-states = [
-    # 0
-    {'\t': 0, '\n': 13, '\x0c': 0,
-     '\r': 14, ' ': 0, '!': 10, '"': 16,
-     '#': 18, '%': 12, '&': 12, "'": 15,
-     '(': 13, ')': 13, '*': 7, '+': 12,
-     ',': 13, '-': 12, '.': 6, '/': 11,
-     '0': 4, '1': 5, '2': 5, '3': 5,
-     '4': 5, '5': 5, '6': 5, '7': 5,
-     '8': 5, '9': 5, ':': 13, ';': 13,
-     '<': 9, '=': 12, '>': 8, '@': 13,
-     'A': 1, 'B': 2, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
-     'I': 1, 'J': 1, 'K': 1, 'L': 1,
-     'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
-     'U': 2, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '[': 13, '\\': 17,
-     ']': 13, '^': 12, '_': 1, '`': 13,
-     'a': 1, 'b': 2, 'c': 1, 'd': 1,
-     'e': 1, 'f': 1, 'g': 1, 'h': 1,
-     'i': 1, 'j': 1, 'k': 1, 'l': 1,
-     'm': 1, 'n': 1, 'o': 1, 'p': 1,
-     'q': 1, 'r': 3, 's': 1, 't': 1,
-     'u': 2, 'v': 1, 'w': 1, 'x': 1,
-     'y': 1, 'z': 1, '{': 13, '|': 12,
-     '}': 13, '~': 13},
-    # 1
-    {'0': 1, '1': 1, '2': 1, '3': 1,
-     '4': 1, '5': 1, '6': 1, '7': 1,
-     '8': 1, '9': 1, 'A': 1, 'B': 1,
-     'C': 1, 'D': 1, 'E': 1, 'F': 1,
-     'G': 1, 'H': 1, 'I': 1, 'J': 1,
-     'K': 1, 'L': 1, 'M': 1, 'N': 1,
-     'O': 1, 'P': 1, 'Q': 1, 'R': 1,
-     'S': 1, 'T': 1, 'U': 1, 'V': 1,
-     'W': 1, 'X': 1, 'Y': 1, 'Z': 1,
-     '_': 1, 'a': 1, 'b': 1, 'c': 1,
-     'd': 1, 'e': 1, 'f': 1, 'g': 1,
-     'h': 1, 'i': 1, 'j': 1, 'k': 1,
-     'l': 1, 'm': 1, 'n': 1, 'o': 1,
-     'p': 1, 'q': 1, 'r': 1, 's': 1,
-     't': 1, 'u': 1, 'v': 1, 'w': 1,
-     'x': 1, 'y': 1, 'z': 1},
-    # 2
-    {'"': 16, "'": 15, '0': 1, '1': 1,
-     '2': 1, '3': 1, '4': 1, '5': 1,
-     '6': 1, '7': 1, '8': 1, '9': 1,
-     'A': 1, 'B': 1, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
-     'I': 1, 'J': 1, 'K': 1, 'L': 1,
-     'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
-     'U': 1, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
-     'b': 1, 'c': 1, 'd': 1, 'e': 1,
-     'f': 1, 'g': 1, 'h': 1, 'i': 1,
-     'j': 1, 'k': 1, 'l': 1, 'm': 1,
-     'n': 1, 'o': 1, 'p': 1, 'q': 1,
-     'r': 3, 's': 1, 't': 1, 'u': 1,
-     'v': 1, 'w': 1, 'x': 1, 'y': 1,
-     'z': 1},
-    # 3
-    {'"': 16, "'": 15, '0': 1, '1': 1,
-     '2': 1, '3': 1, '4': 1, '5': 1,
-     '6': 1, '7': 1, '8': 1, '9': 1,
-     'A': 1, 'B': 1, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
-     'I': 1, 'J': 1, 'K': 1, 'L': 1,
-     'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 1, 'S': 1, 'T': 1,
-     'U': 1, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
-     'b': 1, 'c': 1, 'd': 1, 'e': 1,
-     'f': 1, 'g': 1, 'h': 1, 'i': 1,
-     'j': 1, 'k': 1, 'l': 1, 'm': 1,
-     'n': 1, 'o': 1, 'p': 1, 'q': 1,
-     'r': 1, 's': 1, 't': 1, 'u': 1,
-     'v': 1, 'w': 1, 'x': 1, 'y': 1,
-     'z': 1},
-    # 4
-    {'.': 24, '0': 21, '1': 21, '2': 21,
-     '3': 21, '4': 21, '5': 21, '6': 21,
-     '7': 21, '8': 23, '9': 23, 'B': 22,
-     'E': 25, 'J': 13, 'L': 13, 'O': 20,
-     'X': 19, 'b': 22, 'e': 25, 'j': 13,
-     'l': 13, 'o': 20, 'x': 19},
-    # 5
-    {'.': 24, '0': 5, '1': 5, '2': 5,
-     '3': 5, '4': 5, '5': 5, '6': 5,
-     '7': 5, '8': 5, '9': 5, 'E': 25,
-     'J': 13, 'L': 13, 'e': 25, 'j': 13,
-     'l': 13},
-    # 6
-    {'0': 26, '1': 26, '2': 26, '3': 26,
-     '4': 26, '5': 26, '6': 26, '7': 26,
-     '8': 26, '9': 26},
-    # 7
-    {'*': 12, '=': 13},
-    # 8
-    {'=': 13, '>': 12},
-    # 9
-    {'<': 12, '=': 13, '>': 13},
-    # 10
-    {'=': 13},
-    # 11
-    {'/': 12, '=': 13},
-    # 12
-    {'=': 13},
-    # 13
-    {},
-    # 14
-    {'\n': 13},
-    # 15
-    {automata.DEFAULT: 30, '\n': 27,
-     '\r': 27, "'": 28, '\\': 29},
-    # 16
-    {automata.DEFAULT: 33, '\n': 27,
-     '\r': 27, '"': 31, '\\': 32},
-    # 17
-    {'\n': 13, '\r': 14},
-    # 18
-    {automata.DEFAULT: 18, '\n': 27, '\r': 27},
-    # 19
-    {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34,
-     '8': 34, '9': 34, 'A': 34, 'B': 34,
-     'C': 34, 'D': 34, 'E': 34, 'F': 34,
-     'a': 34, 'b': 34, 'c': 34, 'd': 34,
-     'e': 34, 'f': 34},
-    # 20
-    {'0': 35, '1': 35, '2': 35, '3': 35,
-     '4': 35, '5': 35, '6': 35, '7': 35},
-    # 21
-    {'.': 24, '0': 21, '1': 21, '2': 21,
-     '3': 21, '4': 21, '5': 21, '6': 21,
-     '7': 21, '8': 23, '9': 23, 'E': 25,
-     'J': 13, 'L': 13, 'e': 25, 'j': 13,
-     'l': 13},
-    # 22
-    {'0': 36, '1': 36},
-    # 23
-    {'.': 24, '0': 23, '1': 23, '2': 23,
-     '3': 23, '4': 23, '5': 23, '6': 23,
-     '7': 23, '8': 23, '9': 23, 'E': 25,
-     'J': 13, 'e': 25, 'j': 13},
-    # 24
-    {'0': 24, '1': 24, '2': 24, '3': 24,
-     '4': 24, '5': 24, '6': 24, '7': 24,
-     '8': 24, '9': 24, 'E': 37, 'J': 13,
-     'e': 37, 'j': 13},
-    # 25
-    {'+': 38, '-': 38, '0': 39, '1': 39,
-     '2': 39, '3': 39, '4': 39, '5': 39,
-     '6': 39, '7': 39, '8': 39, '9': 39},
-    # 26
-    {'0': 26, '1': 26, '2': 26, '3': 26,
-     '4': 26, '5': 26, '6': 26, '7': 26,
-     '8': 26, '9': 26, 'E': 37, 'J': 13,
-     'e': 37, 'j': 13},
-    # 27
-    {},
-    # 28
-    {"'": 13},
-    # 29
-    {automata.DEFAULT: 40, '\n': 13, '\r': 14},
-    # 30
-    {automata.DEFAULT: 30, '\n': 27,
-     '\r': 27, "'": 13, '\\': 29},
-    # 31
-    {'"': 13},
-    # 32
-    {automata.DEFAULT: 41, '\n': 13, '\r': 14},
-    # 33
-    {automata.DEFAULT: 33, '\n': 27,
-     '\r': 27, '"': 13, '\\': 32},
-    # 34
-    {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34,
-     '8': 34, '9': 34, 'A': 34, 'B': 34,
-     'C': 34, 'D': 34, 'E': 34, 'F': 34,
-     'L': 13, 'a': 34, 'b': 34, 'c': 34,
-     'd': 34, 'e': 34, 'f': 34, 'l': 13},
-    # 35
-    {'0': 35, '1': 35, '2': 35, '3': 35,
-     '4': 35, '5': 35, '6': 35, '7': 35,
-     'L': 13, 'l': 13},
-    # 36
-    {'0': 36, '1': 36, 'L': 13, 'l': 13},
-    # 37
-    {'+': 42, '-': 42, '0': 43, '1': 43,
-     '2': 43, '3': 43, '4': 43, '5': 43,
-     '6': 43, '7': 43, '8': 43, '9': 43},
-    # 38
-    {'0': 39, '1': 39, '2': 39, '3': 39,
-     '4': 39, '5': 39, '6': 39, '7': 39,
-     '8': 39, '9': 39},
-    # 39
-    {'0': 39, '1': 39, '2': 39, '3': 39,
-     '4': 39, '5': 39, '6': 39, '7': 39,
-     '8': 39, '9': 39, 'J': 13, 'j': 13},
-    # 40
-    {automata.DEFAULT: 40, '\n': 27,
-     '\r': 27, "'": 13, '\\': 29},
-    # 41
-    {automata.DEFAULT: 41, '\n': 27,
-     '\r': 27, '"': 13, '\\': 32},
-    # 42
-    {'0': 43, '1': 43, '2': 43, '3': 43,
-     '4': 43, '5': 43, '6': 43, '7': 43,
-     '8': 43, '9': 43},
-    # 43
-    {'0': 43, '1': 43, '2': 43, '3': 43,
-     '4': 43, '5': 43, '6': 43, '7': 43,
-     '8': 43, '9': 43, 'J': 13, 'j': 13},
-    ]
-pseudoDFA = automata.DFA(states, accepts)
-
-accepts = [False, False, False, False, False, True]
-states = [
-    # 0
-    {automata.DEFAULT: 0, '"': 1, '\\': 2},
-    # 1
-    {automata.DEFAULT: 4, '"': 3, '\\': 2},
-    # 2
-    {automata.DEFAULT: 4},
-    # 3
-    {automata.DEFAULT: 4, '"': 5, '\\': 2},
-    # 4
-    {automata.DEFAULT: 4, '"': 1, '\\': 2},
-    # 5
-    {automata.DEFAULT: 4, '"': 5, '\\': 2},
-    ]
-double3DFA = automata.NonGreedyDFA(states, accepts)
-
-accepts = [False, False, False, False, False, True]
-states = [
-    # 0
-    {automata.DEFAULT: 0, "'": 1, '\\': 2},
-    # 1
-    {automata.DEFAULT: 4, "'": 3, '\\': 2},
-    # 2
-    {automata.DEFAULT: 4},
-    # 3
-    {automata.DEFAULT: 4, "'": 5, '\\': 2},
-    # 4
-    {automata.DEFAULT: 4, "'": 1, '\\': 2},
-    # 5
-    {automata.DEFAULT: 4, "'": 5, '\\': 2},
-    ]
-single3DFA = automata.NonGreedyDFA(states, accepts)
-
-accepts = [False, True, False, False]
-states = [
-    # 0
-    {automata.DEFAULT: 0, "'": 1, '\\': 2},
-    # 1
-    {},
-    # 2
-    {automata.DEFAULT: 3},
-    # 3
-    {automata.DEFAULT: 3, "'": 1, '\\': 2},
-    ]
-singleDFA = automata.DFA(states, accepts)
-
-accepts = [False, True, False, False]
-states = [
-    # 0
-    {automata.DEFAULT: 0, '"': 1, '\\': 2},
-    # 1
-    {},
-    # 2
-    {automata.DEFAULT: 3},
-    # 3
-    {automata.DEFAULT: 3, '"': 1, '\\': 2},
-    ]
-doubleDFA = automata.DFA(states, accepts)
-
-#_______________________________________________________________________
-# End of automatically generated DFA's
-
 endDFAs = {"'" : singleDFA,
            '"' : doubleDFA,
            'r' : None,
diff --git a/pypy/interpreter/pyparser/test/test_gendfa.py b/pypy/interpreter/pyparser/test/test_gendfa.py
--- a/pypy/interpreter/pyparser/test/test_gendfa.py
+++ b/pypy/interpreter/pyparser/test/test_gendfa.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.pyparser.automata import DFA, DEFAULT
-from pypy.interpreter.pyparser.genpytokenize import output
+from pypy.interpreter.pyparser.gendfa import output
 
 def test_states():
     states = [{"\x00": 1}, {"\x01": 0}]

From pypy.commits at gmail.com  Wed May 23 15:05:01 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 12:05:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5b05bb5d.1c69fb81.30369.8486@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94667:cfbe109033a4
Date: 2018-05-23 21:04 +0200
http://bitbucket.org/pypy/pypy/changeset/cfbe109033a4/

Log:	hg merge default


From pypy.commits at gmail.com  Wed May 23 16:47:31 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 13:47:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5-reverse-debugger: Close branch,
 ready to merge
Message-ID: <5b05d363.1c69fb81.153ca.06b7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5-reverse-debugger
Changeset: r94674:3e1574fb1516
Date: 2018-05-23 22:44 +0200
http://bitbucket.org/pypy/pypy/changeset/3e1574fb1516/

Log:	Close branch, ready to merge


From pypy.commits at gmail.com  Wed May 23 16:47:29 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 13:47:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge reverse-debugger
Message-ID: <5b05d361.1c69fb81.5fbd0.2345@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94673:c67b31661f36
Date: 2018-05-23 22:43 +0200
http://bitbucket.org/pypy/pypy/changeset/c67b31661f36/

Log:	hg merge reverse-debugger

diff too long, truncating to 2000 out of 7443 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,7 +33,12 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
+e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1
+a24d6c7000c8099c73d3660857f7e3cee5ac045c RevDB-pypy2.7-v5.6.2
 fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
 b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -194,5 +194,6 @@
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
 jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
+def_op('LOAD_REVDB_VAR', 205)         # reverse debugger (syntax example: $5)
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -57,6 +57,11 @@
     "termios", "_minimal_curses",
 ])
 
+reverse_debugger_disable_modules = set([
+    "_continuation", "_vmprof", "_multiprocessing",
+    "micronumpy",
+    ])
+
 # XXX this should move somewhere else, maybe to platform ("is this posixish"
 #     check or something)
 if sys.platform == "win32":
@@ -292,6 +297,9 @@
     modules = working_modules.copy()
     if config.translation.sandbox:
         modules = default_modules
+    if config.translation.reverse_debugger:
+        for mod in reverse_debugger_disable_modules:
+            setattr(config.objspace.usemodules, mod, False)
     # ignore names from 'essential_modules', notably 'exceptions', which
     # may not be present in config.objspace.usemodules at all
     modules = [name for name in modules if name not in essential_modules]
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -22,3 +22,8 @@
 .. branch: gc-more-logging
 
 Log additional gc-minor and gc-collect-step info in the PYPYLOG
+
+.. branch: reverse-debugger
+
+The reverse-debugger branch has been merged.  For more information, see
+https://bitbucket.org/pypy/revdb
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -83,11 +83,24 @@
     sys.excepthook(), catching SystemExit, printing a newline after
     sys.stdout if needed, etc.
     """
+    # don't use try:except: here, otherwise the exception remains
+    # visible in user code.  Make sure revdb_stop is a callable, so
+    # that we can call it immediately after finally: below.  Doing
+    # so minimizes the number of "blind" lines that we need to go
+    # back from, with "bstep", after we do "continue" in revdb.
+    if '__pypy__' in sys.builtin_module_names:
+        from __pypy__ import revdb_stop
+    else:
+        revdb_stop = None
+    if revdb_stop is None:
+        revdb_stop = lambda: None
+
     try:
         # run it
         try:
             f(*fargs, **fkwds)
         finally:
+            revdb_stop()
             sys.settrace(None)
             sys.setprofile(None)
 
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -673,6 +673,7 @@
     ops.JUMP_IF_NOT_DEBUG: 0,
 
     ops.BUILD_LIST_FROM_ARG: 1,
+    ops.LOAD_REVDB_VAR: 1,
 }
 
 
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1593,6 +1593,8 @@
             return Num.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Str):
             return Str.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_RevDBMetaVar):
+            return RevDBMetaVar.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Attribute):
             return Attribute.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Subscript):
@@ -2456,6 +2458,41 @@
 State.ast_type('Str', 'expr', ['s'])
 
 
+class RevDBMetaVar(expr):
+
+    def __init__(self, metavar, lineno, col_offset):
+        self.metavar = metavar
+        expr.__init__(self, lineno, col_offset)
+
+    def walkabout(self, visitor):
+        visitor.visit_RevDBMetaVar(self)
+
+    def mutate_over(self, visitor):
+        return visitor.visit_RevDBMetaVar(self)
+
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_RevDBMetaVar)
+        w_metavar = space.newint(self.metavar)  # int
+        space.setattr(w_node, space.newtext('metavar'), w_metavar)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
+        return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        w_metavar = get_field(space, w_node, 'metavar', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _metavar = space.int_w(w_metavar)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
+        return RevDBMetaVar(_metavar, _lineno, _col_offset)
+
+State.ast_type('RevDBMetaVar', 'expr', ['metavar'])
+
+
 class Attribute(expr):
 
     def __init__(self, value, attr, ctx, lineno, col_offset):
@@ -3588,6 +3625,8 @@
         return self.default_visitor(node)
     def visit_Str(self, node):
         return self.default_visitor(node)
+    def visit_RevDBMetaVar(self, node):
+        return self.default_visitor(node)
     def visit_Attribute(self, node):
         return self.default_visitor(node)
     def visit_Subscript(self, node):
@@ -3804,6 +3843,9 @@
     def visit_Str(self, node):
         pass
 
+    def visit_RevDBMetaVar(self, node):
+        pass
+
     def visit_Attribute(self, node):
         node.value.walkabout(self)
 
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1165,6 +1165,11 @@
         elif first_child_type == tokens.BACKQUOTE:
             expr = self.handle_testlist(atom_node.get_child(1))
             return ast.Repr(expr, atom_node.get_lineno(), atom_node.get_column())
+        elif first_child_type == tokens.REVDBMETAVAR:
+            string = atom_node.get_child(0).get_value()
+            return ast.RevDBMetaVar(int(string[1:]),
+                                    atom_node.get_lineno(),
+                                    atom_node.get_column())
         else:
             raise AssertionError("unknown atom")
 
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1203,6 +1203,20 @@
             sub.value.walkabout(self)
         self._compile_slice(sub.slice, sub.ctx)
 
+    def _revdb_metavar(self, node):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import dbstate
+        if not dbstate.standard_code:
+            self.emit_op_arg(ops.LOAD_REVDB_VAR, node.metavar)
+            return True
+        return False
+
+    def visit_RevDBMetaVar(self, node):
+        if self.space.reverse_debugging and self._revdb_metavar(node):
+            return
+        self.error("Unknown character ('$NUM' is only valid in the "
+                   "reverse-debugger)", node)
+
 
 class TopLevelCodeGenerator(PythonCodeGenerator):
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -953,6 +953,20 @@
         yield (self.st, "x=(lambda: (-0.0, 0.0), lambda: (0.0, -0.0))[1]()",
                         'repr(x)', '(0.0, -0.0)')
 
+    def test_revdb_metavar(self):
+        from pypy.interpreter.reverse_debugging import dbstate, setup_revdb
+        self.space.config.translation.reverse_debugger = True
+        self.space.reverse_debugging = True
+        try:
+            setup_revdb(self.space)
+            dbstate.standard_code = False
+            dbstate.metavars = [self.space.wrap(6)]
+            self.simple_test("x = 7*$0", "x", 42)
+            dbstate.standard_code = True
+            self.error_test("x = 7*$0", SyntaxError)
+        finally:
+            self.space.reverse_debugging = False
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -71,6 +71,7 @@
 	     | Repr(expr value)
 	     | Num(object n) -- a number as a PyObject.
 	     | Str(string s) -- need to specify raw, unicode, etc?
+             | RevDBMetaVar(int metavar)
 	     -- other literals? bools?
 
 	     -- the following expression can appear in assignment context
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -430,6 +430,8 @@
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
 
+    reverse_debugging = False
+
     @not_rpython
     def __init__(self, config=None):
         "Basic initialization of objects."
@@ -441,6 +443,7 @@
             from pypy.config.pypyoption import get_pypy_config
             config = get_pypy_config(translating=False)
         self.config = config
+        self.reverse_debugging = config.translation.reverse_debugger
 
         self.builtin_modules = {}
         self.reloading_modules = {}
@@ -458,6 +461,9 @@
 
     def startup(self):
         # To be called before using the space
+        if self.reverse_debugging:
+            self._revdb_startup()
+
         self.threadlocals.enter_thread(self)
 
         # Initialize already imported builtin modules
@@ -868,7 +874,8 @@
         w_s1 = self.interned_strings.get(s)
         if w_s1 is None:
             w_s1 = w_s
-            self.interned_strings.set(s, w_s1)
+            if self._side_effects_ok():
+                self.interned_strings.set(s, w_s1)
         return w_s1
 
     def new_interned_str(self, s):
@@ -878,9 +885,39 @@
         w_s1 = self.interned_strings.get(s)
         if w_s1 is None:
             w_s1 = self.newtext(s)
-            self.interned_strings.set(s, w_s1)
+            if self._side_effects_ok():
+                self.interned_strings.set(s, w_s1)
         return w_s1
 
+    def _revdb_startup(self):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import setup_revdb
+        setup_revdb(self)
+
+    def _revdb_standard_code(self):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import dbstate
+        return dbstate.standard_code
+
+    def _side_effects_ok(self):
+        # For the reverse debugger: we run compiled watchpoint
+        # expressions in a fast way that will crash if they have
+        # side-effects.  The obvious Python code with side-effects is
+        # documented "don't do that"; but some non-obvious side
+        # effects are also common, like interning strings (from
+        # unmarshalling the code object containing the watchpoint
+        # expression) to the two attribute caches in mapdict.py and
+        # typeobject.py.  For now, we have to identify such places
+        # that are not acceptable for "reasonable" read-only
+        # watchpoint expressions, and write:
+        #
+        #     if not space._side_effects_ok():
+        #         don't cache.
+        #
+        if self.reverse_debugging:
+            return self._revdb_standard_code()
+        return True
+
     def is_interned_str(self, s):
         # interface for marshal_impl
         if not we_are_translated():
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -64,6 +64,8 @@
         return frame
 
     def enter(self, frame):
+        if self.space.reverse_debugging:
+            self._revdb_enter(frame)
         frame.f_backref = self.topframeref
         self.topframeref = jit.virtual_ref(frame)
 
@@ -84,6 +86,8 @@
                 # be accessed also later
                 frame_vref()
             jit.virtual_ref_finish(frame_vref, frame)
+            if self.space.reverse_debugging:
+                self._revdb_leave(got_exception)
 
     # ________________________________________________________________
 
@@ -153,6 +157,8 @@
         Like bytecode_trace() but doesn't invoke any other events besides the
         trace function.
         """
+        if self.space.reverse_debugging:
+            self._revdb_potential_stop_point(frame)
         if (frame.get_w_f_trace() is None or self.is_tracing or
             self.gettrace() is None):
             return
@@ -385,6 +391,21 @@
         if self.space.check_signal_action is not None:
             self.space.check_signal_action.perform(self, None)
 
+    def _revdb_enter(self, frame):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import enter_call
+        enter_call(self.topframeref(), frame)
+
+    def _revdb_leave(self, got_exception):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import leave_call
+        leave_call(self.topframeref(), got_exception)
+
+    def _revdb_potential_stop_point(self, frame):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import potential_stop_point
+        potential_stop_point(frame)
+
     def _freeze_(self):
         raise Exception("ExecutionContext instances should not be seen during"
                         " translation.  Now is a good time to inspect the"
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -832,9 +832,11 @@
     def fget_f_builtins(self, space):
         return self.get_builtin().getdict(space)
 
+    def get_f_back(self):
+        return ExecutionContext.getnextframe_nohidden(self)
+
     def fget_f_back(self, space):
-        f_back = ExecutionContext.getnextframe_nohidden(self)
-        return f_back
+        return self.get_f_back()
 
     def fget_f_lasti(self, space):
         return self.space.newint(self.last_instr)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -437,6 +437,8 @@
                 self.WITH_CLEANUP(oparg, next_instr)
             elif opcode == opcodedesc.YIELD_VALUE.index:
                 self.YIELD_VALUE(oparg, next_instr)
+            elif opcode == opcodedesc.LOAD_REVDB_VAR.index:
+                self.LOAD_REVDB_VAR(oparg, next_instr)
             else:
                 self.MISSING_OPCODE(oparg, next_instr)
 
@@ -1047,9 +1049,16 @@
     def YIELD_VALUE(self, oparg, next_instr):
         raise Yield
 
+    def _revdb_jump_backward(self, jumpto):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import jump_backward
+        jump_backward(self, jumpto)
+
     def jump_absolute(self, jumpto, ec):
         # this function is overridden by pypy.module.pypyjit.interp_jit
         check_nonneg(jumpto)
+        if self.space.reverse_debugging:
+            self._revdb_jump_backward(jumpto)
         return jumpto
 
     def JUMP_FORWARD(self, jumpby, next_instr):
@@ -1303,6 +1312,18 @@
         w_dict = self.peekvalue()
         self.space.setitem(w_dict, w_key, w_value)
 
+    def _revdb_load_var(self, oparg):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import load_metavar
+        w_var = load_metavar(oparg)
+        self.pushvalue(w_var)
+
+    def LOAD_REVDB_VAR(self, oparg, next_instr):
+        if self.space.reverse_debugging:
+            self._revdb_load_var(oparg)
+        else:
+            self.MISSING_OPCODE(oparg, next_instr)
+
 
 ### ____________________________________________________________ ###
 
diff --git a/pypy/interpreter/pyparser/data/Grammar2.7 b/pypy/interpreter/pyparser/data/Grammar2.7
--- a/pypy/interpreter/pyparser/data/Grammar2.7
+++ b/pypy/interpreter/pyparser/data/Grammar2.7
@@ -104,7 +104,7 @@
        '[' [listmaker] ']' |
        '{' [dictorsetmaker] '}' |
        '`' testlist1 '`' |
-       NAME | NUMBER | STRING+)
+       NAME | NUMBER | STRING+ | '$NUM')
 listmaker: test ( list_for | (',' test)* [','] )
 testlist_comp: test ( comp_for | (',' test)* [','] )
 lambdef: 'lambda' [varargslist] ':' test
diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py
--- a/pypy/interpreter/pyparser/dfa_generated.py
+++ b/pypy/interpreter/pyparser/dfa_generated.py
@@ -6,37 +6,38 @@
 from pypy.interpreter.pyparser import automata
 accepts = [True, True, True, True, True, True, True, True,
            True, True, False, True, True, True, True, False,
-           False, False, True, False, False, True, False,
-           False, True, False, True, False, True, False,
-           False, True, False, False, True, True, True,
-           False, False, True, False, False, False, True]
+           False, False, False, True, False, False, True,
+           False, False, True, False, True, True, False,
+           True, False, False, True, False, False, True,
+           True, True, False, False, True, False, False,
+           False, True]
 states = [
     # 0
     {'\t': 0, '\n': 13, '\x0c': 0,
-     '\r': 14, ' ': 0, '!': 10, '"': 16,
-     '#': 18, '%': 12, '&': 12, "'": 15,
-     '(': 13, ')': 13, '*': 7, '+': 12,
-     ',': 13, '-': 12, '.': 6, '/': 11,
-     '0': 4, '1': 5, '2': 5, '3': 5,
-     '4': 5, '5': 5, '6': 5, '7': 5,
-     '8': 5, '9': 5, ':': 13, ';': 13,
-     '<': 9, '=': 12, '>': 8, '@': 13,
-     'A': 1, 'B': 2, 'C': 1, 'D': 1,
-     'E': 1, 'F': 1, 'G': 1, 'H': 1,
-     'I': 1, 'J': 1, 'K': 1, 'L': 1,
-     'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
-     'U': 2, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '[': 13, '\\': 17,
-     ']': 13, '^': 12, '_': 1, '`': 13,
-     'a': 1, 'b': 2, 'c': 1, 'd': 1,
-     'e': 1, 'f': 1, 'g': 1, 'h': 1,
-     'i': 1, 'j': 1, 'k': 1, 'l': 1,
-     'm': 1, 'n': 1, 'o': 1, 'p': 1,
-     'q': 1, 'r': 3, 's': 1, 't': 1,
-     'u': 2, 'v': 1, 'w': 1, 'x': 1,
-     'y': 1, 'z': 1, '{': 13, '|': 12,
-     '}': 13, '~': 13},
+     '\r': 14, ' ': 0, '!': 10, '"': 17,
+     '#': 19, '$': 15, '%': 12, '&': 12,
+     "'": 16, '(': 13, ')': 13, '*': 7,
+     '+': 12, ',': 13, '-': 12, '.': 6,
+     '/': 11, '0': 4, '1': 5, '2': 5,
+     '3': 5, '4': 5, '5': 5, '6': 5,
+     '7': 5, '8': 5, '9': 5, ':': 13,
+     ';': 13, '<': 9, '=': 12, '>': 8,
+     '@': 13, 'A': 1, 'B': 2, 'C': 1,
+     'D': 1, 'E': 1, 'F': 1, 'G': 1,
+     'H': 1, 'I': 1, 'J': 1, 'K': 1,
+     'L': 1, 'M': 1, 'N': 1, 'O': 1,
+     'P': 1, 'Q': 1, 'R': 3, 'S': 1,
+     'T': 1, 'U': 2, 'V': 1, 'W': 1,
+     'X': 1, 'Y': 1, 'Z': 1, '[': 13,
+     '\\': 18, ']': 13, '^': 12, '_': 1,
+     '`': 13, 'a': 1, 'b': 2, 'c': 1,
+     'd': 1, 'e': 1, 'f': 1, 'g': 1,
+     'h': 1, 'i': 1, 'j': 1, 'k': 1,
+     'l': 1, 'm': 1, 'n': 1, 'o': 1,
+     'p': 1, 'q': 1, 'r': 3, 's': 1,
+     't': 1, 'u': 2, 'v': 1, 'w': 1,
+     'x': 1, 'y': 1, 'z': 1, '{': 13,
+     '|': 12, '}': 13, '~': 13},
     # 1
     {'0': 1, '1': 1, '2': 1, '3': 1,
      '4': 1, '5': 1, '6': 1, '7': 1,
@@ -55,7 +56,7 @@
      't': 1, 'u': 1, 'v': 1, 'w': 1,
      'x': 1, 'y': 1, 'z': 1},
     # 2
-    {'"': 16, "'": 15, '0': 1, '1': 1,
+    {'"': 17, "'": 16, '0': 1, '1': 1,
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 1, 'C': 1, 'D': 1,
@@ -73,7 +74,7 @@
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1},
     # 3
-    {'"': 16, "'": 15, '0': 1, '1': 1,
+    {'"': 17, "'": 16, '0': 1, '1': 1,
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 1, 'C': 1, 'D': 1,
@@ -91,22 +92,22 @@
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1},
     # 4
-    {'.': 24, '0': 21, '1': 21, '2': 21,
-     '3': 21, '4': 21, '5': 21, '6': 21,
-     '7': 21, '8': 23, '9': 23, 'B': 22,
-     'E': 25, 'J': 13, 'L': 13, 'O': 20,
-     'X': 19, 'b': 22, 'e': 25, 'j': 13,
-     'l': 13, 'o': 20, 'x': 19},
+    {'.': 25, '0': 22, '1': 22, '2': 22,
+     '3': 22, '4': 22, '5': 22, '6': 22,
+     '7': 22, '8': 24, '9': 24, 'B': 23,
+     'E': 26, 'J': 13, 'L': 13, 'O': 21,
+     'X': 20, 'b': 23, 'e': 26, 'j': 13,
+     'l': 13, 'o': 21, 'x': 20},
     # 5
-    {'.': 24, '0': 5, '1': 5, '2': 5,
+    {'.': 25, '0': 5, '1': 5, '2': 5,
      '3': 5, '4': 5, '5': 5, '6': 5,
-     '7': 5, '8': 5, '9': 5, 'E': 25,
-     'J': 13, 'L': 13, 'e': 25, 'j': 13,
+     '7': 5, '8': 5, '9': 5, 'E': 26,
+     'J': 13, 'L': 13, 'e': 26, 'j': 13,
      'l': 13},
     # 6
-    {'0': 26, '1': 26, '2': 26, '3': 26,
-     '4': 26, '5': 26, '6': 26, '7': 26,
-     '8': 26, '9': 26},
+    {'0': 27, '1': 27, '2': 27, '3': 27,
+     '4': 27, '5': 27, '6': 27, '7': 27,
+     '8': 27, '9': 27},
     # 7
     {'*': 12, '=': 13},
     # 8
@@ -124,107 +125,115 @@
     # 14
     {'\n': 13},
     # 15
-    {automata.DEFAULT: 30, '\n': 27,
-     '\r': 27, "'": 28, '\\': 29},
+    {'0': 28, '1': 28, '2': 28, '3': 28,
+     '4': 28, '5': 28, '6': 28, '7': 28,
+     '8': 28, '9': 28},
     # 16
-    {automata.DEFAULT: 33, '\n': 27,
-     '\r': 27, '"': 31, '\\': 32},
+    {automata.DEFAULT: 32, '\n': 29,
+     '\r': 29, "'": 30, '\\': 31},
     # 17
+    {automata.DEFAULT: 35, '\n': 29,
+     '\r': 29, '"': 33, '\\': 34},
+    # 18
     {'\n': 13, '\r': 14},
-    # 18
-    {automata.DEFAULT: 18, '\n': 27, '\r': 27},
     # 19
-    {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34,
-     '8': 34, '9': 34, 'A': 34, 'B': 34,
-     'C': 34, 'D': 34, 'E': 34, 'F': 34,
-     'a': 34, 'b': 34, 'c': 34, 'd': 34,
-     'e': 34, 'f': 34},
+    {automata.DEFAULT: 19, '\n': 29, '\r': 29},
     # 20
-    {'0': 35, '1': 35, '2': 35, '3': 35,
-     '4': 35, '5': 35, '6': 35, '7': 35},
+    {'0': 36, '1': 36, '2': 36, '3': 36,
+     '4': 36, '5': 36, '6': 36, '7': 36,
+     '8': 36, '9': 36, 'A': 36, 'B': 36,
+     'C': 36, 'D': 36, 'E': 36, 'F': 36,
+     'a': 36, 'b': 36, 'c': 36, 'd': 36,
+     'e': 36, 'f': 36},
     # 21
-    {'.': 24, '0': 21, '1': 21, '2': 21,
-     '3': 21, '4': 21, '5': 21, '6': 21,
-     '7': 21, '8': 23, '9': 23, 'E': 25,
-     'J': 13, 'L': 13, 'e': 25, 'j': 13,
+    {'0': 37, '1': 37, '2': 37, '3': 37,
+     '4': 37, '5': 37, '6': 37, '7': 37},
+    # 22
+    {'.': 25, '0': 22, '1': 22, '2': 22,
+     '3': 22, '4': 22, '5': 22, '6': 22,
+     '7': 22, '8': 24, '9': 24, 'E': 26,
+     'J': 13, 'L': 13, 'e': 26, 'j': 13,
      'l': 13},
-    # 22
-    {'0': 36, '1': 36},
     # 23
-    {'.': 24, '0': 23, '1': 23, '2': 23,
-     '3': 23, '4': 23, '5': 23, '6': 23,
-     '7': 23, '8': 23, '9': 23, 'E': 25,
-     'J': 13, 'e': 25, 'j': 13},
+    {'0': 38, '1': 38},
     # 24
-    {'0': 24, '1': 24, '2': 24, '3': 24,
-     '4': 24, '5': 24, '6': 24, '7': 24,
-     '8': 24, '9': 24, 'E': 37, 'J': 13,
-     'e': 37, 'j': 13},
+    {'.': 25, '0': 24, '1': 24, '2': 24,
+     '3': 24, '4': 24, '5': 24, '6': 24,
+     '7': 24, '8': 24, '9': 24, 'E': 26,
+     'J': 13, 'e': 26, 'j': 13},
     # 25
-    {'+': 38, '-': 38, '0': 39, '1': 39,
-     '2': 39, '3': 39, '4': 39, '5': 39,
-     '6': 39, '7': 39, '8': 39, '9': 39},
+    {'0': 25, '1': 25, '2': 25, '3': 25,
+     '4': 25, '5': 25, '6': 25, '7': 25,
+     '8': 25, '9': 25, 'E': 39, 'J': 13,
+     'e': 39, 'j': 13},
     # 26
-    {'0': 26, '1': 26, '2': 26, '3': 26,
-     '4': 26, '5': 26, '6': 26, '7': 26,
-     '8': 26, '9': 26, 'E': 37, 'J': 13,
-     'e': 37, 'j': 13},
+    {'+': 40, '-': 40, '0': 41, '1': 41,
+     '2': 41, '3': 41, '4': 41, '5': 41,
+     '6': 41, '7': 41, '8': 41, '9': 41},
     # 27
+    {'0': 27, '1': 27, '2': 27, '3': 27,
+     '4': 27, '5': 27, '6': 27, '7': 27,
+     '8': 27, '9': 27, 'E': 39, 'J': 13,
+     'e': 39, 'j': 13},
+    # 28
+    {'0': 28, '1': 28, '2': 28, '3': 28,
+     '4': 28, '5': 28, '6': 28, '7': 28,
+     '8': 28, '9': 28},
+    # 29
     {},
-    # 28
+    # 30
     {"'": 13},
-    # 29
-    {automata.DEFAULT: 40, '\n': 13, '\r': 14},
-    # 30
-    {automata.DEFAULT: 30, '\n': 27,
-     '\r': 27, "'": 13, '\\': 29},
     # 31
+    {automata.DEFAULT: 42, '\n': 13, '\r': 14},
+    # 32
+    {automata.DEFAULT: 32, '\n': 29,
+     '\r': 29, "'": 13, '\\': 31},
+    # 33
     {'"': 13},
-    # 32
-    {automata.DEFAULT: 41, '\n': 13, '\r': 14},
-    # 33
-    {automata.DEFAULT: 33, '\n': 27,
-     '\r': 27, '"': 13, '\\': 32},
     # 34
-    {'0': 34, '1': 34, '2': 34, '3': 34,
-     '4': 34, '5': 34, '6': 34, '7': 34,
-     '8': 34, '9': 34, 'A': 34, 'B': 34,
-     'C': 34, 'D': 34, 'E': 34, 'F': 34,
-     'L': 13, 'a': 34, 'b': 34, 'c': 34,
-     'd': 34, 'e': 34, 'f': 34, 'l': 13},
+    {automata.DEFAULT: 43, '\n': 13, '\r': 14},
     # 35
-    {'0': 35, '1': 35, '2': 35, '3': 35,
-     '4': 35, '5': 35, '6': 35, '7': 35,
+    {automata.DEFAULT: 35, '\n': 29,
+     '\r': 29, '"': 13, '\\': 34},
+    # 36
+    {'0': 36, '1': 36, '2': 36, '3': 36,
+     '4': 36, '5': 36, '6': 36, '7': 36,
+     '8': 36, '9': 36, 'A': 36, 'B': 36,
+     'C': 36, 'D': 36, 'E': 36, 'F': 36,
+     'L': 13, 'a': 36, 'b': 36, 'c': 36,
+     'd': 36, 'e': 36, 'f': 36, 'l': 13},
+    # 37
+    {'0': 37, '1': 37, '2': 37, '3': 37,
+     '4': 37, '5': 37, '6': 37, '7': 37,
      'L': 13, 'l': 13},
-    # 36
-    {'0': 36, '1': 36, 'L': 13, 'l': 13},
-    # 37
-    {'+': 42, '-': 42, '0': 43, '1': 43,
-     '2': 43, '3': 43, '4': 43, '5': 43,
-     '6': 43, '7': 43, '8': 43, '9': 43},
     # 38
-    {'0': 39, '1': 39, '2': 39, '3': 39,
-     '4': 39, '5': 39, '6': 39, '7': 39,
-     '8': 39, '9': 39},
+    {'0': 38, '1': 38, 'L': 13, 'l': 13},
     # 39
-    {'0': 39, '1': 39, '2': 39, '3': 39,
-     '4': 39, '5': 39, '6': 39, '7': 39,
-     '8': 39, '9': 39, 'J': 13, 'j': 13},
+    {'+': 44, '-': 44, '0': 45, '1': 45,
+     '2': 45, '3': 45, '4': 45, '5': 45,
+     '6': 45, '7': 45, '8': 45, '9': 45},
     # 40
-    {automata.DEFAULT: 40, '\n': 27,
-     '\r': 27, "'": 13, '\\': 29},
+    {'0': 41, '1': 41, '2': 41, '3': 41,
+     '4': 41, '5': 41, '6': 41, '7': 41,
+     '8': 41, '9': 41},
     # 41
-    {automata.DEFAULT: 41, '\n': 27,
-     '\r': 27, '"': 13, '\\': 32},
+    {'0': 41, '1': 41, '2': 41, '3': 41,
+     '4': 41, '5': 41, '6': 41, '7': 41,
+     '8': 41, '9': 41, 'J': 13, 'j': 13},
     # 42
-    {'0': 43, '1': 43, '2': 43, '3': 43,
-     '4': 43, '5': 43, '6': 43, '7': 43,
-     '8': 43, '9': 43},
+    {automata.DEFAULT: 42, '\n': 29,
+     '\r': 29, "'": 13, '\\': 31},
     # 43
-    {'0': 43, '1': 43, '2': 43, '3': 43,
-     '4': 43, '5': 43, '6': 43, '7': 43,
-     '8': 43, '9': 43, 'J': 13, 'j': 13},
+    {automata.DEFAULT: 43, '\n': 29,
+     '\r': 29, '"': 13, '\\': 34},
+    # 44
+    {'0': 45, '1': 45, '2': 45, '3': 45,
+     '4': 45, '5': 45, '6': 45, '7': 45,
+     '8': 45, '9': 45},
+    # 45
+    {'0': 45, '1': 45, '2': 45, '3': 45,
+     '4': 45, '5': 45, '6': 45, '7': 45,
+     '8': 45, '9': 45, 'J': 13, 'j': 13},
     ]
 pseudoDFA = automata.DFA(states, accepts)
 
diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py
old mode 100755
new mode 100644
--- a/pypy/interpreter/pyparser/gendfa.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -144,7 +144,10 @@
     special = group(states,
                     makeEOL(),
                     groupStr(states, "@:;.,`"))
-    funny = group(states, operator, bracket, special)
+    revdb_metavar = chain(states,
+                          groupStr(states, "$"),
+                          atleastonce(states, makeDigits()))
+    funny = group(states, operator, bracket, special, revdb_metavar)
     # ____________________________________________________________
     def makeStrPrefix ():
         return chain(states,
diff --git a/pypy/interpreter/pyparser/pytoken.py b/pypy/interpreter/pyparser/pytoken.py
--- a/pypy/interpreter/pyparser/pytoken.py
+++ b/pypy/interpreter/pyparser/pytoken.py
@@ -67,5 +67,6 @@
 # extra PyPy-specific tokens
 _add_tok("COMMENT")
 _add_tok("NL")
+_add_tok("REVDBMETAVAR", "$NUM")
 
 del _add_tok
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -220,6 +220,10 @@
                     last_comment = ''
                 elif initial == '\\':                      # continued stmt
                     continued = 1
+                elif initial == '$':
+                    token_list.append((tokens.REVDBMETAVAR, token,
+                                       lnum, start, line))
+                    last_comment = ''
                 else:
                     if initial in '([{':
                         parenstack.append((initial, lnum, start, line))
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -167,6 +167,15 @@
             tree = self.parse(fmt % linefeed)
             assert expected_tree == tree
 
+    def test_revdb_dollar_num(self):
+        self.parse('$0')
+        self.parse('$5')
+        self.parse('$42')
+        self.parse('2+$42.attrname')
+        py.test.raises(SyntaxError, self.parse, '$')
+        py.test.raises(SyntaxError, self.parse, '$a')
+        py.test.raises(SyntaxError, self.parse, '$.5')
+
     def test_error_forgotten_chars(self):
         info = py.test.raises(SyntaxError, self.parse, "if 1\n    print 4")
         assert "(expected ':')" in info.value.msg
diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/reverse_debugging.py
@@ -0,0 +1,852 @@
+import sys
+from rpython.rlib import revdb
+from rpython.rlib.debug import make_sure_not_resized
+from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter import gateway, typedef, pycode, pytraceback, pyframe
+from pypy.module.marshal import interp_marshal
+from pypy.interpreter.executioncontext import AbstractActionFlag, ActionFlag
+
+
+class DBState:
+    standard_code = True
+    breakpoint_stack_id = 0
+    breakpoint_funcnames = None
+    breakpoint_filelines = None
+    breakpoint_by_file = None
+    breakpoint_version = 0
+    printed_objects = {}
+    metavars = []
+    watch_progs = []
+    watch_futures = {}
+
+dbstate = DBState()
+
+
+pycode.PyCode.co_revdb_linestarts = None   # or a string: see below
+pycode.PyCode.co_revdb_bkpt_version = 0    # see check_and_trigger_bkpt()
+pycode.PyCode.co_revdb_bkpt_cache = None   # see check_and_trigger_bkpt()
+
+# invariant: "f_revdb_nextline_instr" is the bytecode offset of
+# the start of the line that follows "last_instr".
+pyframe.PyFrame.f_revdb_nextline_instr = -1
+
+
+# ____________________________________________________________
+
+
+def setup_revdb(space):
+    """Called at run-time, before the space is set up.
+
+    The various register_debug_command() lines attach functions
+    to some commands that 'revdb.py' can call, if we are running
+    in replay mode.
+    """
+    assert space.config.translation.reverse_debugger
+    dbstate.space = space
+
+    make_sure_not_resized(dbstate.watch_progs)
+    make_sure_not_resized(dbstate.metavars)
+
+    revdb.register_debug_command(revdb.CMD_PRINT, lambda_print)
+    revdb.register_debug_command(revdb.CMD_BACKTRACE, lambda_backtrace)
+    revdb.register_debug_command(revdb.CMD_LOCALS, lambda_locals)
+    revdb.register_debug_command(revdb.CMD_BREAKPOINTS, lambda_breakpoints)
+    revdb.register_debug_command(revdb.CMD_STACKID, lambda_stackid)
+    revdb.register_debug_command("ALLOCATING", lambda_allocating)
+    revdb.register_debug_command(revdb.CMD_ATTACHID, lambda_attachid)
+    revdb.register_debug_command(revdb.CMD_COMPILEWATCH, lambda_compilewatch)
+    revdb.register_debug_command(revdb.CMD_CHECKWATCH, lambda_checkwatch)
+    revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues)
+
+
+# ____________________________________________________________
+
+
+def enter_call(caller_frame, callee_frame):
+    if dbstate.breakpoint_funcnames is not None:
+        name = callee_frame.getcode().co_name
+        if name in dbstate.breakpoint_funcnames:
+            revdb.breakpoint(dbstate.breakpoint_funcnames[name])
+    if dbstate.breakpoint_stack_id != 0 and caller_frame is not None:
+        if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame):
+            revdb.breakpoint(-1)
+    #
+    code = callee_frame.pycode
+    if code.co_revdb_linestarts is None:
+        build_co_revdb_linestarts(code)
+
+def leave_call(caller_frame, got_exception):
+    if dbstate.breakpoint_stack_id != 0 and caller_frame is not None:
+        if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame):
+            revdb.breakpoint(-2)
+    if we_are_translated():
+        stop_point_activate(-2 + got_exception)
+
+def stop_point():
+    if we_are_translated():
+        revdb.breakpoint(-3)
+
+
+def jump_backward(frame, jumpto):
+    # When we see a jump backward, we set 'f_revdb_nextline_instr' in
+    # such a way that the next instruction, at 'jumpto', will trigger
+    # stop_point_activate().  We have to trigger it even if
+    # 'jumpto' is not actually a start of line.  For example, after a
+    # 'while foo:', the body ends with a JUMP_ABSOLUTE which
+    # jumps back to the *second* opcode of the while.
+    frame.f_revdb_nextline_instr = jumpto
+
+
+def potential_stop_point(frame):
+    if not we_are_translated():
+        return
+    #
+    # We only record a stop_point at every line, not every bytecode.
+    # Uses roughly the same algo as ExecutionContext.run_trace_func()
+    # to know where the line starts are, but tweaked for speed,
+    # avoiding the quadratic complexity when run N times with a large
+    # code object.
+    #
+    cur = frame.last_instr
+    if cur < frame.f_revdb_nextline_instr:
+        return    # fast path: we're still inside the same line as before
+    #
+    call_stop_point_at_line = True
+    co_revdb_linestarts = frame.pycode.co_revdb_linestarts
+    if cur > frame.f_revdb_nextline_instr:
+        #
+        # We jumped forward over the start of the next line.  We're
+        # inside a different line, but we will only trigger a stop
+        # point if we're at the starting bytecode of that line.  Fetch
+        # from co_revdb_linestarts the start of the line that is at or
+        # follows 'cur'.
+        ch = ord(co_revdb_linestarts[cur])
+        if ch == 0:
+            pass   # we are at the start of a line now
+        else:
+            # We are not, so don't call stop_point_activate().
+            # We still have to fill f_revdb_nextline_instr.
+            call_stop_point_at_line = False
+    #
+    if call_stop_point_at_line:
+        stop_point_activate(pycode=frame.pycode, opindex=cur)
+        cur += 1
+        ch = ord(co_revdb_linestarts[cur])
+    #
+    # Update f_revdb_nextline_instr.  Check if 'ch' was greater than
+    # 255, in which case it was rounded down to 255 and we have to
+    # continue looking
+    nextline_instr = cur + ch
+    while ch == 255:
+        ch = ord(co_revdb_linestarts[nextline_instr])
+        nextline_instr += ch
+    frame.f_revdb_nextline_instr = nextline_instr
+
+
+def build_co_revdb_linestarts(code):
+    # Inspired by findlinestarts() in the 'dis' standard module.
+    # Set up 'bits' so that it contains \x00 at line starts and \xff
+    # in-between.
+    bits = ['\xff'] * (len(code.co_code) + 1)
+    if not code.hidden_applevel:
+        lnotab = code.co_lnotab
+        addr = 0
+        p = 0
+        newline = 1
+        while p + 1 < len(lnotab):
+            byte_incr = ord(lnotab[p])
+            line_incr = ord(lnotab[p+1])
+            if byte_incr:
+                if newline != 0:
+                    bits[addr] = '\x00'
+                    newline = 0
+                addr += byte_incr
+            newline |= line_incr
+            p += 2
+        if newline:
+            bits[addr] = '\x00'
+    bits[len(code.co_code)] = '\x00'
+    #
+    # Change 'bits' so that the character at 'i', if not \x00, measures
+    # how far the next \x00 is
+    next_null = len(code.co_code)
+    p = next_null - 1
+    while p >= 0:
+        if bits[p] == '\x00':
+            next_null = p
+        else:
+            ch = next_null - p
+            if ch > 255: ch = 255
+            bits[p] = chr(ch)
+        p -= 1
+    lstart = ''.join(bits)
+    code.co_revdb_linestarts = lstart
+    return lstart
+
+def get_final_lineno(code):
+    lineno = code.co_firstlineno
+    lnotab = code.co_lnotab
+    p = 1
+    while p < len(lnotab):
+        line_incr = ord(lnotab[p])
+        lineno += line_incr
+        p += 2
+    return lineno
+
+def find_line_starts(code):
+    # RPython version of dis.findlinestarts()
+    lnotab = code.co_lnotab
+    lastlineno = -1
+    lineno = code.co_firstlineno
+    addr = 0
+    p = 0
+    result = []
+    while p < len(lnotab) - 1:
+        byte_incr = ord(lnotab[p + 0])
+        line_incr = ord(lnotab[p + 1])
+        if byte_incr:
+            if lineno != lastlineno:
+                result.append((addr, lineno))
+                lastlineno = lineno
+            addr += byte_incr
+        lineno += line_incr
+        p += 2
+    if lineno != lastlineno:
+        result.append((addr, lineno))
+    return result
+
+class NonStandardCode(object):
+    def __enter__(self):
+        dbstate.standard_code = False
+        self.t = dbstate.space.actionflag._ticker
+        self.c = dbstate.space.actionflag._ticker_revdb_count
+    def __exit__(self, *args):
+        dbstate.space.actionflag._ticker = self.t
+        dbstate.space.actionflag._ticker_revdb_count = self.c
+        dbstate.standard_code = True
+non_standard_code = NonStandardCode()
+
+
+def stop_point_activate(place=0, pycode=None, opindex=-1):
+    if revdb.watch_save_state():
+        any_watch_point = False
+        # ^^ this flag is set to True if we must continue to enter this
+        # block of code.  If it is still False for watch_restore_state()
+        # below, then future watch_save_state() will return False too---
+        # until the next time revdb.c:set_revdb_breakpoints() is called.
+        space = dbstate.space
+        with non_standard_code:
+            watch_id = -1
+            if dbstate.breakpoint_by_file is not None:
+                any_watch_point = True
+                if pycode is not None:
+                    watch_id = check_and_trigger_bkpt(pycode, opindex)
+            if watch_id == -1:
+                for prog, watch_id, expected in dbstate.watch_progs:
+                    any_watch_point = True
+                    try:
+                        got = _run_watch(space, prog)
+                    except OperationError as e:
+                        got = e.errorstr(space)
+                    except Exception:
+                        break
+                    if got != expected:
+                        break
+                else:
+                    watch_id = -1
+        revdb.watch_restore_state(any_watch_point)
+        if watch_id != -1:
+            revdb.breakpoint(watch_id)
+    revdb.stop_point(place)
+
+
+def future_object(space):
+    return space.w_Ellipsis    # a random prebuilt object
+
+def load_metavar(index):
+    assert index >= 0
+    space = dbstate.space
+    metavars = dbstate.metavars
+    w_var = metavars[index] if index < len(metavars) else None
+    if w_var is None:
+        raise oefmt(space.w_NameError, "no constant object '$%d'",
+                    index)
+    if w_var is future_object(space):
+        raise oefmt(space.w_RuntimeError,
+                    "'$%d' refers to an object created later in time",
+                    index)
+    return w_var
+
+def set_metavar(index, w_obj):
+    assert index >= 0
+    if index >= len(dbstate.metavars):
+        missing = index + 1 - len(dbstate.metavars)
+        dbstate.metavars = dbstate.metavars + [None] * missing
+    dbstate.metavars[index] = w_obj
+
+
+# ____________________________________________________________
+
+
+def fetch_cur_frame(silent=False):
+    ec = dbstate.space.threadlocals.get_ec()
+    if ec is None:
+        frame = None
+    else:
+        frame = ec.topframeref()
+    if frame is None and not silent:
+        revdb.send_output("No stack.\n")
+    return frame
+
+def compile(source, mode):
+    assert not dbstate.standard_code
+    space = dbstate.space
+    compiler = space.createcompiler()
+    code = compiler.compile(source, '<revdb>', mode, 0,
+                            hidden_applevel=True)
+    return code
+
+
+class W_RevDBOutput(W_Root):
+    softspace = 0
+
+    def __init__(self, space):
+        self.space = space
+
+    def descr_write(self, w_buffer):
+        assert not dbstate.standard_code
+        space = self.space
+        if space.isinstance_w(w_buffer, space.w_unicode):
+            w_buffer = space.call_method(w_buffer, 'encode',
+                                         space.newtext('utf-8'))   # safe?
+        revdb.send_output(space.bytes_w(w_buffer))
+
+def descr_get_softspace(space, revdb):
+    return space.newint(revdb.softspace)
+def descr_set_softspace(space, revdb, w_newvalue):
+    revdb.softspace = space.int_w(w_newvalue)
+
+W_RevDBOutput.typedef = typedef.TypeDef(
+    "revdb_output",
+    write = gateway.interp2app(W_RevDBOutput.descr_write),
+    softspace = typedef.GetSetProperty(descr_get_softspace,
+                                       descr_set_softspace,
+                                       cls=W_RevDBOutput),
+    )
+
+def revdb_displayhook(space, w_obj):
+    """Modified sys.displayhook() that also outputs '$NUM = ',
+    for non-prebuilt objects.  Such objects are then recorded in
+    'printed_objects'.
+    """
+    assert not dbstate.standard_code
+    if space.is_w(w_obj, space.w_None):
+        return
+    uid = revdb.get_unique_id(w_obj)
+    if uid > 0:
+        dbstate.printed_objects[uid] = w_obj
+        revdb.send_nextnid(uid)   # outputs '$NUM = '
+    space.setitem(space.builtin.w_dict, space.newtext('_'), w_obj)
+    # do repr() after setitem: if w_obj was produced successfully,
+    # but its repr crashes because it tries to do I/O, then we already
+    # have it recorded in '_' and in '$NUM ='.
+    w_repr = space.repr(w_obj)
+    if space.isinstance_w(w_repr, space.w_unicode):
+        w_repr = space.call_method(w_repr, 'encode',
+                                   space.newtext('utf-8'))   # safe?
+    revdb.send_output(space.bytes_w(w_repr))
+    revdb.send_output("\n")
+
+ at gateway.unwrap_spec(name='text0', level=int)
+def revdb_importhook(space, name, w_globals=None,
+                     w_locals=None, w_fromlist=None, level=-1):
+    # Incredibly simplified version of __import__, which only returns
+    # already-imported modules and doesn't call any custom import
+    # hooks.  Recognizes only absolute imports.  With a 'fromlist'
+    # argument that is a non-empty list, returns the module 'name3' if
+    # the 'name' argument is 'name1.name2.name3'.  With an empty or
+    # None 'fromlist' argument, returns the module 'name1' instead.
+    return space.appexec([space.newtext(name), w_fromlist or space.w_None,
+                          space.newint(level), space.sys],
+    """(name, fromlist, level, sys):
+        if level > 0:
+            raise ImportError("only absolute imports are "
+                               "supported in the debugger")
+        basename = name.split('.')[0]
+        try:
+            basemod = sys.modules[basename]
+            mod = sys.modules[name]
+        except KeyError:
+            raise ImportError("'%s' not found or not imported yet "
+                    "(the debugger can't import new modules, "
+                    "and only supports absolute imports)" % (name,))
+        if fromlist:
+            return mod
+        return basemod
+    """)
+
+ at specialize.memo()
+def get_revdb_displayhook(space):
+    return gateway.interp2app(revdb_displayhook).spacebind(space)
+
+ at specialize.memo()
+def get_revdb_importhook(space):
+    return gateway.interp2app(revdb_importhook).spacebind(space)
+
+
+def prepare_print_environment(space):
+    assert not dbstate.standard_code
+    w_revdb_output = W_RevDBOutput(space)
+    w_displayhook = get_revdb_displayhook(space)
+    w_import = get_revdb_importhook(space)
+    space.sys.setdictvalue(space, 'stdout', w_revdb_output)
+    space.sys.setdictvalue(space, 'stderr', w_revdb_output)
+    space.sys.setdictvalue(space, 'displayhook', w_displayhook)
+    space.builtin.setdictvalue(space, '__import__', w_import)
+
+def command_print(cmd, expression):
+    frame = fetch_cur_frame()
+    if frame is None:
+        return
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            prepare_print_environment(space)
+            code = compile(expression, 'single')
+            try:
+                code.exec_code(space,
+                               frame.get_w_globals(),
+                               frame.getdictscope())
+
+            except OperationError as operationerr:
+                # can't use sys.excepthook: it will likely try to do 'import
+                # traceback', which might not be doable without using I/O
+                tb = operationerr.get_traceback()
+                if tb is not None:
+                    revdb.send_output("Traceback (most recent call last):\n")
+                    while tb is not None:
+                        if not isinstance(tb, pytraceback.PyTraceback):
+                            revdb.send_output("  ??? %s\n" % tb)
+                            break
+                        show_frame(tb.frame, tb.get_lineno(), indent='  ')
+                        tb = tb.next
+                revdb.send_output('%s\n' % operationerr.errorstr(space))
+
+                # set the sys.last_xxx attributes
+                w_type = operationerr.w_type
+                w_value = operationerr.get_w_value(space)
+                w_tb = operationerr.get_traceback()
+                w_dict = space.sys.w_dict
+                space.setitem(w_dict, space.newtext('last_type'), w_type)
+                space.setitem(w_dict, space.newtext('last_value'), w_value)
+                space.setitem(w_dict, space.newtext('last_traceback'), w_tb)
+
+        except OperationError as e:
+            revdb.send_output('%s\n' % e.errorstr(space, use_repr=True))
+lambda_print = lambda: command_print
+
+
+def file_and_lineno(frame, lineno):
+    code = frame.getcode()
+    return 'File "%s", line %d in %s' % (
+        code.co_filename, lineno, code.co_name)
+
+def show_frame(frame, lineno=0, indent=''):
+    if lineno == 0:
+        lineno = frame.get_last_lineno()
+    revdb.send_output("%s%s\n%s  " % (
+        indent,
+        file_and_lineno(frame, lineno),
+        indent))
+    revdb.send_linecache(frame.getcode().co_filename, lineno)
+
+def display_function_part(frame, max_lines_before, max_lines_after):
+    if frame is None:
+        return
+    code = frame.getcode()
+    if code.co_filename.startswith('<builtin>'):
+        return
+    first_lineno = code.co_firstlineno
+    current_lineno = frame.get_last_lineno()
+    final_lineno = get_final_lineno(code)
+    #
+    ellipsis_after = False
+    if first_lineno < current_lineno - max_lines_before - 1:
+        first_lineno = current_lineno - max_lines_before
+        revdb.send_output("...\n")
+    if final_lineno > current_lineno + max_lines_after + 1:
+        final_lineno = current_lineno + max_lines_after
+        ellipsis_after = True
+    #
+    for i in range(first_lineno, final_lineno + 1):
+        if i == current_lineno:
+            if revdb.current_place() == -2: # <= this is the arg to stop_point()
+                prompt = "<< "     # return
+            elif revdb.current_place() == -1:
+                prompt = "!! "     # exceptional return
+            else:
+                prompt = " > "     # plain line
+            revdb.send_output(prompt)
+        else:
+            revdb.send_output("   ")
+        revdb.send_linecache(code.co_filename, i, strip=False)
+    #
+    if ellipsis_after:
+        revdb.send_output("...\n")
+
+def command_backtrace(cmd, extra):
+    frame = fetch_cur_frame(silent=True)
+    if cmd.c_arg1 == 0:
+        if frame is not None:
+            revdb.send_output("%s:\n" % (
+                file_and_lineno(frame, frame.get_last_lineno()),))
+        display_function_part(frame, max_lines_before=8, max_lines_after=5)
+    elif cmd.c_arg1 == 2:
+        display_function_part(frame, max_lines_before=1000,max_lines_after=1000)
+    else:
+        revdb.send_output("Current call stack (most recent call last):\n")
+        if frame is None:
+            revdb.send_output("  (empty)\n")
+        frames = []
+        while frame is not None:
+            frames.append(frame)
+            if len(frames) == 200:
+                revdb.send_output("  ...\n")
+                break
+            frame = frame.get_f_back()
+        while len(frames) > 0:
+            show_frame(frames.pop(), indent='  ')
+lambda_backtrace = lambda: command_backtrace
+
+
+def command_locals(cmd, extra):
+    frame = fetch_cur_frame()
+    if frame is None:
+        return
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            prepare_print_environment(space)
+            space.appexec([space.sys,
+                           frame.getdictscope()], """(sys, locals):
+                lst = locals.keys()
+                lst.sort()
+                print 'Locals:'
+                for key in lst:
+                    try:
+                        print '    %s =' % key,
+                        s = '%r' % locals[key]
+                        if len(s) > 140:
+                            s = s[:100] + '...' + s[-30:]
+                        print s
+                    except:
+                        exc, val, tb = sys.exc_info()
+                        print '!<%s: %r>' % (exc, val)
+            """)
+        except OperationError as e:
+            revdb.send_output('%s\n' % e.errorstr(space, use_repr=True))
+lambda_locals = lambda: command_locals
+
+# ____________________________________________________________
+
+
+def check_and_trigger_bkpt(pycode, opindex):
+    # We cache on 'pycode.co_revdb_bkpt_cache' either None or a dict
+    # mapping {opindex: bkpt_num}.  This cache is updated when the
+    # version in 'pycode.co_revdb_bkpt_version' does not match
+    # 'dbstate.breakpoint_version' any more.
+    if pycode.co_revdb_bkpt_version != dbstate.breakpoint_version:
+        update_bkpt_cache(pycode)
+    cache = pycode.co_revdb_bkpt_cache
+    if cache is not None and opindex in cache:
+        return cache[opindex]
+    else:
+        return -1
+
+def update_bkpt_cache(pycode):
+    # initialized by command_breakpoints():
+    #     dbstate.breakpoint_filelines == [('FILENAME', lineno, bkpt_num)]
+    # computed lazily (here, first half of the logic):
+    #     dbstate.breakpoint_by_file == {'co_filename': {lineno: bkpt_num}}
+    # the goal is to set:
+    #     pycode.co_revdb_bkpt_cache == {opindex: bkpt_num}
+
+    co_filename = pycode.co_filename
+    try:
+        linenos = dbstate.breakpoint_by_file[co_filename]
+    except KeyError:
+        linenos = None
+        match = co_filename.upper()    # ignore cAsE in filename matching
+        for filename, lineno, bkpt_num in dbstate.breakpoint_filelines:
+            if match.endswith(filename) and (
+                    len(match) == len(filename) or
+                    match[-len(filename)-1] in '/\\'):    # a valid prefix
+                if linenos is None:
+                    linenos = {}
+                linenos[lineno] = bkpt_num
+        dbstate.breakpoint_by_file[co_filename] = linenos
+
+    newcache = None
+    if linenos is not None:
+        # parse co_lnotab to figure out the opindexes that correspond
+        # to the marked line numbers.  here, linenos == {lineno: bkpt_num}
+        for addr, lineno in find_line_starts(pycode):
+            if lineno in linenos:
+                if newcache is None:
+                    newcache = {}
+                newcache[addr] = linenos[lineno]
+
+    pycode.co_revdb_bkpt_cache = newcache
+    pycode.co_revdb_bkpt_version = dbstate.breakpoint_version
+
+
+def valid_identifier(s):
+    if not s:
+        return False
+    if s[0].isdigit():
+        return False
+    for c in s:
+        if not (c.isalnum() or c == '_'):
+            return False
+    return True
+
+def add_breakpoint_funcname(name, i):
+    if dbstate.breakpoint_funcnames is None:
+        dbstate.breakpoint_funcnames = {}
+    dbstate.breakpoint_funcnames[name] = i
+
+def add_breakpoint_fileline(filename, lineno, i):
+    # dbstate.breakpoint_filelines is just a list of (FILENAME, lineno, i).
+    # dbstate.breakpoint_by_file is {co_filename: {lineno: i}}, but
+    # computed lazily when we encounter a code object with the given
+    # co_filename.  Any suffix 'filename' matches 'co_filename'.
+    if dbstate.breakpoint_filelines is None:
+        dbstate.breakpoint_filelines = []
+        dbstate.breakpoint_by_file = {}
+    dbstate.breakpoint_filelines.append((filename.upper(), lineno, i))
+
+def add_breakpoint(name, i):
+    # if it is empty, complain
+    if not name:
+        revdb.send_output("Empty breakpoint name\n")
+        revdb.send_change_breakpoint(i)
+        return
+    # if it is surrounded by < >, it is the name of a code object
+    if name.startswith('<') and name.endswith('>'):
+        add_breakpoint_funcname(name, i)
+        return
+    # if it has no ':', it can be a valid identifier (which we
+    # register as a function name), or a lineno
+    original_name = name
+    j = name.rfind(':')
+    if j < 0:
+        try:
+            lineno = int(name)
+        except ValueError:
+            if name.endswith('()'):
+                n = len(name) - 2
+                assert n >= 0
+                name = name[:n]
+            if not valid_identifier(name):
+                revdb.send_output(
+                    'Note: "%s()" doesn''t look like a function name. '
+                    'Setting breakpoint anyway\n' % name)
+            add_breakpoint_funcname(name, i)
+            name += '()'
+            if name != original_name:
+                revdb.send_change_breakpoint(i, name)
+            return
+        # "number" does the same as ":number"
+        filename = ''
+    else:
+        # if it has a ':', it must end in ':lineno'
+        try:
+            lineno = int(name[j+1:])
+        except ValueError:
+            revdb.send_output('expected a line number after colon\n')
+            revdb.send_change_breakpoint(i)
+            return
+        filename = name[:j]
+
+    # the text before must be a pathname, possibly a relative one,
+    # or be escaped by < >.  if it isn't, make it absolute and normalized
+    # and warn if it doesn't end in '.py'.
+    if filename == '':
+        frame = fetch_cur_frame()
+        if frame is None:
+            revdb.send_change_breakpoint(i)
+            return
+        filename = frame.getcode().co_filename
+    elif filename.startswith('<') and filename.endswith('>'):
+        pass    # use unmodified
+    elif not filename.lower().endswith('.py'):
+        # use unmodified, but warn
+        revdb.send_output(
+            'Note: "%s" doesn''t look like a Python filename. '
+            'Setting breakpoint anyway\n' % (filename,))
+
+    add_breakpoint_fileline(filename, lineno, i)
+    name = '%s:%d' % (filename, lineno)
+    if name != original_name:
+        revdb.send_change_breakpoint(i, name)
+
+def command_breakpoints(cmd, extra):
+    space = dbstate.space
+    dbstate.breakpoint_stack_id = cmd.c_arg1
+    revdb.set_thread_breakpoint(cmd.c_arg2)
+    dbstate.breakpoint_funcnames = None
+    dbstate.breakpoint_filelines = None
+    dbstate.breakpoint_by_file = None
+    dbstate.breakpoint_version += 1
+    watch_progs = []
+    with non_standard_code:
+        for i, kind, name in revdb.split_breakpoints_arg(extra):
+            if kind == 'B':
+                add_breakpoint(name, i)
+            elif kind == 'W':
+                code = interp_marshal.loads(space, space.newbytes(name))
+                watch_progs.append((code, i, ''))
+    dbstate.watch_progs = watch_progs[:]
+lambda_breakpoints = lambda: command_breakpoints
+
+
+def command_watchvalues(cmd, extra):
+    expected = extra.split('\x00')
+    for j in range(len(dbstate.watch_progs)):
+        prog, i, _ = dbstate.watch_progs[j]
+        if i >= len(expected):
+            raise IndexError
+        dbstate.watch_progs[j] = prog, i, expected[i]
+lambda_watchvalues = lambda: command_watchvalues
+
+
+def command_stackid(cmd, extra):
+    frame = fetch_cur_frame(silent=True)
+    if frame is not None and cmd.c_arg1 != 0:     # parent_flag
+        frame = dbstate.space.getexecutioncontext().getnextframe_nohidden(frame)
+    if frame is None:
+        uid = 0
+    else:
+        uid = revdb.get_unique_id(frame)
+    if revdb.current_place() == -2:
+        hidden_level = 1      # hide the "<<" events from next/bnext commands
+    else:
+        hidden_level = 0
+    revdb.send_answer(revdb.ANSWER_STACKID, uid, hidden_level)
+lambda_stackid = lambda: command_stackid
+
+
+def command_allocating(uid, gcref):
+    w_obj = cast_gcref_to_instance(W_Root, gcref)
+    dbstate.printed_objects[uid] = w_obj
+    try:
+        index_metavar = dbstate.watch_futures.pop(uid)
+    except KeyError:
+        pass
+    else:
+        set_metavar(index_metavar, w_obj)
+lambda_allocating = lambda: command_allocating
+
+
+def command_attachid(cmd, extra):
+    space = dbstate.space
+    index_metavar = cmd.c_arg1
+    uid = cmd.c_arg2
+    try:
+        w_obj = dbstate.printed_objects[uid]
+    except KeyError:
+        # uid not found, probably a future object
+        dbstate.watch_futures[uid] = index_metavar
+        w_obj = future_object(space)
+    set_metavar(index_metavar, w_obj)
+lambda_attachid = lambda: command_attachid
+
+
+def command_compilewatch(cmd, expression):
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            code = compile(expression, 'eval')
+            marshalled_code = space.bytes_w(interp_marshal.dumps(
+                space, code,
+                space.newint(interp_marshal.Py_MARSHAL_VERSION)))
+        except OperationError as e:
+            revdb.send_watch(e.errorstr(space), ok_flag=0)
+        else:
+            revdb.send_watch(marshalled_code, ok_flag=1)
+lambda_compilewatch = lambda: command_compilewatch
+
+def command_checkwatch(cmd, marshalled_code):
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            code = interp_marshal.loads(space, space.newbytes(marshalled_code))
+            text = _run_watch(space, code)
+        except OperationError as e:
+            revdb.send_watch(e.errorstr(space), ok_flag=0)
+        else:
+            revdb.send_watch(text, ok_flag=1)
+lambda_checkwatch = lambda: command_checkwatch
+
+
+def _run_watch(space, prog):
+    # must be called from non_standard_code!
+    w_dict = space.builtin.w_dict
+    w_res = prog.exec_code(space, w_dict, w_dict)
+    return space.text_w(space.repr(w_res))
+
+
+# ____________________________________________________________
+
+
+ActionFlag._ticker_revdb_count = -1
+
+class RDBSignalActionFlag(AbstractActionFlag):
+    # Used instead of pypy.module.signal.interp_signal.SignalActionFlag
+    # when we have reverse-debugging.  That other class would work too,
+    # but inefficiently: it would generate two words of data per bytecode.
+    # This class is tweaked to generate one byte per _SIG_TICKER_COUNT
+    # bytecodes, at the expense of not reacting to signals instantly.
+
+    # Threads: after 10'000 calls to decrement_ticker(), it should
+    # return -1.  It should also return -1 if there was a signal.
+    # This is done by calling _update_ticker_from_signals() every 100
+    # calls, and invoking rsignal.pypysig_check_and_reset(); this in
+    # turn returns -1 if there was a signal or if it was called 100
+    # times.
+
+    _SIG_TICKER_COUNT = 100
+    _ticker = 0
+    _ticker_revdb_count = _SIG_TICKER_COUNT * 10
+
+    def get_ticker(self):
+        return self._ticker
+
+    def reset_ticker(self, value):
+        self._ticker = value
+
+    def rearm_ticker(self):
+        self._ticker = -1
+
+    def decrement_ticker(self, by):
+        if we_are_translated():
+            c = self._ticker_revdb_count - 1
+            if c < 0:
+                c = self._update_ticker_from_signals()
+            self._ticker_revdb_count = c
+        #if self.has_bytecode_counter:    # this 'if' is constant-folded
+        #    print ("RDBSignalActionFlag: has_bytecode_counter: "
+        #           "not supported for now")
+        #    raise NotImplementedError
+        return self._ticker
+
+    def _update_ticker_from_signals(self):
+        from rpython.rlib import rsignal
+        if dbstate.standard_code:
+            if rsignal.pypysig_check_and_reset():
+                self.rearm_ticker()
+        return self._SIG_TICKER_COUNT
+    _update_ticker_from_signals._dont_inline_ = True
diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/test_reverse_debugging.py
@@ -0,0 +1,121 @@
+import dis
+from pypy.interpreter.reverse_debugging import *
+from pypy.interpreter import reverse_debugging
+from rpython.rlib import revdb
+
+
+class FakeCode:
+    hidden_applevel = False
+    def __init__(self, co_code='', co_lnotab='', co_filename='?'):
+        self.co_firstlineno = 43
+        self.co_code = co_code
+        self.co_lnotab = co_lnotab
+        self.co_revdb_linestarts = None
+        self.co_filename = co_filename
+
+
+try:
+    from hypothesis import given, strategies, example
+except ImportError:
+    pass
+else:
+    @given(strategies.binary())
+    @example("\x01\x02\x03\x04"
+             "\x00\xFF\x20\x30\x00\xFF\x00\x40"
+             "\xFF\x00\x0A\x0B\xFF\x00\x0C\x00")
+    def test_build_co_revdb_linestarts(lnotab):
+        if len(lnotab) & 1:
+            lnotab = lnotab + '\x00'   # make the length even
+        code = FakeCode("?" * sum(map(ord, lnotab[0::2])), lnotab)
+        lstart = build_co_revdb_linestarts(code)
+        assert lstart is code.co_revdb_linestarts
+
+        expected_starts = set()
+        for addr, lineno in dis.findlinestarts(code):
+            expected_starts.add(addr)
+
+        next_start = len(code.co_code)
+        for index in range(len(code.co_code), -1, -1):
+            if index in expected_starts:
+                next_start = index
+            assert lstart[index] == chr(next_start - index
+                                        if next_start - index <= 255
+                                        else 255)
+
+
+class FakeFrame:
+    def __init__(self, code):
+        self.__code = code
+    def getcode(self):
+        return self.__code
+
+def check_add_breakpoint(input, curfilename=None,
+                         expected_funcname=None,
+                         expected_fileline=None,
+                         expected_output=None,
+                         expected_chbkpt=None):
+    dbstate.__dict__.clear()
+    prev = revdb.send_answer, reverse_debugging.fetch_cur_frame
+    try:
+        messages = []
+        def got_message(cmd, arg1=0, arg2=0, arg3=0, extra=""):
+            messages.append((cmd, arg1, arg2, arg3, extra))
+        def my_cur_frame():
+            assert curfilename is not None
+            return FakeFrame(FakeCode(co_filename=curfilename))
+        revdb.send_answer = got_message
+        reverse_debugging.fetch_cur_frame = my_cur_frame
+        add_breakpoint(input, 5)
+    finally:
+        revdb.send_answer, reverse_debugging.fetch_cur_frame = prev
+
+    if expected_funcname is None:
+        assert dbstate.breakpoint_funcnames is None
+    else:
+        assert dbstate.breakpoint_funcnames == {expected_funcname: 5}
+
+    if expected_fileline is None:
+        assert dbstate.breakpoint_filelines is None
+    else:
+        filename, lineno = expected_fileline
+        assert dbstate.breakpoint_filelines == [(filename.upper(), lineno, 5)]
+
+    got_output = None
+    got_chbkpt = None
+    for msg in messages:
+        if msg[0] == revdb.ANSWER_TEXT:
+            assert got_output is None
+            got_output = msg[-1]
+        elif msg[0] == revdb.ANSWER_CHBKPT:
+            assert got_chbkpt is None
+            assert msg[1] == 5
+            got_chbkpt = msg[-1]
+
+    assert got_output == expected_output
+    assert got_chbkpt == expected_chbkpt
+
+def test_add_breakpoint():
+    check_add_breakpoint('', expected_output="Empty breakpoint name\n",
+                         expected_chbkpt='')
+    check_add_breakpoint('foo42', expected_funcname="foo42",
+                         expected_chbkpt="foo42()")
+    check_add_breakpoint('foo42()', expected_funcname="foo42")
+    check_add_breakpoint('foo.bar', expected_funcname="foo.bar",
+        expected_output='Note: "foo.bar()" doesn''t look like a function name.'
+                        ' Setting breakpoint anyway\n',
+        expected_chbkpt="foo.bar()")
+    check_add_breakpoint('<foo.bar>', expected_funcname="<foo.bar>")
+    check_add_breakpoint('42', curfilename='abcd',
+                         expected_fileline=('abcd', 42),
+                         expected_chbkpt='abcd:42')
+    check_add_breakpoint(':42', curfilename='abcd',
+                         expected_fileline=('abcd', 42),
+                         expected_chbkpt='abcd:42')
+    check_add_breakpoint('abcd:42', expected_fileline=('abcd', 42),
+        expected_output='Note: "abcd" doesnt look like a Python filename.'
+                        ' Setting breakpoint anyway\n')
+    check_add_breakpoint('abcd.py:42',
+                         expected_fileline=('abcd.py', 42))
+    check_add_breakpoint('42:abc',
+        expected_output='expected a line number after colon\n',
+        expected_chbkpt='')
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -106,6 +106,7 @@
                           'interp_magic.save_module_content_for_future_reload',
         'decode_long'               : 'interp_magic.decode_long',
         '_promote'                   : 'interp_magic._promote',
+        'side_effects_ok'           : 'interp_magic.side_effects_ok',
         'stack_almost_full'         : 'interp_magic.stack_almost_full',
     }
     if sys.platform == 'win32':
@@ -145,3 +146,7 @@
             features = detect_cpu.getcpufeatures(model)
             self.extra_interpdef('jit_backend_features',
                                     'space.wrap(%r)' % features)
+        if self.space.config.translation.reverse_debugger:
+            self.extra_interpdef('revdb_stop', 'interp_magic.revdb_stop')
+        else:
+            self.extra_interpdef('revdb_stop', 'space.w_None')
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -191,3 +191,23 @@
 def stack_almost_full(space):
     """Return True if the stack is more than 15/16th full."""
     return space.newbool(rstack.stack_almost_full())
+
+def side_effects_ok(space):
+    """For use with the reverse-debugger: this function normally returns
+    True, but will return False if we are evaluating a debugging command
+    like a watchpoint.  You are responsible for not doing any side effect
+    at all (including no caching) when evaluating watchpoints.  This
+    function is meant to help a bit---you can write:
+
+        if not __pypy__.side_effects_ok():
+            skip the caching logic
+
+    inside getter methods or properties, to make them usable from
+    watchpoints.  Note that you need to re-run ``REVDB=.. pypy''
+    after changing the Python code.
+    """
+    return space.newbool(space._side_effects_ok())
+
+def revdb_stop(space):
+    from pypy.interpreter.reverse_debugging import stop_point
+    stop_point()
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -3,9 +3,9 @@
 """
 import sys, os, py
 
-from rpython.rlib import clibffi, jit, rgc, objectmodel
+from rpython.rlib import clibffi, jit, objectmodel
 from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.module._cffi_backend import cerrno, misc, parse_c_type
@@ -13,6 +13,7 @@
 from pypy.module._cffi_backend.ctypefunc import SIZE_OF_FFI_ARG, W_CTypeFunc
 from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveSigned
 from pypy.module._cffi_backend.ctypevoid import W_CTypeVoid
+from pypy.module._cffi_backend.hide_reveal import hide_reveal1
 
 BIG_ENDIAN = sys.byteorder == 'big'
 
@@ -30,9 +31,7 @@
     return cdata
 
 def reveal_callback(raw_ptr):
-    addr = rffi.cast(llmemory.Address, raw_ptr)
-    gcref = rgc.reveal_gcref(addr)
-    return rgc.try_cast_gcref_to_instance(W_ExternPython, gcref)
+    return hide_reveal1().reveal_object(W_ExternPython, raw_ptr)
 
 
 class Closure(object):
@@ -92,9 +91,7 @@
         return ctype
 
     def hide_object(self):
-        gcref = rgc.cast_instance_to_gcref(self)
-        raw = rgc.hide_nonmovable_gcref(gcref)
-        return rffi.cast(rffi.VOIDP, raw)
+        return hide_reveal1().hide_object(rffi.VOIDP, self)
 
     def _repr_extra(self):
         space = self.space
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -168,7 +168,8 @@
     rawbytes = cache.wdict.get(w_x)
     if rawbytes is None:
         data = space.bytes_w(w_x)
-        if we_are_translated() and not rgc.can_move(data):
+        if (we_are_translated() and not rgc.can_move(data)
+                                and not rgc.must_split_gc_address_space()):
             lldata = llstr(data)
             data_start = (llmemory.cast_ptr_to_adr(lldata) +
                           rffi.offsetof(STR, 'chars') +
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -3,8 +3,9 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module._cffi_backend import ctypeobj, ctypeptr, cdataobj
+from pypy.module._cffi_backend.hide_reveal import hide_reveal2
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.rlib import rgc, objectmodel, jit
+from rpython.rlib import objectmodel, jit
 
 # ____________________________________________________________
 
@@ -15,9 +16,7 @@
     # we can cast the CCHARP back to a W_CDataHandle with reveal_gcref().
     new_cdataobj = objectmodel.instantiate(cdataobj.W_CDataHandle,
                                            nonmovable=True)
-    gcref = rgc.cast_instance_to_gcref(new_cdataobj)
-    _cdata = rgc.hide_nonmovable_gcref(gcref)
-    _cdata = rffi.cast(rffi.CCHARP, _cdata)
+    _cdata = hide_reveal2().hide_object(rffi.CCHARP, new_cdataobj)
     cdataobj.W_CDataHandle.__init__(new_cdataobj, space, _cdata, w_ctype, w_x)
     return new_cdataobj
 
@@ -43,11 +42,10 @@
 @jit.dont_look_inside
 def _reveal(space, ptr):
     addr = rffi.cast(llmemory.Address, ptr)
-    gcref = rgc.reveal_gcref(addr)
-    if not gcref:
+    if not addr:
         raise oefmt(space.w_RuntimeError,
                     "cannot use from_handle() on NULL pointer")
-    cd = rgc.try_cast_gcref_to_instance(cdataobj.W_CDataHandle, gcref)
+    cd = hide_reveal2().reveal_object(cdataobj.W_CDataHandle, addr)
     if cd is None:
         raise oefmt(space.w_SystemError,
                     "ffi.from_handle(): dead or bogus object handle")
diff --git a/pypy/module/_cffi_backend/hide_reveal.py b/pypy/module/_cffi_backend/hide_reveal.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/hide_reveal.py
@@ -0,0 +1,66 @@
+from rpython.rlib import rgc
+from rpython.rlib.rweaklist import RWeakListMixin
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+
+
+class HideRevealRWeakList:
+    """Slow implementation of HideReveal: uses a RWeakListMixin."""
+
+    def __init__(self):
+        class GlobGcrefs(RWeakListMixin):
+            pass
+        glob_gcrefs = GlobGcrefs()
+        glob_gcrefs.initialize()
+
+        def hide_object(PTR, obj):
+            # XXX leaks if we call this function often on the same object
+            index = glob_gcrefs.add_handle(obj)
+            return rffi.cast(PTR, index + 1)
+
+        def reveal_object(Class, addr):
+            index = rffi.cast(lltype.Signed, addr) - 1
+            return glob_gcrefs.fetch_handle(index)
+
+        self.hide_object = hide_object
+        self.reveal_object = reveal_object
+
+    def _freeze_(self):
+        return True
+
+
+class HideRevealCast:
+    """Fast implementation of HideReveal: just a cast."""
+
+    def __init__(self):
+

From pypy.commits at gmail.com  Wed May 23 16:47:26 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 13:47:26 -0700 (PDT)
Subject: [pypy-commit] pypy reverse-debugger: Close branch, ready to merge,
 finally!
Message-ID: <5b05d35e.1c69fb81.17292.4b54@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: reverse-debugger
Changeset: r94672:cc66db75546c
Date: 2018-05-23 22:43 +0200
http://bitbucket.org/pypy/pypy/changeset/cc66db75546c/

Log:	Close branch, ready to merge, finally!


From pypy.commits at gmail.com  Wed May 23 16:47:35 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 13:47:35 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: document merge
Message-ID: <5b05d367.1c69fb81.ad4cb.198b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94676:9f6f5b98888f
Date: 2018-05-23 22:46 +0200
http://bitbucket.org/pypy/pypy/changeset/9f6f5b98888f/

Log:	document merge

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -12,3 +12,8 @@
 .. branch: py3.5-user-site-impl
 
 Use implementation-specific site directories in sysconfig like in Python2
+
+.. branch: py3.5-reverse-debugger
+
+The reverse-debugger branch has been merged.  For more information, see
+https://bitbucket.org/pypy/revdb

From pypy.commits at gmail.com  Wed May 23 16:47:33 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 13:47:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge py3.5-reverse-debugger
Message-ID: <5b05d365.1c69fb81.1710.b1f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94675:f984fd051c58
Date: 2018-05-23 22:44 +0200
http://bitbucket.org/pypy/pypy/changeset/f984fd051c58/

Log:	hg merge py3.5-reverse-debugger

diff too long, truncating to 2000 out of 7499 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -33,7 +33,12 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1
+d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
+e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1
+a24d6c7000c8099c73d3660857f7e3cee5ac045c RevDB-pypy2.7-v5.6.2
 fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
 b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py
--- a/lib-python/3/opcode.py
+++ b/lib-python/3/opcode.py
@@ -223,5 +223,6 @@
 def_op('CALL_METHOD', 202)            # #args not including 'self'
 def_op('BUILD_LIST_FROM_ARG', 203)
 jrel_op('JUMP_IF_NOT_DEBUG', 204)     # jump over assert statements
+def_op('LOAD_REVDB_VAR', 205)         # reverse debugger (syntax example: $5)
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -61,6 +61,11 @@
     "termios", "_minimal_curses",
 ])
 
+reverse_debugger_disable_modules = set([
+    "_continuation", "_vmprof", "_multiprocessing",
+    "micronumpy",
+    ])
+
 # XXX this should move somewhere else, maybe to platform ("is this posixish"
 #     check or something)
 if sys.platform == "win32":
@@ -297,6 +302,9 @@
     modules = working_modules.copy()
     if config.translation.sandbox:
         modules = default_modules
+    if config.translation.reverse_debugger:
+        for mod in reverse_debugger_disable_modules:
+            setattr(config.objspace.usemodules, mod, False)
     # ignore names from 'essential_modules', notably 'exceptions', which
     # may not be present in config.objspace.usemodules at all
     modules = [name for name in modules if name not in essential_modules]
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -22,3 +22,8 @@
 .. branch: gc-more-logging
 
 Log additional gc-minor and gc-collect-step info in the PYPYLOG
+
+.. branch: reverse-debugger
+
+The reverse-debugger branch has been merged.  For more information, see
+https://bitbucket.org/pypy/revdb
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -88,11 +88,24 @@
     run_protected() handles details like forwarding exceptions to
     sys.excepthook(), catching SystemExit, etc.
     """
+    # don't use try:except: here, otherwise the exception remains
+    # visible in user code.  Make sure revdb_stop is a callable, so
+    # that we can call it immediately after finally: below.  Doing
+    # so minimizes the number of "blind" lines that we need to go
+    # back from, with "bstep", after we do "continue" in revdb.
+    if '__pypy__' in sys.builtin_module_names:
+        from __pypy__ import revdb_stop
+    else:
+        revdb_stop = None
+    if revdb_stop is None:
+        revdb_stop = lambda: None
+
     try:
         # run it
         try:
             f(*fargs, **fkwds)
         finally:
+            revdb_stop()
             sys.settrace(None)
             sys.setprofile(None)
     except SystemExit as e:
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -692,6 +692,7 @@
 
     # TODO
     ops.BUILD_LIST_FROM_ARG: 1,
+    ops.LOAD_REVDB_VAR: 1,
 
     ops.LOAD_CLASSDEREF: 1,
 }
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1733,6 +1733,8 @@
             return Num.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_Str):
             return Str.from_object(space, w_node)
+        if space.isinstance_w(w_node, get(space).w_RevDBMetaVar):
+            return RevDBMetaVar.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_FormattedValue):
             return FormattedValue.from_object(space, w_node)
         if space.isinstance_w(w_node, get(space).w_JoinedStr):
@@ -2643,6 +2645,41 @@
 State.ast_type('Str', 'expr', ['s'])
 
 
+class RevDBMetaVar(expr):
+
+    def __init__(self, metavar, lineno, col_offset):
+        self.metavar = metavar
+        expr.__init__(self, lineno, col_offset)
+
+    def walkabout(self, visitor):
+        visitor.visit_RevDBMetaVar(self)
+
+    def mutate_over(self, visitor):
+        return visitor.visit_RevDBMetaVar(self)
+
+    def to_object(self, space):
+        w_node = space.call_function(get(space).w_RevDBMetaVar)
+        w_metavar = space.newint(self.metavar)  # int
+        space.setattr(w_node, space.newtext('metavar'), w_metavar)
+        w_lineno = space.newint(self.lineno)  # int
+        space.setattr(w_node, space.newtext('lineno'), w_lineno)
+        w_col_offset = space.newint(self.col_offset)  # int
+        space.setattr(w_node, space.newtext('col_offset'), w_col_offset)
+        return w_node
+
+    @staticmethod
+    def from_object(space, w_node):
+        w_metavar = get_field(space, w_node, 'metavar', False)
+        w_lineno = get_field(space, w_node, 'lineno', False)
+        w_col_offset = get_field(space, w_node, 'col_offset', False)
+        _metavar = space.int_w(w_metavar)
+        _lineno = space.int_w(w_lineno)
+        _col_offset = space.int_w(w_col_offset)
+        return RevDBMetaVar(_metavar, _lineno, _col_offset)
+
+State.ast_type('RevDBMetaVar', 'expr', ['metavar'])
+
+
 class FormattedValue(expr):
 
     def __init__(self, value, conversion, format_spec, lineno, col_offset):
@@ -4130,6 +4167,8 @@
         return self.default_visitor(node)
     def visit_Str(self, node):
         return self.default_visitor(node)
+    def visit_RevDBMetaVar(self, node):
+        return self.default_visitor(node)
     def visit_FormattedValue(self, node):
         return self.default_visitor(node)
     def visit_JoinedStr(self, node):
@@ -4363,6 +4402,9 @@
     def visit_Str(self, node):
         pass
 
+    def visit_RevDBMetaVar(self, node):
+        pass
+
     def visit_FormattedValue(self, node):
         node.value.walkabout(self)
         if node.format_spec:
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1264,6 +1264,11 @@
                 else:
                     # a dictionary display
                     return self.handle_dictdisplay(maker, atom_node)
+        elif first_child_type == tokens.REVDBMETAVAR:
+            string = atom_node.get_child(0).get_value()
+            return ast.RevDBMetaVar(int(string[1:]),
+                                    atom_node.get_lineno(),
+                                    atom_node.get_column())
         else:
             raise AssertionError("unknown atom")
 
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1519,6 +1519,20 @@
             fmt.format_spec.walkabout(self)
         self.emit_op_arg(ops.FORMAT_VALUE, arg)
 
+    def _revdb_metavar(self, node):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import dbstate
+        if not dbstate.standard_code:
+            self.emit_op_arg(ops.LOAD_REVDB_VAR, node.metavar)
+            return True
+        return False
+
+    def visit_RevDBMetaVar(self, node):
+        if self.space.reverse_debugging and self._revdb_metavar(node):
+            return
+        self.error("Unknown character ('$NUM' is only valid in the "
+                   "reverse-debugger)", node)
+
 
 class TopLevelCodeGenerator(PythonCodeGenerator):
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1240,6 +1240,20 @@
         src = """# -*- coding: utf-8 -*-\nz=ord(fr'\xc3\x98')\n"""
         yield self.st, src, 'z', 0xd8
 
+    def test_revdb_metavar(self):
+        from pypy.interpreter.reverse_debugging import dbstate, setup_revdb
+        self.space.config.translation.reverse_debugger = True
+        self.space.reverse_debugging = True
+        try:
+            setup_revdb(self.space)
+            dbstate.standard_code = False
+            dbstate.metavars = [self.space.wrap(6)]
+            self.simple_test("x = 7*$0", "x", 42)
+            dbstate.standard_code = True
+            self.error_test("x = 7*$0", SyntaxError)
+        finally:
+            self.space.reverse_debugging = False
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -70,6 +70,7 @@
          | Call(expr func, expr* args, keyword* keywords)
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
+         | RevDBMetaVar(int metavar)
          | FormattedValue(expr value, int? conversion, expr? format_spec)
          | JoinedStr(expr* values)
          | Bytes(bytes s)
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -425,6 +425,9 @@
         node.slice.walkabout(self)
         self._validate_expr(node.value)
 
+    def visit_RevDBMetaVar(self, node):
+        pass
+
     # Subscripts
     def visit_Slice(self, node):
         if node.lower:
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -402,6 +402,8 @@
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
 
+    reverse_debugging = False
+
     @not_rpython
     def __init__(self, config=None):
         "Basic initialization of objects."
@@ -413,6 +415,7 @@
             from pypy.config.pypyoption import get_pypy_config
             config = get_pypy_config(translating=False)
         self.config = config
+        self.reverse_debugging = config.translation.reverse_debugger
 
         self.builtin_modules = {}
         self.reloading_modules = {}
@@ -430,6 +433,9 @@
 
     def startup(self):
         # To be called before using the space
+        if self.reverse_debugging:
+            self._revdb_startup()
+
         self.threadlocals.enter_thread(self)
 
         # Initialize already imported builtin modules
@@ -820,7 +826,8 @@
         w_u1 = self.interned_strings.get(u)
         if w_u1 is None:
             w_u1 = w_u
-            self.interned_strings.set(u, w_u1)
+            if self._side_effects_ok():
+                self.interned_strings.set(u, w_u1)
         return w_u1
 
     def new_interned_str(self, s):
@@ -832,9 +839,39 @@
         w_s1 = self.interned_strings.get(u)
         if w_s1 is None:
             w_s1 = self.newunicode(u)
-            self.interned_strings.set(u, w_s1)
+            if self._side_effects_ok():
+                self.interned_strings.set(u, w_s1)
         return w_s1
 
+    def _revdb_startup(self):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import setup_revdb
+        setup_revdb(self)
+
+    def _revdb_standard_code(self):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import dbstate
+        return dbstate.standard_code
+
+    def _side_effects_ok(self):
+        # For the reverse debugger: we run compiled watchpoint
+        # expressions in a fast way that will crash if they have
+        # side-effects.  The obvious Python code with side-effects is
+        # documented "don't do that"; but some non-obvious side
+        # effects are also common, like interning strings (from
+        # unmarshalling the code object containing the watchpoint
+        # expression) to the two attribute caches in mapdict.py and
+        # typeobject.py.  For now, we have to identify such places
+        # that are not acceptable for "reasonable" read-only
+        # watchpoint expressions, and write:
+        #
+        #     if not space._side_effects_ok():
+        #         don't cache.
+        #
+        if self.reverse_debugging:
+            return self._revdb_standard_code()
+        return True
+
     def get_interned_str(self, s):
         """Assumes an identifier (utf-8 encoded str).  Returns None if
         the identifier is not interned, or not a valid utf-8 string at all.
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -71,6 +71,8 @@
         return frame
 
     def enter(self, frame):
+        if self.space.reverse_debugging:
+            self._revdb_enter(frame)
         frame.f_backref = self.topframeref
         self.topframeref = jit.virtual_ref(frame)
 
@@ -91,6 +93,8 @@
                 # be accessed also later
                 frame_vref()
             jit.virtual_ref_finish(frame_vref, frame)
+            if self.space.reverse_debugging:
+                self._revdb_leave(got_exception)
 
     # ________________________________________________________________
 
@@ -160,6 +164,8 @@
         Like bytecode_trace() but doesn't invoke any other events besides the
         trace function.
         """
+        if self.space.reverse_debugging:
+            self._revdb_potential_stop_point(frame)
         if (frame.get_w_f_trace() is None or self.is_tracing or
             self.gettrace() is None):
             return
@@ -373,6 +379,21 @@
         if self.space.check_signal_action is not None:
             self.space.check_signal_action.perform(self, None)
 
+    def _revdb_enter(self, frame):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import enter_call
+        enter_call(self.topframeref(), frame)
+
+    def _revdb_leave(self, got_exception):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import leave_call
+        leave_call(self.topframeref(), got_exception)
+
+    def _revdb_potential_stop_point(self, frame):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import potential_stop_point
+        potential_stop_point(frame)
+
     def _freeze_(self):
         raise Exception("ExecutionContext instances should not be seen during"
                         " translation.  Now is a good time to inspect the"
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -774,9 +774,11 @@
     def fget_f_builtins(self, space):
         return self.get_builtin().getdict(space)
 
+    def get_f_back(self):
+        return ExecutionContext.getnextframe_nohidden(self)
+
     def fget_f_back(self, space):
-        f_back = ExecutionContext.getnextframe_nohidden(self)
-        return f_back
+        return self.get_f_back()
 
     def fget_f_lasti(self, space):
         return self.space.newint(self.last_instr)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -438,6 +438,8 @@
                 self.FORMAT_VALUE(oparg, next_instr)
             elif opcode == opcodedesc.BUILD_STRING.index:
                 self.BUILD_STRING(oparg, next_instr)
+            elif opcode == opcodedesc.LOAD_REVDB_VAR.index:
+                self.LOAD_REVDB_VAR(oparg, next_instr)
             else:
                 self.MISSING_OPCODE(oparg, next_instr)
 
@@ -1114,9 +1116,16 @@
         # final result and returns.  In that case, we can just continue
         # with the next bytecode.
 
+    def _revdb_jump_backward(self, jumpto):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import jump_backward
+        jump_backward(self, jumpto)
+
     def jump_absolute(self, jumpto, ec):
         # this function is overridden by pypy.module.pypyjit.interp_jit
         check_nonneg(jumpto)
+        if self.space.reverse_debugging:
+            self._revdb_jump_backward(jumpto)
         return jumpto
 
     def JUMP_FORWARD(self, jumpby, next_instr):
@@ -1656,6 +1665,19 @@
         w_res = space.newunicode(u''.join(lst))
         self.pushvalue(w_res)
 
+    def _revdb_load_var(self, oparg):
+        # moved in its own function for the import statement
+        from pypy.interpreter.reverse_debugging import load_metavar
+        w_var = load_metavar(oparg)
+        self.pushvalue(w_var)
+
+    def LOAD_REVDB_VAR(self, oparg, next_instr):
+        if self.space.reverse_debugging:
+            self._revdb_load_var(oparg)
+        else:
+            self.MISSING_OPCODE(oparg, next_instr)
+
+
 ### ____________________________________________________________ ###
 
 class ExitFrame(Exception):
diff --git a/pypy/interpreter/pyparser/data/Grammar2.7 b/pypy/interpreter/pyparser/data/Grammar2.7
--- a/pypy/interpreter/pyparser/data/Grammar2.7
+++ b/pypy/interpreter/pyparser/data/Grammar2.7
@@ -104,7 +104,7 @@
        '[' [listmaker] ']' |
        '{' [dictorsetmaker] '}' |
        '`' testlist1 '`' |
-       NAME | NUMBER | STRING+)
+       NAME | NUMBER | STRING+ | '$NUM')
 listmaker: test ( list_for | (',' test)* [','] )
 testlist_comp: test ( comp_for | (',' test)* [','] )
 lambdef: 'lambda' [varargslist] ':' test
diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -103,7 +103,7 @@
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
-       NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
+       NAME | NUMBER | STRING+ | '$NUM' | '...' | 'None' | 'True' | 'False')
 testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
diff --git a/pypy/interpreter/pyparser/data/Grammar3.3 b/pypy/interpreter/pyparser/data/Grammar3.3
--- a/pypy/interpreter/pyparser/data/Grammar3.3
+++ b/pypy/interpreter/pyparser/data/Grammar3.3
@@ -103,7 +103,7 @@
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
-       NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
+       NAME | NUMBER | STRING+ | '$NUM' | '...' | 'None' | 'True' | 'False')
 testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
diff --git a/pypy/interpreter/pyparser/data/Grammar3.5 b/pypy/interpreter/pyparser/data/Grammar3.5
--- a/pypy/interpreter/pyparser/data/Grammar3.5
+++ b/pypy/interpreter/pyparser/data/Grammar3.5
@@ -108,7 +108,7 @@
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
-       NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
+       NAME | NUMBER | STRING+ | '$NUM' | '...' | 'None' | 'True' | 'False')
 testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py
--- a/pypy/interpreter/pyparser/dfa_generated.py
+++ b/pypy/interpreter/pyparser/dfa_generated.py
@@ -6,38 +6,39 @@
 from pypy.interpreter.pyparser import automata
 accepts = [True, True, True, True, True, True, True, True,
            True, True, True, False, True, True, True, True,
-           True, False, False, False, True, False, False,
-           False, True, False, True, False, True, False,
-           False, True, False, False, True, False, False,
-           True, True, True, False, False, True, False,
-           False, False, True]
+           True, False, False, False, False, True, False,
+           False, False, True, False, True, False, True,
+           False, True, False, True, False, False, True,
+           False, False, True, True, True, False, False,
+           True, False, False, False, True]
 states = [
     # 0
     {'\t': 0, '\n': 15, '\x0c': 0,
-     '\r': 16, ' ': 0, '!': 11, '"': 18,
-     '#': 20, '%': 14, '&': 14, "'": 17,
-     '(': 15, ')': 15, '*': 8, '+': 14,
-     ',': 15, '-': 12, '.': 7, '/': 13,
-     '0': 5, '1': 6, '2': 6, '3': 6,
-     '4': 6, '5': 6, '6': 6, '7': 6,
-     '8': 6, '9': 6, ':': 15, ';': 15,
-     '<': 10, '=': 14, '>': 9, '@': 14,
-     'A': 1, 'B': 2, 'C': 1, 'D': 1,
-     'E': 1, 'F': 2, 'G': 1, 'H': 1,
-     'I': 1, 'J': 1, 'K': 1, 'L': 1,
-     'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
-     'U': 4, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '[': 15, '\\': 19,
-     ']': 15, '^': 14, '_': 1, '`': 15,
-     'a': 1, 'b': 2, 'c': 1, 'd': 1,
-     'e': 1, 'f': 2, 'g': 1, 'h': 1,
-     'i': 1, 'j': 1, 'k': 1, 'l': 1,
-     'm': 1, 'n': 1, 'o': 1, 'p': 1,
-     'q': 1, 'r': 3, 's': 1, 't': 1,
-     'u': 4, 'v': 1, 'w': 1, 'x': 1,
-     'y': 1, 'z': 1, '{': 15, '|': 14,
-     '}': 15, '~': 15, '\x80': 1},
+     '\r': 16, ' ': 0, '!': 11, '"': 19,
+     '#': 21, '$': 17, '%': 14, '&': 14,
+     "'": 18, '(': 15, ')': 15, '*': 8,
+     '+': 14, ',': 15, '-': 12, '.': 7,
+     '/': 13, '0': 5, '1': 6, '2': 6,
+     '3': 6, '4': 6, '5': 6, '6': 6,
+     '7': 6, '8': 6, '9': 6, ':': 15,
+     ';': 15, '<': 10, '=': 14, '>': 9,
+     '@': 14, 'A': 1, 'B': 2, 'C': 1,
+     'D': 1, 'E': 1, 'F': 2, 'G': 1,
+     'H': 1, 'I': 1, 'J': 1, 'K': 1,
+     'L': 1, 'M': 1, 'N': 1, 'O': 1,
+     'P': 1, 'Q': 1, 'R': 3, 'S': 1,
+     'T': 1, 'U': 4, 'V': 1, 'W': 1,
+     'X': 1, 'Y': 1, 'Z': 1, '[': 15,
+     '\\': 20, ']': 15, '^': 14, '_': 1,
+     '`': 15, 'a': 1, 'b': 2, 'c': 1,
+     'd': 1, 'e': 1, 'f': 2, 'g': 1,
+     'h': 1, 'i': 1, 'j': 1, 'k': 1,
+     'l': 1, 'm': 1, 'n': 1, 'o': 1,
+     'p': 1, 'q': 1, 'r': 3, 's': 1,
+     't': 1, 'u': 4, 'v': 1, 'w': 1,
+     'x': 1, 'y': 1, 'z': 1, '{': 15,
+     '|': 14, '}': 15, '~': 15,
+     '\x80': 1},
     # 1
     {'0': 1, '1': 1, '2': 1, '3': 1,
      '4': 1, '5': 1, '6': 1, '7': 1,
@@ -56,7 +57,7 @@
      't': 1, 'u': 1, 'v': 1, 'w': 1,
      'x': 1, 'y': 1, 'z': 1, '\x80': 1},
     # 2
-    {'"': 18, "'": 17, '0': 1, '1': 1,
+    {'"': 19, "'": 18, '0': 1, '1': 1,
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 1, 'C': 1, 'D': 1,
@@ -74,7 +75,7 @@
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1, '\x80': 1},
     # 3
-    {'"': 18, "'": 17, '0': 1, '1': 1,
+    {'"': 19, "'": 18, '0': 1, '1': 1,
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 4, 'C': 1, 'D': 1,
@@ -92,7 +93,7 @@
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1, '\x80': 1},
     # 4
-    {'"': 18, "'": 17, '0': 1, '1': 1,
+    {'"': 19, "'": 18, '0': 1, '1': 1,
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 1, 'C': 1, 'D': 1,
@@ -110,21 +111,21 @@
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1, '\x80': 1},
     # 5
-    {'.': 26, '0': 24, '1': 25, '2': 25,
-     '3': 25, '4': 25, '5': 25, '6': 25,
-     '7': 25, '8': 25, '9': 25, 'B': 23,
-     'E': 27, 'J': 15, 'O': 22, 'X': 21,
-     'b': 23, 'e': 27, 'j': 15, 'o': 22,
-     'x': 21},
+    {'.': 27, '0': 25, '1': 26, '2': 26,
+     '3': 26, '4': 26, '5': 26, '6': 26,
+     '7': 26, '8': 26, '9': 26, 'B': 24,
+     'E': 28, 'J': 15, 'O': 23, 'X': 22,
+     'b': 24, 'e': 28, 'j': 15, 'o': 23,
+     'x': 22},
     # 6
-    {'.': 26, '0': 6, '1': 6, '2': 6,
+    {'.': 27, '0': 6, '1': 6, '2': 6,
      '3': 6, '4': 6, '5': 6, '6': 6,
-     '7': 6, '8': 6, '9': 6, 'E': 27,
-     'J': 15, 'e': 27, 'j': 15},
+     '7': 6, '8': 6, '9': 6, 'E': 28,
+     'J': 15, 'e': 28, 'j': 15},
     # 7
-    {'.': 29, '0': 28, '1': 28, '2': 28,
-     '3': 28, '4': 28, '5': 28, '6': 28,
-     '7': 28, '8': 28, '9': 28},
+    {'.': 30, '0': 29, '1': 29, '2': 29,
+     '3': 29, '4': 29, '5': 29, '6': 29,
+     '7': 29, '8': 29, '9': 29},
     # 8
     {'*': 14, '=': 15},
     # 9
@@ -144,107 +145,115 @@
     # 16
     {'\n': 15},
     # 17
-    {automata.DEFAULT: 33, '\n': 30,
-     '\r': 30, "'": 31, '\\': 32},
+    {'0': 31, '1': 31, '2': 31, '3': 31,
+     '4': 31, '5': 31, '6': 31, '7': 31,
+     '8': 31, '9': 31},
     # 18
-    {automata.DEFAULT: 36, '\n': 30,
-     '\r': 30, '"': 34, '\\': 35},
+    {automata.DEFAULT: 35, '\n': 32,
+     '\r': 32, "'": 33, '\\': 34},
     # 19
+    {automata.DEFAULT: 38, '\n': 32,
+     '\r': 32, '"': 36, '\\': 37},
+    # 20
     {'\n': 15, '\r': 16},
-    # 20
-    {automata.DEFAULT: 20, '\n': 30, '\r': 30},
     # 21
-    {'0': 37, '1': 37, '2': 37, '3': 37,
-     '4': 37, '5': 37, '6': 37, '7': 37,
-     '8': 37, '9': 37, 'A': 37, 'B': 37,
-     'C': 37, 'D': 37, 'E': 37, 'F': 37,
-     'a': 37, 'b': 37, 'c': 37, 'd': 37,
-     'e': 37, 'f': 37},
+    {automata.DEFAULT: 21, '\n': 32, '\r': 32},
     # 22
-    {'0': 38, '1': 38, '2': 38, '3': 38,
-     '4': 38, '5': 38, '6': 38, '7': 38},
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39, 'A': 39, 'B': 39,
+     'C': 39, 'D': 39, 'E': 39, 'F': 39,
+     'a': 39, 'b': 39, 'c': 39, 'd': 39,
+     'e': 39, 'f': 39},
     # 23
-    {'0': 39, '1': 39},
+    {'0': 40, '1': 40, '2': 40, '3': 40,
+     '4': 40, '5': 40, '6': 40, '7': 40},
     # 24
-    {'.': 26, '0': 24, '1': 25, '2': 25,
-     '3': 25, '4': 25, '5': 25, '6': 25,
-     '7': 25, '8': 25, '9': 25, 'E': 27,
-     'J': 15, 'e': 27, 'j': 15},
+    {'0': 41, '1': 41},
     # 25
-    {'.': 26, '0': 25, '1': 25, '2': 25,
-     '3': 25, '4': 25, '5': 25, '6': 25,
-     '7': 25, '8': 25, '9': 25, 'E': 27,
-     'J': 15, 'e': 27, 'j': 15},
+    {'.': 27, '0': 25, '1': 26, '2': 26,
+     '3': 26, '4': 26, '5': 26, '6': 26,
+     '7': 26, '8': 26, '9': 26, 'E': 28,
+     'J': 15, 'e': 28, 'j': 15},
     # 26
-    {'0': 26, '1': 26, '2': 26, '3': 26,
-     '4': 26, '5': 26, '6': 26, '7': 26,
-     '8': 26, '9': 26, 'E': 40, 'J': 15,
-     'e': 40, 'j': 15},
+    {'.': 27, '0': 26, '1': 26, '2': 26,
+     '3': 26, '4': 26, '5': 26, '6': 26,
+     '7': 26, '8': 26, '9': 26, 'E': 28,
+     'J': 15, 'e': 28, 'j': 15},
     # 27
-    {'+': 41, '-': 41, '0': 42, '1': 42,
-     '2': 42, '3': 42, '4': 42, '5': 42,
-     '6': 42, '7': 42, '8': 42, '9': 42},
+    {'0': 27, '1': 27, '2': 27, '3': 27,
+     '4': 27, '5': 27, '6': 27, '7': 27,
+     '8': 27, '9': 27, 'E': 42, 'J': 15,
+     'e': 42, 'j': 15},
     # 28
-    {'0': 28, '1': 28, '2': 28, '3': 28,
-     '4': 28, '5': 28, '6': 28, '7': 28,
-     '8': 28, '9': 28, 'E': 40, 'J': 15,
-     'e': 40, 'j': 15},
+    {'+': 43, '-': 43, '0': 44, '1': 44,
+     '2': 44, '3': 44, '4': 44, '5': 44,
+     '6': 44, '7': 44, '8': 44, '9': 44},
     # 29
+    {'0': 29, '1': 29, '2': 29, '3': 29,
+     '4': 29, '5': 29, '6': 29, '7': 29,
+     '8': 29, '9': 29, 'E': 42, 'J': 15,
+     'e': 42, 'j': 15},
+    # 30
     {'.': 15},
-    # 30
+    # 31
+    {'0': 31, '1': 31, '2': 31, '3': 31,
+     '4': 31, '5': 31, '6': 31, '7': 31,
+     '8': 31, '9': 31},
+    # 32
     {},
-    # 31
+    # 33
     {"'": 15},
-    # 32
-    {automata.DEFAULT: 43, '\n': 15, '\r': 16},
-    # 33
-    {automata.DEFAULT: 33, '\n': 30,
-     '\r': 30, "'": 15, '\\': 32},
     # 34
+    {automata.DEFAULT: 45, '\n': 15, '\r': 16},
+    # 35
+    {automata.DEFAULT: 35, '\n': 32,
+     '\r': 32, "'": 15, '\\': 34},
+    # 36
     {'"': 15},
-    # 35
-    {automata.DEFAULT: 44, '\n': 15, '\r': 16},
-    # 36
-    {automata.DEFAULT: 36, '\n': 30,
-     '\r': 30, '"': 15, '\\': 35},
     # 37
-    {'0': 37, '1': 37, '2': 37, '3': 37,
-     '4': 37, '5': 37, '6': 37, '7': 37,
-     '8': 37, '9': 37, 'A': 37, 'B': 37,
-     'C': 37, 'D': 37, 'E': 37, 'F': 37,
-     'a': 37, 'b': 37, 'c': 37, 'd': 37,
-     'e': 37, 'f': 37},
+    {automata.DEFAULT: 46, '\n': 15, '\r': 16},
     # 38
-    {'0': 38, '1': 38, '2': 38, '3': 38,
-     '4': 38, '5': 38, '6': 38, '7': 38},
+    {automata.DEFAULT: 38, '\n': 32,
+     '\r': 32, '"': 15, '\\': 37},
     # 39
-    {'0': 39, '1': 39},
+    {'0': 39, '1': 39, '2': 39, '3': 39,
+     '4': 39, '5': 39, '6': 39, '7': 39,
+     '8': 39, '9': 39, 'A': 39, 'B': 39,
+     'C': 39, 'D': 39, 'E': 39, 'F': 39,
+     'a': 39, 'b': 39, 'c': 39, 'd': 39,
+     'e': 39, 'f': 39},
     # 40
-    {'+': 45, '-': 45, '0': 46, '1': 46,
-     '2': 46, '3': 46, '4': 46, '5': 46,
-     '6': 46, '7': 46, '8': 46, '9': 46},
+    {'0': 40, '1': 40, '2': 40, '3': 40,
+     '4': 40, '5': 40, '6': 40, '7': 40},
     # 41
-    {'0': 42, '1': 42, '2': 42, '3': 42,
-     '4': 42, '5': 42, '6': 42, '7': 42,
-     '8': 42, '9': 42},
+    {'0': 41, '1': 41},
     # 42
-    {'0': 42, '1': 42, '2': 42, '3': 42,
-     '4': 42, '5': 42, '6': 42, '7': 42,
-     '8': 42, '9': 42, 'J': 15, 'j': 15},
+    {'+': 47, '-': 47, '0': 48, '1': 48,
+     '2': 48, '3': 48, '4': 48, '5': 48,
+     '6': 48, '7': 48, '8': 48, '9': 48},
     # 43
-    {automata.DEFAULT: 43, '\n': 30,
-     '\r': 30, "'": 15, '\\': 32},
+    {'0': 44, '1': 44, '2': 44, '3': 44,
+     '4': 44, '5': 44, '6': 44, '7': 44,
+     '8': 44, '9': 44},
     # 44
-    {automata.DEFAULT: 44, '\n': 30,
-     '\r': 30, '"': 15, '\\': 35},
+    {'0': 44, '1': 44, '2': 44, '3': 44,
+     '4': 44, '5': 44, '6': 44, '7': 44,
+     '8': 44, '9': 44, 'J': 15, 'j': 15},
     # 45
-    {'0': 46, '1': 46, '2': 46, '3': 46,
-     '4': 46, '5': 46, '6': 46, '7': 46,
-     '8': 46, '9': 46},
+    {automata.DEFAULT: 45, '\n': 32,
+     '\r': 32, "'": 15, '\\': 34},
     # 46
-    {'0': 46, '1': 46, '2': 46, '3': 46,
-     '4': 46, '5': 46, '6': 46, '7': 46,
-     '8': 46, '9': 46, 'J': 15, 'j': 15},
+    {automata.DEFAULT: 46, '\n': 32,
+     '\r': 32, '"': 15, '\\': 37},
+    # 47
+    {'0': 48, '1': 48, '2': 48, '3': 48,
+     '4': 48, '5': 48, '6': 48, '7': 48,
+     '8': 48, '9': 48},
+    # 48
+    {'0': 48, '1': 48, '2': 48, '3': 48,
+     '4': 48, '5': 48, '6': 48, '7': 48,
+     '8': 48, '9': 48, 'J': 15, 'j': 15},
     ]
 pseudoDFA = automata.DFA(states, accepts)
 
diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/gendfa.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -146,7 +146,10 @@
                     makeEOL(),
                     chainStr(states, "..."),
                     groupStr(states, "@:;.,`"))
-    funny = group(states, operator, bracket, special)
+    revdb_metavar = chain(states,
+                          groupStr(states, "$"),
+                          atleastonce(states, makeDigits()))
+    funny = group(states, operator, bracket, special, revdb_metavar)
     # ____________________________________________________________
     def makeStrPrefix ():
         return group(states,
diff --git a/pypy/interpreter/pyparser/pytoken.py b/pypy/interpreter/pyparser/pytoken.py
--- a/pypy/interpreter/pyparser/pytoken.py
+++ b/pypy/interpreter/pyparser/pytoken.py
@@ -72,5 +72,6 @@
 # extra PyPy-specific tokens
 _add_tok("COMMENT")
 _add_tok("NL")
+_add_tok("REVDBMETAVAR", "$NUM")
 
 del _add_tok
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -329,6 +329,10 @@
                     last_comment = ''
                 elif initial == '\\':                      # continued stmt
                     continued = 1
+                elif initial == '$':
+                    token_list.append((tokens.REVDBMETAVAR, token,
+                                       lnum, start, line))
+                    last_comment = ''
                 else:
                     if initial in '([{':
                         parenstack.append((initial, lnum, start, line))
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -147,6 +147,15 @@
             tree = self.parse(fmt % linefeed)
             assert expected_tree == tree
 
+    def test_revdb_dollar_num(self):
+        self.parse('$0')
+        self.parse('$5')
+        self.parse('$42')
+        self.parse('2+$42.attrname')
+        py.test.raises(SyntaxError, self.parse, '$')
+        py.test.raises(SyntaxError, self.parse, '$a')
+        py.test.raises(SyntaxError, self.parse, '$.5')
+
     def test_py3k_reject_old_binary_literal(self):
         py.test.raises(SyntaxError, self.parse, '0777')
 
diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/reverse_debugging.py
@@ -0,0 +1,852 @@
+import sys
+from rpython.rlib import revdb
+from rpython.rlib.debug import make_sure_not_resized
+from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+from pypy.interpreter.error import OperationError, oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter import gateway, typedef, pycode, pytraceback, pyframe
+from pypy.module.marshal import interp_marshal
+from pypy.interpreter.executioncontext import AbstractActionFlag, ActionFlag
+
+
+class DBState:
+    standard_code = True
+    breakpoint_stack_id = 0
+    breakpoint_funcnames = None
+    breakpoint_filelines = None
+    breakpoint_by_file = None
+    breakpoint_version = 0
+    printed_objects = {}
+    metavars = []
+    watch_progs = []
+    watch_futures = {}
+
+dbstate = DBState()
+
+
+pycode.PyCode.co_revdb_linestarts = None   # or a string: see below
+pycode.PyCode.co_revdb_bkpt_version = 0    # see check_and_trigger_bkpt()
+pycode.PyCode.co_revdb_bkpt_cache = None   # see check_and_trigger_bkpt()
+
+# invariant: "f_revdb_nextline_instr" is the bytecode offset of
+# the start of the line that follows "last_instr".
+pyframe.PyFrame.f_revdb_nextline_instr = -1
+
+
+# ____________________________________________________________
+
+
+def setup_revdb(space):
+    """Called at run-time, before the space is set up.
+
+    The various register_debug_command() lines attach functions
+    to some commands that 'revdb.py' can call, if we are running
+    in replay mode.
+    """
+    assert space.config.translation.reverse_debugger
+    dbstate.space = space
+
+    make_sure_not_resized(dbstate.watch_progs)
+    make_sure_not_resized(dbstate.metavars)
+
+    revdb.register_debug_command(revdb.CMD_PRINT, lambda_print)
+    revdb.register_debug_command(revdb.CMD_BACKTRACE, lambda_backtrace)
+    revdb.register_debug_command(revdb.CMD_LOCALS, lambda_locals)
+    revdb.register_debug_command(revdb.CMD_BREAKPOINTS, lambda_breakpoints)
+    revdb.register_debug_command(revdb.CMD_STACKID, lambda_stackid)
+    revdb.register_debug_command("ALLOCATING", lambda_allocating)
+    revdb.register_debug_command(revdb.CMD_ATTACHID, lambda_attachid)
+    revdb.register_debug_command(revdb.CMD_COMPILEWATCH, lambda_compilewatch)
+    revdb.register_debug_command(revdb.CMD_CHECKWATCH, lambda_checkwatch)
+    revdb.register_debug_command(revdb.CMD_WATCHVALUES, lambda_watchvalues)
+
+
+# ____________________________________________________________
+
+
+def enter_call(caller_frame, callee_frame):
+    if dbstate.breakpoint_funcnames is not None:
+        name = callee_frame.getcode().co_name
+        if name in dbstate.breakpoint_funcnames:
+            revdb.breakpoint(dbstate.breakpoint_funcnames[name])
+    if dbstate.breakpoint_stack_id != 0 and caller_frame is not None:
+        if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame):
+            revdb.breakpoint(-1)
+    #
+    code = callee_frame.pycode
+    if code.co_revdb_linestarts is None:
+        build_co_revdb_linestarts(code)
+
+def leave_call(caller_frame, got_exception):
+    if dbstate.breakpoint_stack_id != 0 and caller_frame is not None:
+        if dbstate.breakpoint_stack_id == revdb.get_unique_id(caller_frame):
+            revdb.breakpoint(-2)
+    if we_are_translated():
+        stop_point_activate(-2 + got_exception)
+
+def stop_point():
+    if we_are_translated():
+        revdb.breakpoint(-3)
+
+
+def jump_backward(frame, jumpto):
+    # When we see a jump backward, we set 'f_revdb_nextline_instr' in
+    # such a way that the next instruction, at 'jumpto', will trigger
+    # stop_point_activate().  We have to trigger it even if
+    # 'jumpto' is not actually a start of line.  For example, after a
+    # 'while foo:', the body ends with a JUMP_ABSOLUTE which
+    # jumps back to the *second* opcode of the while.
+    frame.f_revdb_nextline_instr = jumpto
+
+
+def potential_stop_point(frame):
+    if not we_are_translated():
+        return
+    #
+    # We only record a stop_point at every line, not every bytecode.
+    # Uses roughly the same algo as ExecutionContext.run_trace_func()
+    # to know where the line starts are, but tweaked for speed,
+    # avoiding the quadratic complexity when run N times with a large
+    # code object.
+    #
+    cur = frame.last_instr
+    if cur < frame.f_revdb_nextline_instr:
+        return    # fast path: we're still inside the same line as before
+    #
+    call_stop_point_at_line = True
+    co_revdb_linestarts = frame.pycode.co_revdb_linestarts
+    if cur > frame.f_revdb_nextline_instr:
+        #
+        # We jumped forward over the start of the next line.  We're
+        # inside a different line, but we will only trigger a stop
+        # point if we're at the starting bytecode of that line.  Fetch
+        # from co_revdb_linestarts the start of the line that is at or
+        # follows 'cur'.
+        ch = ord(co_revdb_linestarts[cur])
+        if ch == 0:
+            pass   # we are at the start of a line now
+        else:
+            # We are not, so don't call stop_point_activate().
+            # We still have to fill f_revdb_nextline_instr.
+            call_stop_point_at_line = False
+    #
+    if call_stop_point_at_line:
+        stop_point_activate(pycode=frame.pycode, opindex=cur)
+        cur += 1
+        ch = ord(co_revdb_linestarts[cur])
+    #
+    # Update f_revdb_nextline_instr.  Check if 'ch' was greater than
+    # 255, in which case it was rounded down to 255 and we have to
+    # continue looking
+    nextline_instr = cur + ch
+    while ch == 255:
+        ch = ord(co_revdb_linestarts[nextline_instr])
+        nextline_instr += ch
+    frame.f_revdb_nextline_instr = nextline_instr
+
+
+def build_co_revdb_linestarts(code):
+    # Inspired by findlinestarts() in the 'dis' standard module.
+    # Set up 'bits' so that it contains \x00 at line starts and \xff
+    # in-between.
+    bits = ['\xff'] * (len(code.co_code) + 1)
+    if not code.hidden_applevel:
+        lnotab = code.co_lnotab
+        addr = 0
+        p = 0
+        newline = 1
+        while p + 1 < len(lnotab):
+            byte_incr = ord(lnotab[p])
+            line_incr = ord(lnotab[p+1])
+            if byte_incr:
+                if newline != 0:
+                    bits[addr] = '\x00'
+                    newline = 0
+                addr += byte_incr
+            newline |= line_incr
+            p += 2
+        if newline:
+            bits[addr] = '\x00'
+    bits[len(code.co_code)] = '\x00'
+    #
+    # Change 'bits' so that the character at 'i', if not \x00, measures
+    # how far the next \x00 is
+    next_null = len(code.co_code)
+    p = next_null - 1
+    while p >= 0:
+        if bits[p] == '\x00':
+            next_null = p
+        else:
+            ch = next_null - p
+            if ch > 255: ch = 255
+            bits[p] = chr(ch)
+        p -= 1
+    lstart = ''.join(bits)
+    code.co_revdb_linestarts = lstart
+    return lstart
+
+def get_final_lineno(code):
+    lineno = code.co_firstlineno
+    lnotab = code.co_lnotab
+    p = 1
+    while p < len(lnotab):
+        line_incr = ord(lnotab[p])
+        lineno += line_incr
+        p += 2
+    return lineno
+
+def find_line_starts(code):
+    # RPython version of dis.findlinestarts()
+    lnotab = code.co_lnotab
+    lastlineno = -1
+    lineno = code.co_firstlineno
+    addr = 0
+    p = 0
+    result = []
+    while p < len(lnotab) - 1:
+        byte_incr = ord(lnotab[p + 0])
+        line_incr = ord(lnotab[p + 1])
+        if byte_incr:
+            if lineno != lastlineno:
+                result.append((addr, lineno))
+                lastlineno = lineno
+            addr += byte_incr
+        lineno += line_incr
+        p += 2
+    if lineno != lastlineno:
+        result.append((addr, lineno))
+    return result
+
+class NonStandardCode(object):
+    def __enter__(self):
+        dbstate.standard_code = False
+        self.t = dbstate.space.actionflag._ticker
+        self.c = dbstate.space.actionflag._ticker_revdb_count
+    def __exit__(self, *args):
+        dbstate.space.actionflag._ticker = self.t
+        dbstate.space.actionflag._ticker_revdb_count = self.c
+        dbstate.standard_code = True
+non_standard_code = NonStandardCode()
+
+
+def stop_point_activate(place=0, pycode=None, opindex=-1):
+    if revdb.watch_save_state():
+        any_watch_point = False
+        # ^^ this flag is set to True if we must continue to enter this
+        # block of code.  If it is still False for watch_restore_state()
+        # below, then future watch_save_state() will return False too---
+        # until the next time revdb.c:set_revdb_breakpoints() is called.
+        space = dbstate.space
+        with non_standard_code:
+            watch_id = -1
+            if dbstate.breakpoint_by_file is not None:
+                any_watch_point = True
+                if pycode is not None:
+                    watch_id = check_and_trigger_bkpt(pycode, opindex)
+            if watch_id == -1:
+                for prog, watch_id, expected in dbstate.watch_progs:
+                    any_watch_point = True
+                    try:
+                        got = _run_watch(space, prog)
+                    except OperationError as e:
+                        got = e.errorstr(space)
+                    except Exception:
+                        break
+                    if got != expected:
+                        break
+                else:
+                    watch_id = -1
+        revdb.watch_restore_state(any_watch_point)
+        if watch_id != -1:
+            revdb.breakpoint(watch_id)
+    revdb.stop_point(place)
+
+
+def future_object(space):
+    return space.w_Ellipsis    # a random prebuilt object
+
+def load_metavar(index):
+    assert index >= 0
+    space = dbstate.space
+    metavars = dbstate.metavars
+    w_var = metavars[index] if index < len(metavars) else None
+    if w_var is None:
+        raise oefmt(space.w_NameError, "no constant object '$%d'",
+                    index)
+    if w_var is future_object(space):
+        raise oefmt(space.w_RuntimeError,
+                    "'$%d' refers to an object created later in time",
+                    index)
+    return w_var
+
+def set_metavar(index, w_obj):
+    assert index >= 0
+    if index >= len(dbstate.metavars):
+        missing = index + 1 - len(dbstate.metavars)
+        dbstate.metavars = dbstate.metavars + [None] * missing
+    dbstate.metavars[index] = w_obj
+
+
+# ____________________________________________________________
+
+
+def fetch_cur_frame(silent=False):
+    ec = dbstate.space.threadlocals.get_ec()
+    if ec is None:
+        frame = None
+    else:
+        frame = ec.topframeref()
+    if frame is None and not silent:
+        revdb.send_output("No stack.\n")
+    return frame
+
+def compile(source, mode):
+    assert not dbstate.standard_code
+    space = dbstate.space
+    compiler = space.createcompiler()
+    code = compiler.compile(source, '<revdb>', mode, 0,
+                            hidden_applevel=True)
+    return code
+
+
+class W_RevDBOutput(W_Root):
+    softspace = 0
+
+    def __init__(self, space):
+        self.space = space
+
+    def descr_write(self, w_buffer):
+        assert not dbstate.standard_code
+        space = self.space
+        if space.isinstance_w(w_buffer, space.w_unicode):
+            w_buffer = space.call_method(w_buffer, 'encode',
+                                         space.newtext('utf-8'))   # safe?
+        revdb.send_output(space.bytes_w(w_buffer))
+
+def descr_get_softspace(space, revdb):
+    return space.newint(revdb.softspace)
+def descr_set_softspace(space, revdb, w_newvalue):
+    revdb.softspace = space.int_w(w_newvalue)
+
+W_RevDBOutput.typedef = typedef.TypeDef(
+    "revdb_output",
+    write = gateway.interp2app(W_RevDBOutput.descr_write),
+    # XXX is 'softspace' still necessary in Python 3?
+    softspace = typedef.GetSetProperty(descr_get_softspace,
+                                       descr_set_softspace,
+                                       cls=W_RevDBOutput),
+    )
+
+def revdb_displayhook(space, w_obj):
+    """Modified sys.displayhook() that also outputs '$NUM = ',
+    for non-prebuilt objects.  Such objects are then recorded in
+    'printed_objects'.
+    """
+    assert not dbstate.standard_code
+    if space.is_w(w_obj, space.w_None):
+        return
+    uid = revdb.get_unique_id(w_obj)
+    if uid > 0:
+        dbstate.printed_objects[uid] = w_obj
+        revdb.send_nextnid(uid)   # outputs '$NUM = '
+    space.setitem(space.builtin.w_dict, space.newtext('_'), w_obj)
+    # do repr() after setitem: if w_obj was produced successfully,
+    # but its repr crashes because it tries to do I/O, then we already
+    # have it recorded in '_' and in '$NUM ='.
+    w_repr = space.repr(w_obj)
+    if space.isinstance_w(w_repr, space.w_unicode):
+        w_repr = space.call_method(w_repr, 'encode',
+                                   space.newtext('utf-8'))   # safe?
+    revdb.send_output(space.bytes_w(w_repr))
+    revdb.send_output("\n")
+
+ at gateway.unwrap_spec(name='text0', level=int)
+def revdb_importhook(space, name, w_globals=None,
+                     w_locals=None, w_fromlist=None, level=-1):
+    # Incredibly simplified version of __import__, which only returns
+    # already-imported modules and doesn't call any custom import
+    # hooks.  Recognizes only absolute imports.  With a 'fromlist'
+    # argument that is a non-empty list, returns the module 'name3' if
+    # the 'name' argument is 'name1.name2.name3'.  With an empty or
+    # None 'fromlist' argument, returns the module 'name1' instead.
+    return space.appexec([space.newtext(name), w_fromlist or space.w_None,
+                          space.newint(level), space.sys],
+    """(name, fromlist, level, sys):
+        if level > 0:
+            raise ImportError("only absolute imports are "
+                               "supported in the debugger")
+        basename = name.split('.')[0]
+        try:
+            basemod = sys.modules[basename]
+            mod = sys.modules[name]
+        except KeyError:
+            raise ImportError("'%s' not found or not imported yet "
+                    "(the debugger can't import new modules, "
+                    "and only supports absolute imports)" % (name,))
+        if fromlist:
+            return mod
+        return basemod
+    """)
+
+ at specialize.memo()
+def get_revdb_displayhook(space):
+    return gateway.interp2app(revdb_displayhook).spacebind(space)
+
+ at specialize.memo()
+def get_revdb_importhook(space):
+    return gateway.interp2app(revdb_importhook).spacebind(space)
+
+
+def prepare_print_environment(space):
+    assert not dbstate.standard_code
+    w_revdb_output = W_RevDBOutput(space)
+    w_displayhook = get_revdb_displayhook(space)
+    w_import = get_revdb_importhook(space)
+    space.sys.setdictvalue(space, 'stdout', w_revdb_output)
+    space.sys.setdictvalue(space, 'stderr', w_revdb_output)
+    space.sys.setdictvalue(space, 'displayhook', w_displayhook)
+    space.builtin.setdictvalue(space, '__import__', w_import)
+
+def command_print(cmd, expression):
+    frame = fetch_cur_frame()
+    if frame is None:
+        return
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            prepare_print_environment(space)
+            code = compile(expression, 'single')
+            try:
+                code.exec_code(space,
+                               frame.get_w_globals(),
+                               frame.getdictscope())
+
+            except OperationError as operationerr:
+                # can't use sys.excepthook: it will likely try to do 'import
+                # traceback', which might not be doable without using I/O
+                tb = operationerr.get_traceback()
+                if tb is not None:
+                    revdb.send_output("Traceback (most recent call last):\n")
+                    while tb is not None:
+                        if not isinstance(tb, pytraceback.PyTraceback):
+                            revdb.send_output("  ??? %s\n" % tb)
+                            break
+                        show_frame(tb.frame, tb.get_lineno(), indent='  ')
+                        tb = tb.next
+                revdb.send_output('%s\n' % operationerr.errorstr(space))
+
+                # set the sys.last_xxx attributes
+                w_type = operationerr.w_type
+                w_value = operationerr.get_w_value(space)
+                w_tb = operationerr.get_traceback()
+                w_dict = space.sys.w_dict
+                space.setitem(w_dict, space.newtext('last_type'), w_type)
+                space.setitem(w_dict, space.newtext('last_value'), w_value)
+                space.setitem(w_dict, space.newtext('last_traceback'), w_tb)
+
+        except OperationError as e:
+            revdb.send_output('%s\n' % e.errorstr(space, use_repr=True))
+lambda_print = lambda: command_print
+
+
+def file_and_lineno(frame, lineno):
+    code = frame.getcode()
+    return 'File "%s", line %d in %s' % (
+        code.co_filename, lineno, code.co_name)
+
+def show_frame(frame, lineno=0, indent=''):
+    if lineno == 0:
+        lineno = frame.get_last_lineno()
+    revdb.send_output("%s%s\n%s  " % (
+        indent,
+        file_and_lineno(frame, lineno),
+        indent))
+    revdb.send_linecache(frame.getcode().co_filename, lineno)
+
+def display_function_part(frame, max_lines_before, max_lines_after):
+    if frame is None:
+        return
+    code = frame.getcode()
+    if code.co_filename.startswith('<builtin>'):
+        return
+    first_lineno = code.co_firstlineno
+    current_lineno = frame.get_last_lineno()
+    final_lineno = get_final_lineno(code)
+    #
+    ellipsis_after = False
+    if first_lineno < current_lineno - max_lines_before - 1:
+        first_lineno = current_lineno - max_lines_before
+        revdb.send_output("...\n")
+    if final_lineno > current_lineno + max_lines_after + 1:
+        final_lineno = current_lineno + max_lines_after
+        ellipsis_after = True
+    #
+    for i in range(first_lineno, final_lineno + 1):
+        if i == current_lineno:
+            if revdb.current_place() == -2: # <= this is the arg to stop_point()
+                prompt = "<< "     # return
+            elif revdb.current_place() == -1:
+                prompt = "!! "     # exceptional return
+            else:
+                prompt = " > "     # plain line
+            revdb.send_output(prompt)
+        else:
+            revdb.send_output("   ")
+        revdb.send_linecache(code.co_filename, i, strip=False)
+    #
+    if ellipsis_after:
+        revdb.send_output("...\n")
+
+def command_backtrace(cmd, extra):
+    frame = fetch_cur_frame(silent=True)
+    if cmd.c_arg1 == 0:
+        if frame is not None:
+            revdb.send_output("%s:\n" % (
+                file_and_lineno(frame, frame.get_last_lineno()),))
+        display_function_part(frame, max_lines_before=8, max_lines_after=5)
+    elif cmd.c_arg1 == 2:
+        display_function_part(frame, max_lines_before=1000,max_lines_after=1000)
+    else:
+        revdb.send_output("Current call stack (most recent call last):\n")
+        if frame is None:
+            revdb.send_output("  (empty)\n")
+        frames = []
+        while frame is not None:
+            frames.append(frame)
+            if len(frames) == 200:
+                revdb.send_output("  ...\n")
+                break
+            frame = frame.get_f_back()
+        while len(frames) > 0:
+            show_frame(frames.pop(), indent='  ')
+lambda_backtrace = lambda: command_backtrace
+
+
+def command_locals(cmd, extra):
+    frame = fetch_cur_frame()
+    if frame is None:
+        return
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            prepare_print_environment(space)
+            space.appexec([space.sys,
+                           frame.getdictscope()], """(sys, locals):
+                lst = sorted(locals.keys())
+                print('Locals:')
+                for key in lst:
+                    try:
+                        print('    %s =' % key, end=' ', flush=True)
+                        s = '%r' % locals[key]
+                        if len(s) > 140:
+                            s = s[:100] + '...' + s[-30:]
+                        print(s)
+                    except:
+                        exc, val, tb = sys.exc_info()
+                        print('!<%s: %r>' % (exc, val))
+            """)
+        except OperationError as e:
+            revdb.send_output('%s\n' % e.errorstr(space, use_repr=True))
+lambda_locals = lambda: command_locals
+
+# ____________________________________________________________
+
+
+def check_and_trigger_bkpt(pycode, opindex):
+    # We cache on 'pycode.co_revdb_bkpt_cache' either None or a dict
+    # mapping {opindex: bkpt_num}.  This cache is updated when the
+    # version in 'pycode.co_revdb_bkpt_version' does not match
+    # 'dbstate.breakpoint_version' any more.
+    if pycode.co_revdb_bkpt_version != dbstate.breakpoint_version:
+        update_bkpt_cache(pycode)
+    cache = pycode.co_revdb_bkpt_cache
+    if cache is not None and opindex in cache:
+        return cache[opindex]
+    else:
+        return -1
+
+def update_bkpt_cache(pycode):
+    # initialized by command_breakpoints():
+    #     dbstate.breakpoint_filelines == [('FILENAME', lineno, bkpt_num)]
+    # computed lazily (here, first half of the logic):
+    #     dbstate.breakpoint_by_file == {'co_filename': {lineno: bkpt_num}}
+    # the goal is to set:
+    #     pycode.co_revdb_bkpt_cache == {opindex: bkpt_num}
+
+    co_filename = pycode.co_filename
+    try:
+        linenos = dbstate.breakpoint_by_file[co_filename]
+    except KeyError:
+        linenos = None
+        match = co_filename.upper()    # ignore cAsE in filename matching
+        for filename, lineno, bkpt_num in dbstate.breakpoint_filelines:
+            if match.endswith(filename) and (
+                    len(match) == len(filename) or
+                    match[-len(filename)-1] in '/\\'):    # a valid prefix
+                if linenos is None:
+                    linenos = {}
+                linenos[lineno] = bkpt_num
+        dbstate.breakpoint_by_file[co_filename] = linenos
+
+    newcache = None
+    if linenos is not None:
+        # parse co_lnotab to figure out the opindexes that correspond
+        # to the marked line numbers.  here, linenos == {lineno: bkpt_num}
+        for addr, lineno in find_line_starts(pycode):
+            if lineno in linenos:
+                if newcache is None:
+                    newcache = {}
+                newcache[addr] = linenos[lineno]
+
+    pycode.co_revdb_bkpt_cache = newcache
+    pycode.co_revdb_bkpt_version = dbstate.breakpoint_version
+
+
+def valid_identifier(s):
+    if not s:
+        return False
+    if s[0].isdigit():
+        return False
+    for c in s:
+        if not (c.isalnum() or c == '_'):
+            return False
+    return True
+
+def add_breakpoint_funcname(name, i):
+    if dbstate.breakpoint_funcnames is None:
+        dbstate.breakpoint_funcnames = {}
+    dbstate.breakpoint_funcnames[name] = i
+
+def add_breakpoint_fileline(filename, lineno, i):
+    # dbstate.breakpoint_filelines is just a list of (FILENAME, lineno, i).
+    # dbstate.breakpoint_by_file is {co_filename: {lineno: i}}, but
+    # computed lazily when we encounter a code object with the given
+    # co_filename.  Any suffix 'filename' matches 'co_filename'.
+    if dbstate.breakpoint_filelines is None:
+        dbstate.breakpoint_filelines = []
+        dbstate.breakpoint_by_file = {}
+    dbstate.breakpoint_filelines.append((filename.upper(), lineno, i))
+
+def add_breakpoint(name, i):
+    # if it is empty, complain
+    if not name:
+        revdb.send_output("Empty breakpoint name\n")
+        revdb.send_change_breakpoint(i)
+        return
+    # if it is surrounded by < >, it is the name of a code object
+    if name.startswith('<') and name.endswith('>'):
+        add_breakpoint_funcname(name, i)
+        return
+    # if it has no ':', it can be a valid identifier (which we
+    # register as a function name), or a lineno
+    original_name = name
+    j = name.rfind(':')
+    if j < 0:
+        try:
+            lineno = int(name)
+        except ValueError:
+            if name.endswith('()'):
+                n = len(name) - 2
+                assert n >= 0
+                name = name[:n]
+            if not valid_identifier(name):
+                revdb.send_output(
+                    'Note: "%s()" doesn''t look like a function name. '
+                    'Setting breakpoint anyway\n' % name)
+            add_breakpoint_funcname(name, i)
+            name += '()'
+            if name != original_name:
+                revdb.send_change_breakpoint(i, name)
+            return
+        # "number" does the same as ":number"
+        filename = ''
+    else:
+        # if it has a ':', it must end in ':lineno'
+        try:
+            lineno = int(name[j+1:])
+        except ValueError:
+            revdb.send_output('expected a line number after colon\n')
+            revdb.send_change_breakpoint(i)
+            return
+        filename = name[:j]
+
+    # the text before must be a pathname, possibly a relative one,
+    # or be escaped by < >.  if it isn't, make it absolute and normalized
+    # and warn if it doesn't end in '.py'.
+    if filename == '':
+        frame = fetch_cur_frame()
+        if frame is None:
+            revdb.send_change_breakpoint(i)
+            return
+        filename = frame.getcode().co_filename
+    elif filename.startswith('<') and filename.endswith('>'):
+        pass    # use unmodified
+    elif not filename.lower().endswith('.py'):
+        # use unmodified, but warn
+        revdb.send_output(
+            'Note: "%s" doesn''t look like a Python filename. '
+            'Setting breakpoint anyway\n' % (filename,))
+
+    add_breakpoint_fileline(filename, lineno, i)
+    name = '%s:%d' % (filename, lineno)
+    if name != original_name:
+        revdb.send_change_breakpoint(i, name)
+
+def command_breakpoints(cmd, extra):
+    space = dbstate.space
+    dbstate.breakpoint_stack_id = cmd.c_arg1
+    revdb.set_thread_breakpoint(cmd.c_arg2)
+    dbstate.breakpoint_funcnames = None
+    dbstate.breakpoint_filelines = None
+    dbstate.breakpoint_by_file = None
+    dbstate.breakpoint_version += 1
+    watch_progs = []
+    with non_standard_code:
+        for i, kind, name in revdb.split_breakpoints_arg(extra):
+            if kind == 'B':
+                add_breakpoint(name, i)
+            elif kind == 'W':
+                code = interp_marshal.loads(space, space.newbytes(name))
+                watch_progs.append((code, i, ''))
+    dbstate.watch_progs = watch_progs[:]
+lambda_breakpoints = lambda: command_breakpoints
+
+
+def command_watchvalues(cmd, extra):
+    expected = extra.split('\x00')
+    for j in range(len(dbstate.watch_progs)):
+        prog, i, _ = dbstate.watch_progs[j]
+        if i >= len(expected):
+            raise IndexError
+        dbstate.watch_progs[j] = prog, i, expected[i]
+lambda_watchvalues = lambda: command_watchvalues
+
+
+def command_stackid(cmd, extra):
+    frame = fetch_cur_frame(silent=True)
+    if frame is not None and cmd.c_arg1 != 0:     # parent_flag
+        frame = dbstate.space.getexecutioncontext().getnextframe_nohidden(frame)
+    if frame is None:
+        uid = 0
+    else:
+        uid = revdb.get_unique_id(frame)
+    if revdb.current_place() == -2:
+        hidden_level = 1      # hide the "<<" events from next/bnext commands
+    else:
+        hidden_level = 0
+    revdb.send_answer(revdb.ANSWER_STACKID, uid, hidden_level)
+lambda_stackid = lambda: command_stackid
+
+
+def command_allocating(uid, gcref):
+    w_obj = cast_gcref_to_instance(W_Root, gcref)
+    dbstate.printed_objects[uid] = w_obj
+    try:
+        index_metavar = dbstate.watch_futures.pop(uid)
+    except KeyError:
+        pass
+    else:
+        set_metavar(index_metavar, w_obj)
+lambda_allocating = lambda: command_allocating
+
+
+def command_attachid(cmd, extra):
+    space = dbstate.space
+    index_metavar = cmd.c_arg1
+    uid = cmd.c_arg2
+    try:
+        w_obj = dbstate.printed_objects[uid]
+    except KeyError:
+        # uid not found, probably a future object
+        dbstate.watch_futures[uid] = index_metavar
+        w_obj = future_object(space)
+    set_metavar(index_metavar, w_obj)
+lambda_attachid = lambda: command_attachid
+
+
+def command_compilewatch(cmd, expression):
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            code = compile(expression, 'eval')
+            marshalled_code = space.bytes_w(interp_marshal.dumps(
+                space, code,
+                space.newint(interp_marshal.Py_MARSHAL_VERSION)))
+        except OperationError as e:
+            revdb.send_watch(e.errorstr(space), ok_flag=0)
+        else:
+            revdb.send_watch(marshalled_code, ok_flag=1)
+lambda_compilewatch = lambda: command_compilewatch
+
+def command_checkwatch(cmd, marshalled_code):
+    space = dbstate.space
+    with non_standard_code:
+        try:
+            code = interp_marshal.loads(space, space.newbytes(marshalled_code))
+            text = _run_watch(space, code)
+        except OperationError as e:
+            revdb.send_watch(e.errorstr(space), ok_flag=0)
+        else:
+            revdb.send_watch(text, ok_flag=1)
+lambda_checkwatch = lambda: command_checkwatch
+
+
+def _run_watch(space, prog):
+    # must be called from non_standard_code!
+    w_dict = space.builtin.w_dict
+    w_res = prog.exec_code(space, w_dict, w_dict)
+    return space.text_w(space.repr(w_res))
+
+
+# ____________________________________________________________
+
+
+ActionFlag._ticker_revdb_count = -1
+
+class RDBSignalActionFlag(AbstractActionFlag):
+    # Used instead of pypy.module.signal.interp_signal.SignalActionFlag
+    # when we have reverse-debugging.  That other class would work too,
+    # but inefficiently: it would generate two words of data per bytecode.
+    # This class is tweaked to generate one byte per _SIG_TICKER_COUNT
+    # bytecodes, at the expense of not reacting to signals instantly.
+
+    # Threads: after 10'000 calls to decrement_ticker(), it should
+    # return -1.  It should also return -1 if there was a signal.
+    # This is done by calling _update_ticker_from_signals() every 100
+    # calls, and invoking rsignal.pypysig_check_and_reset(); this in
+    # turn returns -1 if there was a signal or if it was called 100
+    # times.
+
+    _SIG_TICKER_COUNT = 100
+    _ticker = 0
+    _ticker_revdb_count = _SIG_TICKER_COUNT * 10
+
+    def get_ticker(self):
+        return self._ticker
+
+    def reset_ticker(self, value):
+        self._ticker = value
+
+    def rearm_ticker(self):
+        self._ticker = -1
+
+    def decrement_ticker(self, by):
+        if we_are_translated():
+            c = self._ticker_revdb_count - 1
+            if c < 0:
+                c = self._update_ticker_from_signals()
+            self._ticker_revdb_count = c
+        #if self.has_bytecode_counter:    # this 'if' is constant-folded
+        #    print ("RDBSignalActionFlag: has_bytecode_counter: "
+        #           "not supported for now")
+        #    raise NotImplementedError
+        return self._ticker
+
+    def _update_ticker_from_signals(self):
+        from rpython.rlib import rsignal
+        if dbstate.standard_code:
+            if rsignal.pypysig_check_and_reset():
+                self.rearm_ticker()
+        return self._SIG_TICKER_COUNT
+    _update_ticker_from_signals._dont_inline_ = True
diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/test_reverse_debugging.py
@@ -0,0 +1,121 @@
+import dis
+from pypy.interpreter.reverse_debugging import *
+from pypy.interpreter import reverse_debugging
+from rpython.rlib import revdb
+
+
+class FakeCode:
+    hidden_applevel = False
+    def __init__(self, co_code='', co_lnotab='', co_filename='?'):
+        self.co_firstlineno = 43
+        self.co_code = co_code
+        self.co_lnotab = co_lnotab
+        self.co_revdb_linestarts = None
+        self.co_filename = co_filename
+
+
+try:
+    from hypothesis import given, strategies, example
+except ImportError:
+    pass
+else:
+    @given(strategies.binary())
+    @example("\x01\x02\x03\x04"
+             "\x00\xFF\x20\x30\x00\xFF\x00\x40"
+             "\xFF\x00\x0A\x0B\xFF\x00\x0C\x00")
+    def test_build_co_revdb_linestarts(lnotab):
+        if len(lnotab) & 1:
+            lnotab = lnotab + '\x00'   # make the length even
+        code = FakeCode("?" * sum(map(ord, lnotab[0::2])), lnotab)
+        lstart = build_co_revdb_linestarts(code)
+        assert lstart is code.co_revdb_linestarts
+
+        expected_starts = set()
+        for addr, lineno in dis.findlinestarts(code):
+            expected_starts.add(addr)
+
+        next_start = len(code.co_code)
+        for index in range(len(code.co_code), -1, -1):
+            if index in expected_starts:
+                next_start = index
+            assert lstart[index] == chr(next_start - index
+                                        if next_start - index <= 255
+                                        else 255)
+
+
+class FakeFrame:
+    def __init__(self, code):
+        self.__code = code
+    def getcode(self):
+        return self.__code
+
+def check_add_breakpoint(input, curfilename=None,
+                         expected_funcname=None,
+                         expected_fileline=None,
+                         expected_output=None,
+                         expected_chbkpt=None):
+    dbstate.__dict__.clear()
+    prev = revdb.send_answer, reverse_debugging.fetch_cur_frame
+    try:
+        messages = []
+        def got_message(cmd, arg1=0, arg2=0, arg3=0, extra=""):
+            messages.append((cmd, arg1, arg2, arg3, extra))
+        def my_cur_frame():
+            assert curfilename is not None
+            return FakeFrame(FakeCode(co_filename=curfilename))
+        revdb.send_answer = got_message
+        reverse_debugging.fetch_cur_frame = my_cur_frame
+        add_breakpoint(input, 5)
+    finally:
+        revdb.send_answer, reverse_debugging.fetch_cur_frame = prev
+
+    if expected_funcname is None:
+        assert dbstate.breakpoint_funcnames is None
+    else:
+        assert dbstate.breakpoint_funcnames == {expected_funcname: 5}
+
+    if expected_fileline is None:
+        assert dbstate.breakpoint_filelines is None
+    else:
+        filename, lineno = expected_fileline
+        assert dbstate.breakpoint_filelines == [(filename.upper(), lineno, 5)]
+
+    got_output = None
+    got_chbkpt = None
+    for msg in messages:
+        if msg[0] == revdb.ANSWER_TEXT:
+            assert got_output is None
+            got_output = msg[-1]
+        elif msg[0] == revdb.ANSWER_CHBKPT:
+            assert got_chbkpt is None
+            assert msg[1] == 5
+            got_chbkpt = msg[-1]
+
+    assert got_output == expected_output
+    assert got_chbkpt == expected_chbkpt
+
+def test_add_breakpoint():
+    check_add_breakpoint('', expected_output="Empty breakpoint name\n",
+                         expected_chbkpt='')
+    check_add_breakpoint('foo42', expected_funcname="foo42",
+                         expected_chbkpt="foo42()")
+    check_add_breakpoint('foo42()', expected_funcname="foo42")
+    check_add_breakpoint('foo.bar', expected_funcname="foo.bar",
+        expected_output='Note: "foo.bar()" doesn''t look like a function name.'
+                        ' Setting breakpoint anyway\n',
+        expected_chbkpt="foo.bar()")
+    check_add_breakpoint('<foo.bar>', expected_funcname="<foo.bar>")
+    check_add_breakpoint('42', curfilename='abcd',
+                         expected_fileline=('abcd', 42),
+                         expected_chbkpt='abcd:42')
+    check_add_breakpoint(':42', curfilename='abcd',
+                         expected_fileline=('abcd', 42),
+                         expected_chbkpt='abcd:42')
+    check_add_breakpoint('abcd:42', expected_fileline=('abcd', 42),
+        expected_output='Note: "abcd" doesnt look like a Python filename.'
+                        ' Setting breakpoint anyway\n')
+    check_add_breakpoint('abcd.py:42',
+                         expected_fileline=('abcd.py', 42))
+    check_add_breakpoint('42:abc',
+        expected_output='expected a line number after colon\n',
+        expected_chbkpt='')
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -105,6 +105,7 @@
         '_promote'                   : 'interp_magic._promote',
         'normalize_exc'             : 'interp_magic.normalize_exc',
         'StdErrPrinter'             : 'interp_stderrprinter.W_StdErrPrinter',
+        'side_effects_ok'           : 'interp_magic.side_effects_ok',
         'stack_almost_full'         : 'interp_magic.stack_almost_full',
         'fsencode'                  : 'interp_magic.fsencode',
         'fsdecode'                  : 'interp_magic.fsdecode',
@@ -144,3 +145,7 @@
             features = detect_cpu.getcpufeatures(model)
             self.extra_interpdef('jit_backend_features',
                                     'space.wrap(%r)' % features)
+        if self.space.config.translation.reverse_debugger:
+            self.extra_interpdef('revdb_stop', 'interp_magic.revdb_stop')
+        else:
+            self.extra_interpdef('revdb_stop', 'space.w_None')
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -189,3 +189,23 @@
 def fsdecode(space, w_obj):
     """Direct access to the interp-level fsdecode()"""
     return space.fsdecode(w_obj)
+
+def side_effects_ok(space):
+    """For use with the reverse-debugger: this function normally returns
+    True, but will return False if we are evaluating a debugging command
+    like a watchpoint.  You are responsible for not doing any side effect
+    at all (including no caching) when evaluating watchpoints.  This
+    function is meant to help a bit---you can write:
+
+        if not __pypy__.side_effects_ok():
+            skip the caching logic
+
+    inside getter methods or properties, to make them usable from
+    watchpoints.  Note that you need to re-run ``REVDB=.. pypy''
+    after changing the Python code.
+    """
+    return space.newbool(space._side_effects_ok())
+
+def revdb_stop(space):
+    from pypy.interpreter.reverse_debugging import stop_point
+    stop_point()
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -3,9 +3,9 @@
 """
 import sys, os, py
 
-from rpython.rlib import clibffi, jit, rgc, objectmodel
+from rpython.rlib import clibffi, jit, objectmodel
 from rpython.rlib.objectmodel import keepalive_until_here
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.module._cffi_backend import cerrno, misc, parse_c_type
@@ -13,6 +13,7 @@
 from pypy.module._cffi_backend.ctypefunc import SIZE_OF_FFI_ARG, W_CTypeFunc
 from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveSigned
 from pypy.module._cffi_backend.ctypevoid import W_CTypeVoid
+from pypy.module._cffi_backend.hide_reveal import hide_reveal1
 
 BIG_ENDIAN = sys.byteorder == 'big'
 
@@ -30,9 +31,7 @@
     return cdata
 
 def reveal_callback(raw_ptr):
-    addr = rffi.cast(llmemory.Address, raw_ptr)
-    gcref = rgc.reveal_gcref(addr)
-    return rgc.try_cast_gcref_to_instance(W_ExternPython, gcref)
+    return hide_reveal1().reveal_object(W_ExternPython, raw_ptr)
 
 
 class Closure(object):
@@ -92,9 +91,7 @@
         return ctype
 
     def hide_object(self):
-        gcref = rgc.cast_instance_to_gcref(self)
-        raw = rgc.hide_nonmovable_gcref(gcref)
-        return rffi.cast(rffi.VOIDP, raw)
+        return hide_reveal1().hide_object(rffi.VOIDP, self)
 
     def _repr_extra(self):
         space = self.space
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -168,7 +168,8 @@
     rawbytes = cache.wdict.get(w_x)
     if rawbytes is None:
         data = space.bytes_w(w_x)
-        if we_are_translated() and not rgc.can_move(data):
+        if (we_are_translated() and not rgc.can_move(data)
+                                and not rgc.must_split_gc_address_space()):
             lldata = llstr(data)
             data_start = (llmemory.cast_ptr_to_adr(lldata) +
                           rffi.offsetof(STR, 'chars') +
diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py
--- a/pypy/module/_cffi_backend/handle.py
+++ b/pypy/module/_cffi_backend/handle.py
@@ -3,8 +3,9 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module._cffi_backend import ctypeobj, ctypeptr, cdataobj
+from pypy.module._cffi_backend.hide_reveal import hide_reveal2
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.rlib import rgc, objectmodel, jit
+from rpython.rlib import objectmodel, jit
 
 # ____________________________________________________________
 
@@ -15,9 +16,7 @@
     # we can cast the CCHARP back to a W_CDataHandle with reveal_gcref().
     new_cdataobj = objectmodel.instantiate(cdataobj.W_CDataHandle,
                                            nonmovable=True)
-    gcref = rgc.cast_instance_to_gcref(new_cdataobj)
-    _cdata = rgc.hide_nonmovable_gcref(gcref)
-    _cdata = rffi.cast(rffi.CCHARP, _cdata)
+    _cdata = hide_reveal2().hide_object(rffi.CCHARP, new_cdataobj)
     cdataobj.W_CDataHandle.__init__(new_cdataobj, space, _cdata, w_ctype, w_x)

From pypy.commits at gmail.com  Wed May 23 16:47:37 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 13:47:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5b05d369.1c69fb81.1ba47.143b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94677:8e91ac1e6132
Date: 2018-05-23 22:46 +0200
http://bitbucket.org/pypy/pypy/changeset/8e91ac1e6132/

Log:	hg merge default


From pypy.commits at gmail.com  Wed May 23 16:55:38 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 13:55:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: uh revert this
Message-ID: <5b05d54a.1c69fb81.68b2d.70c4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94678:6a1df86a6f7a
Date: 2018-05-23 22:55 +0200
http://bitbucket.org/pypy/pypy/changeset/6a1df86a6f7a/

Log:	uh revert this

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.13"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "6.0.0"
-#define PYPY_VERSION_NUM  0x06000000
+#define PYPY_VERSION "6.1.0-alpha0"
+#define PYPY_VERSION_NUM  0x06010000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (6, 0, 0, "final", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (6, 1, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Wed May 23 16:59:01 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 13:59:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5b05d615.1c69fb81.7b79.7c7b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r94679:7920e2af5244
Date: 2018-05-23 22:58 +0200
http://bitbucket.org/pypy/pypy/changeset/7920e2af5244/

Log:	hg merge default

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.5.3"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "6.0.0"
-#define PYPY_VERSION_NUM  0x06000000
+#define PYPY_VERSION "6.1.0-alpha0"
+#define PYPY_VERSION_NUM  0x06010000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (6, 0, 0, "final", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (6, 1, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Thu May 24 02:54:43 2018
From: pypy.commits at gmail.com (arigo)
Date: Wed, 23 May 2018 23:54:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Tweak revdb
Message-ID: <5b0661b3.1c69fb81.7f3fa.22e8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94680:04a7d1343551
Date: 2018-05-24 08:54 +0200
http://bitbucket.org/pypy/pypy/changeset/04a7d1343551/

Log:	Tweak revdb

diff --git a/pypy/interpreter/reverse_debugging.py b/pypy/interpreter/reverse_debugging.py
--- a/pypy/interpreter/reverse_debugging.py
+++ b/pypy/interpreter/reverse_debugging.py
@@ -298,7 +298,7 @@
     else:
         frame = ec.topframeref()
     if frame is None and not silent:
-        revdb.send_output("No stack.\n")
+        revdb.send_print("No stack.")
     return frame
 
 def compile(source, mode):
@@ -357,8 +357,7 @@
     if space.isinstance_w(w_repr, space.w_unicode):
         w_repr = space.call_method(w_repr, 'encode',
                                    space.newtext('utf-8'))   # safe?
-    revdb.send_output(space.bytes_w(w_repr))
-    revdb.send_output("\n")
+    revdb.send_print(space.bytes_w(w_repr))
 
 @gateway.unwrap_spec(name='text0', level=int)
 def revdb_importhook(space, name, w_globals=None,
@@ -426,14 +425,14 @@
                 # traceback', which might not be doable without using I/O
                 tb = operationerr.get_traceback()
                 if tb is not None:
-                    revdb.send_output("Traceback (most recent call last):\n")
+                    revdb.send_print("Traceback (most recent call last):")
                     while tb is not None:
                         if not isinstance(tb, pytraceback.PyTraceback):
-                            revdb.send_output("  ??? %s\n" % tb)
+                            revdb.send_print("  ??? %s" % tb)
                             break
                         show_frame(tb.frame, tb.get_lineno(), indent='  ')
                         tb = tb.next
-                revdb.send_output('%s\n' % operationerr.errorstr(space))
+                revdb.send_print(operationerr.errorstr(space))
 
                 # set the sys.last_xxx attributes
                 w_type = operationerr.w_type
@@ -445,7 +444,7 @@
                 space.setitem(w_dict, space.newtext('last_traceback'), w_tb)
 
         except OperationError as e:
-            revdb.send_output('%s\n' % e.errorstr(space, use_repr=True))
+            revdb.send_print(e.errorstr(space, use_repr=True))
 lambda_print = lambda: command_print
 
 
@@ -476,7 +475,7 @@
     ellipsis_after = False
     if first_lineno < current_lineno - max_lines_before - 1:
         first_lineno = current_lineno - max_lines_before
-        revdb.send_output("...\n")
+        revdb.send_print("...")
     if final_lineno > current_lineno + max_lines_after + 1:
         final_lineno = current_lineno + max_lines_after
         ellipsis_after = True
@@ -495,26 +494,26 @@
         revdb.send_linecache(code.co_filename, i, strip=False)
     #
     if ellipsis_after:
-        revdb.send_output("...\n")
+        revdb.send_print("...")
 
 def command_backtrace(cmd, extra):
     frame = fetch_cur_frame(silent=True)
     if cmd.c_arg1 == 0:
         if frame is not None:
-            revdb.send_output("%s:\n" % (
+            revdb.send_print("%s:" % (
                 file_and_lineno(frame, frame.get_last_lineno()),))
         display_function_part(frame, max_lines_before=8, max_lines_after=5)
     elif cmd.c_arg1 == 2:
         display_function_part(frame, max_lines_before=1000,max_lines_after=1000)
     else:
-        revdb.send_output("Current call stack (most recent call last):\n")
+        revdb.send_print("Current call stack (most recent call last):")
         if frame is None:
-            revdb.send_output("  (empty)\n")
+            revdb.send_print("  (empty)")
         frames = []
         while frame is not None:
             frames.append(frame)
             if len(frames) == 200:
-                revdb.send_output("  ...\n")
+                revdb.send_print("  ...")
                 break
             frame = frame.get_f_back()
         while len(frames) > 0:
@@ -547,7 +546,7 @@
                         print '!<%s: %r>' % (exc, val)
             """)
         except OperationError as e:
-            revdb.send_output('%s\n' % e.errorstr(space, use_repr=True))
+            revdb.send_print(e.errorstr(space, use_repr=True))
 lambda_locals = lambda: command_locals
 
 # ____________________________________________________________
@@ -631,7 +630,7 @@
 def add_breakpoint(name, i):
     # if it is empty, complain
     if not name:
-        revdb.send_output("Empty breakpoint name\n")
+        revdb.send_print("Empty breakpoint name")
         revdb.send_change_breakpoint(i)
         return
     # if it is surrounded by < >, it is the name of a code object
@@ -651,9 +650,9 @@
                 assert n >= 0
                 name = name[:n]
             if not valid_identifier(name):
-                revdb.send_output(
+                revdb.send_print(
                     'Note: "%s()" doesn''t look like a function name. '
-                    'Setting breakpoint anyway\n' % name)
+                    'Setting breakpoint anyway' % name)
             add_breakpoint_funcname(name, i)
             name += '()'
             if name != original_name:
@@ -666,7 +665,7 @@
         try:
             lineno = int(name[j+1:])
         except ValueError:
-            revdb.send_output('expected a line number after colon\n')
+            revdb.send_print('expected a line number after colon')
             revdb.send_change_breakpoint(i)
             return
         filename = name[:j]
@@ -684,9 +683,9 @@
         pass    # use unmodified
     elif not filename.lower().endswith('.py'):
         # use unmodified, but warn
-        revdb.send_output(
+        revdb.send_print(
             'Note: "%s" doesn''t look like a Python filename. '
-            'Setting breakpoint anyway\n' % (filename,))
+            'Setting breakpoint anyway' % (filename,))
 
     add_breakpoint_fileline(filename, lineno, i)
     name = '%s:%d' % (filename, lineno)
diff --git a/rpython/rlib/revdb.py b/rpython/rlib/revdb.py
--- a/rpython/rlib/revdb.py
+++ b/rpython/rlib/revdb.py
@@ -46,6 +46,9 @@
 def send_output(text):
     send_answer(ANSWER_TEXT, extra=text)
 
+def send_print(text):
+    send_answer(ANSWER_TEXT, 1, extra=text)   # adds a newline
+
 def send_nextnid(unique_id):
     send_answer(ANSWER_NEXTNID, unique_id)
 

From pypy.commits at gmail.com  Thu May 24 09:02:23 2018
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 24 May 2018 06:02:23 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: add an option to control how the gc
 works inside gcbench: you can choose whether to use gc.disable() and/or
 gc.collect(1)
Message-ID: <5b06b7df.1c69fb81.1bdb9.e2f1@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94681:a75d8c7f141f
Date: 2018-05-23 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/a75d8c7f141f/

Log:	add an option to control how the gc works inside gcbench: you can
	choose whether to use gc.disable() and/or gc.collect(1)

diff --git a/rpython/translator/goal/gcbench.py b/rpython/translator/goal/gcbench.py
--- a/rpython/translator/goal/gcbench.py
+++ b/rpython/translator/goal/gcbench.py
@@ -44,8 +44,9 @@
 #               - Results are sensitive to locking cost, but we dont
 #                 check for proper locking
 import time
+import gc
 
-USAGE = """gcbench [num_repetitions] [--depths=N,N,N..] [--threads=N]"""
+USAGE = """gcbench [num_repetitions] [--depths=N,N,N..] [--threads=N] [--gc=off|--gc=manual]"""
 ENABLE_THREADS = True
 
 
@@ -173,6 +174,7 @@
     depths = DEFAULT_DEPTHS
     threads = 0
     repeatcount = 1
+    gc_policy = 'on'
     for arg in argv[1:]:
         if arg.startswith('--threads='):
             arg = arg[len('--threads='):]
@@ -189,13 +191,22 @@
                 depths = [int(s) for s in arg]
             except ValueError:
                 return argerror()
+        elif arg.startswith('--gc=off'):
+            gc_policy = 'off'
+        elif arg.startswith('--gc=manual'):
+            gc_policy = 'manual'
         else:
             try:
                 repeatcount = int(arg)
             except ValueError:
                 return argerror()
+    #
+    if gc_policy == 'off' or gc_policy == 'manual':
+        gc.disable()
     for i in range(repeatcount):
         main(depths, threads)
+        if gc_policy == 'manual':
+            gc.collect(1)
     return 0
 
 

From pypy.commits at gmail.com  Thu May 24 09:02:26 2018
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 24 May 2018 06:02:26 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: the app-level gc.{enable,
 disable} now call also the rgc equivalent,
 to actually disable major collections
Message-ID: <5b06b7e2.1c69fb81.99692.5a40@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94682:91902e174491
Date: 2018-05-24 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/91902e174491/

Log:	the app-level gc.{enable,disable} now call also the rgc equivalent,
	to actually disable major collections

diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py
--- a/pypy/module/gc/interp_gc.py
+++ b/pypy/module/gc/interp_gc.py
@@ -39,18 +39,20 @@
     return space.newint(0)
 
 def enable(space):
-    """Non-recursive version.  Enable finalizers now.
+    """Non-recursive version.  Enable major collections and finalizers.
     If they were already enabled, no-op.
     If they were disabled even several times, enable them anyway.
     """
+    rgc.enable()
     if not space.user_del_action.enabled_at_app_level:
         space.user_del_action.enabled_at_app_level = True
         enable_finalizers(space)
 
 def disable(space):
-    """Non-recursive version.  Disable finalizers now.  Several calls
-    to this function are ignored.
+    """Non-recursive version.  Disable major collections and finalizers.
+    Multiple calls to this function are ignored.
     """
+    rgc.disable()
     if space.user_del_action.enabled_at_app_level:
         space.user_del_action.enabled_at_app_level = False
         disable_finalizers(space)
diff --git a/pypy/module/gc/test/test_gc.py b/pypy/module/gc/test/test_gc.py
--- a/pypy/module/gc/test/test_gc.py
+++ b/pypy/module/gc/test/test_gc.py
@@ -1,7 +1,19 @@
 import py
-
+import pytest
+from rpython.rlib import rgc
+from pypy.interpreter.baseobjspace import ObjSpace
+from pypy.interpreter.gateway import interp2app, unwrap_spec
 
 class AppTestGC(object):
+
+    def setup_class(cls):
+        if cls.runappdirect:
+            pytest.skip("these tests cannot work with -A")
+        space = cls.space
+        def rgc_isenabled(space):
+            return space.newbool(rgc.isenabled())
+        cls.w_rgc_isenabled = space.wrap(interp2app(rgc_isenabled))
+
     def test_collect(self):
         import gc
         gc.collect() # mostly a "does not crash" kind of test
@@ -63,12 +75,16 @@
     def test_enable(self):
         import gc
         assert gc.isenabled()
+        assert self.rgc_isenabled()
         gc.disable()
         assert not gc.isenabled()
+        assert not self.rgc_isenabled()
         gc.enable()
         assert gc.isenabled()
+        assert self.rgc_isenabled()
         gc.enable()
         assert gc.isenabled()
+        assert self.rgc_isenabled()
 
     def test_gc_collect_overrides_gc_disable(self):
         import gc

From pypy.commits at gmail.com  Fri May 25 04:23:26 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 25 May 2018 01:23:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: add the key used in the subscription
 when trying to use getitem on an object
Message-ID: <5b07c7fe.1c69fb81.c1bb2.1d67@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94684:6167d7594c32
Date: 2018-05-25 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/6167d7594c32/

Log:	add the key used in the subscription when trying to use getitem on
	an object that doesn't support that.

	This is a commonly named unhelpful error message that occurs when
	doing subscriptions in a chain (eg when processing JSON data):

	d[a][b][c][d]

	if one of the intermediate dictionaries is None, it's very hard to
	see which one just from at the exception.

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -298,7 +298,8 @@
         w_descr = space.lookup(w_obj, '__getitem__')
         if w_descr is None:
             raise oefmt(space.w_TypeError,
-                        "'%T' object is not subscriptable", w_obj)
+                        "'%T' object is not subscriptable (key %R)",
+                        w_obj, w_key)
         return space.get_and_call_function(w_descr, w_obj, w_key)
 
     def setitem(space, w_obj, w_key, w_val):
diff --git a/pypy/objspace/std/test/test_noneobject.py b/pypy/objspace/std/test/test_noneobject.py
--- a/pypy/objspace/std/test/test_noneobject.py
+++ b/pypy/objspace/std/test/test_noneobject.py
@@ -8,3 +8,4 @@
 
     def test_false(self):
         assert not self.space.is_true(self.space.w_None)
+
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -812,3 +812,8 @@
         class A(object):
             d = D()
         raises(AttributeError, "A().d = 5")
+
+    def test_not_subscriptable_error_gives_keys(self):
+        d = {'key1': {'key2': {'key3': None}}}
+        excinfo = raises(TypeError, "d['key1']['key2']['key3']['key4']['key5']")
+        assert "key4" in str(excinfo.value)

From pypy.commits at gmail.com  Sat May 26 08:12:08 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 26 May 2018 05:12:08 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: fix raises
Message-ID: <5b094f18.1c69fb81.2d304.2083@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94685:a26f84e38118
Date: 2018-05-21 14:13 +0200
http://bitbucket.org/pypy/pypy/changeset/a26f84e38118/

Log:	fix raises

diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -274,7 +274,7 @@
             tree = self.parse(x)
         for x in INVALID_UNDERSCORE_LITERALS:
             print x
-            raises(SyntaxError, self.parse, "x = %s" % x)
+            py.test.raises(SyntaxError, self.parse, "x = %s" % x)
 
 
 class TestPythonParserWithSpace:

From pypy.commits at gmail.com  Sat May 26 08:12:11 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 26 May 2018 05:12:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: merge
Message-ID: <5b094f1b.1c69fb81.132e9.8886@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94686:807d5baaba0b
Date: 2018-05-26 14:11 +0200
http://bitbucket.org/pypy/pypy/changeset/807d5baaba0b/

Log:	merge

diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -274,7 +274,7 @@
             tree = self.parse(x)
         for x in INVALID_UNDERSCORE_LITERALS:
             print x
-            raises(SyntaxError, self.parse, "x = %s" % x)
+            py.test.raises(SyntaxError, self.parse, "x = %s" % x)
 
 
 class TestPythonParserWithSpace:

From pypy.commits at gmail.com  Sat May 26 08:19:57 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 26 May 2018 05:19:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix test_reverse_debugging
Message-ID: <5b0950ed.1c69fb81.28933.739f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94687:24dc51e3f099
Date: 2018-05-26 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/24dc51e3f099/

Log:	Fix test_reverse_debugging

diff --git a/pypy/interpreter/test/test_reverse_debugging.py b/pypy/interpreter/test/test_reverse_debugging.py
--- a/pypy/interpreter/test/test_reverse_debugging.py
+++ b/pypy/interpreter/test/test_reverse_debugging.py
@@ -86,6 +86,9 @@
         if msg[0] == revdb.ANSWER_TEXT:
             assert got_output is None
             got_output = msg[-1]
+            assert msg[1] in (0, 1)
+            if msg[1]:
+                got_output += "\n"
         elif msg[0] == revdb.ANSWER_CHBKPT:
             assert got_chbkpt is None
             assert msg[1] == 5

From pypy.commits at gmail.com  Sat May 26 08:24:32 2018
From: pypy.commits at gmail.com (arigo)
Date: Sat, 26 May 2018 05:24:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: Translation fix on Windows (for
 "inline" in signals.h)
Message-ID: <5b095200.1c69fb81.29a85.0655@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94688:e6a28f470244
Date: 2018-05-26 14:23 +0200
http://bitbucket.org/pypy/pypy/changeset/e6a28f470244/

Log:	Translation fix on Windows (for "inline" in signals.h)

diff --git a/rpython/translator/c/src/precommondefs.h b/rpython/translator/c/src/precommondefs.h
--- a/rpython/translator/c/src/precommondefs.h
+++ b/rpython/translator/c/src/precommondefs.h
@@ -73,6 +73,9 @@
 #  define RPY_EXTERN   extern _RPY_HIDDEN
 #endif
 
+#ifdef _WIN32
+#  define inline _inline
+#endif
 
 #endif /* __PYPY_PRECOMMONDEFS_H */
 
diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h
--- a/rpython/translator/c/src/thread.h
+++ b/rpython/translator/c/src/thread.h
@@ -14,7 +14,6 @@
 #ifdef _WIN32
 #define RPYTHREAD_NAME "nt"
 #include "thread_nt.h"
-#define inline _inline
 #else
 
 /* We should check if unistd.h defines _POSIX_THREADS, but sometimes

From pypy.commits at gmail.com  Sat May 26 09:28:50 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 26 May 2018 06:28:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: support for underscores in float and
 complex literals. tests and approach taken
Message-ID: <5b096112.1c69fb81.b1f47.16d0@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94689:44c25f062a8c
Date: 2018-05-26 14:58 +0200
http://bitbucket.org/pypy/pypy/changeset/44c25f062a8c/

Log:	support for underscores in float and complex literals. tests and
	approach taken from CPython

diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -13,7 +13,7 @@
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
 from pypy.interpreter.typedef import GetSetProperty, TypeDef
 from pypy.objspace.std import newformat
-from pypy.objspace.std.floatobject import _hash_float
+from pypy.objspace.std.floatobject import _hash_float, _remove_underscores
 from pypy.objspace.std.unicodeobject import unicode_to_decimal_w
 
 HASH_IMAG = 1000003
@@ -285,6 +285,11 @@
                                                " arg if first is a string")
             unistr = unicode_to_decimal_w(space, w_real)
             try:
+                unistr = _remove_underscores(unistr)
+            except ValueError:
+                raise oefmt(space.w_ValueError,
+                            "complex() arg is a malformed string")
+            try:
                 realstr, imagstr = _split_complex(unistr)
             except ValueError:
                 raise oefmt(space.w_ValueError,
diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -205,10 +205,16 @@
     def descr__new__(space, w_floattype, w_x):
         def _string_to_float(space, w_source, string):
             try:
-                return rfloat.string_to_float(string)
-            except ParseStringError as e:
-                raise oefmt(space.w_ValueError,
-                            "could not convert string to float: %R", w_source)
+                string = _remove_underscores(string)
+            except ValueError:
+                pass
+            else:
+                try:
+                    return rfloat.string_to_float(string)
+                except ParseStringError as e:
+                    pass
+            raise oefmt(space.w_ValueError,
+                        "could not convert string to float: %R", w_source)
 
         w_value = w_x     # 'x' is the keyword argument name in CPython
         if space.lookup(w_value, "__float__") is not None:
@@ -728,6 +734,26 @@
     hex = interp2app(W_FloatObject.descr_hex),
 )
 
+def _remove_underscores(string):
+    i = 0
+    prev = '?'
+    res = []
+    for i in range(len(string)):
+        c = string[i]
+        if c == '_':
+            # undercores can only come after digits
+            if not ord('0') <= ord(prev) <= ord('9'):
+                raise ValueError
+        else:
+            res.append(c)
+            # undercores can only come before digits
+            if prev == '_' and not ord('0') <= ord(c) <= ord('9'):
+                raise ValueError
+        prev = c
+    if prev == "_": # not allowed at end
+        raise ValueError
+    return "".join(res)
+
 
 def _hash_float(space, v):
     if not isfinite(v):
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -370,6 +370,46 @@
         assert self.almost_equal(complex(real=float2(17.), imag=float2(23.)), 17+23j)
         raises(TypeError, complex, float2(None))
 
+    def test_complex_string_underscores(self):
+        valid = [
+            '1_00_00j',
+            '1_00_00.5j',
+            '1_00_00e5_1j',
+            '.1_4j',
+            '(1_2.5+3_3j)',
+            '(.5_6j)',
+        ]
+        for s in valid:
+            assert complex(s) == complex(s.replace("_", ""))
+            assert eval(s) == eval(s.replace("_", ""))
+
+        invalid = [
+            # Trailing underscores:
+            '1.4j_',
+            # Multiple consecutive underscores:
+            '0.1__4j',
+            '1e1__0j',
+            # Underscore right before a dot:
+            '1_.4j',
+            # Underscore right after a dot:
+            '1._4j',
+            '._5j',
+            # Underscore right after a sign:
+            '1.0e+_1j',
+            # Underscore right before j:
+            '1.4e5_j',
+            # Underscore right before e:
+            '1.4_e1j',
+            # Underscore right after e:
+            '1.4e_1j',
+            # Complex cases with parens:
+            '(1+1.5_j_)',
+            '(1+1.5_j)',
+        ]
+        for s in invalid:
+            raises(ValueError, complex, s)
+            raises(SyntaxError, eval, s)
+
     @py.test.mark.skipif("not config.option.runappdirect and sys.maxunicode == 0xffff")
     def test_constructor_unicode(self):
         b1 = '\N{MATHEMATICAL BOLD DIGIT ONE}' # &#120783;
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -1,9 +1,10 @@
 # -*- encoding: utf-8 -*-
+import pytest
 import sys
 
 import py
 
-from pypy.objspace.std.floatobject import W_FloatObject
+from pypy.objspace.std.floatobject import W_FloatObject, _remove_underscores
 
 
 class TestW_FloatObject:
@@ -59,6 +60,74 @@
         finally:
             W_LongObject.fromfloat = saved
 
+    def test_remove_undercores(self):
+        valid = [
+            '0_0_0',
+            '4_2',
+            '1_0000_0000',
+            '0b1001_0100',
+            '0o5_7_7',
+            '1_00_00.5',
+            '1_00_00.5e5',
+            '1_00_00e5_1',
+            '1e1_0',
+            '.1_4',
+            '.1_4e1',
+            '1_00_00j',
+            '1_00_00.5j',
+            '1_00_00e5_1j',
+            '.1_4j',
+            '(1_2.5+3_3j)',
+            '(.5_6j)',
+        ]
+        for s in valid:
+            assert _remove_underscores(s) == s.replace("_", "")
+
+        invalid = [
+            # Trailing underscores:
+            '0_',
+            '42_',
+            '1.4j_',
+            # Multiple consecutive underscores:
+            '4_______2',
+            '0.1__4',
+            '0.1__4j',
+            '0b1001__0100',
+            '0xffff__ffff',
+            '0x___',
+            '0o5__77',
+            '1e1__0',
+            '1e1__0j',
+            # Underscore right before a dot:
+            '1_.4',
+            '1_.4j',
+            # Underscore right after a dot:
+            '1._4',
+            '1._4j',
+            '._5',
+            '._5j',
+            # Underscore right after a sign:
+            '1.0e+_1',
+            '1.0e+_1j',
+            # Underscore right before j:
+            '1.4_j',
+            '1.4e5_j',
+            # Underscore right before e:
+            '1_e1',
+            '1.4_e1',
+            '1.4_e1j',
+            # Underscore right after e:
+            '1e_1',
+            '1.4e_1',
+            '1.4e_1j',
+            # Complex cases with parens:
+            '(1+1.5_j_)',
+            '(1+1.5_j)',
+        ]
+        for s in invalid:
+            pytest.raises(ValueError, _remove_underscores, s)
+
+
 
 class AppTestAppFloatTest:
     spaceconfig = dict(usemodules=['binascii', 'time'])
@@ -151,6 +220,53 @@
 
         raises(UnicodeEncodeError, float, u"\ud800")
 
+    def test_float_string_underscores(self):
+        valid = [
+            '0_0_0',
+            '4_2',
+            '1_0000_0000',
+            '1_00_00.5',
+            '1_00_00.5e5',
+            '1_00_00e5_1',
+            '1e1_0',
+            '.1_4',
+            '.1_4e1',
+        ]
+        for s in valid:
+            assert float(s) == float(s.replace("_", ""))
+            assert eval(s) == eval(s.replace("_", ""))
+
+        invalid = [
+            # Trailing underscores:
+            '0_',
+            '42_',
+            # Multiple consecutive underscores:
+            '4_______2',
+            '0.1__4',
+            '0.1__4j',
+            '0b1001__0100',
+            '0xffff__ffff',
+            '0x___',
+            '0o5__77',
+            '1e1__0',
+            # Underscore right before a dot:
+            '1_.4',
+            # Underscore right after a dot:
+            '1._4',
+            '._5',
+            # Underscore right after a sign:
+            '1.0e+_1',
+            # Underscore right before e:
+            '1_e1',
+            '1.4_e1',
+            # Underscore right after e:
+            '1e_1',
+            '1.4e_1',
+        ]
+        for s in invalid:
+            raises(ValueError, float, s)
+            raises(SyntaxError, eval, s)
+
     def test_float_unicode(self):
         # u00A0 and u2000 are some kind of spaces
         assert 42.75 == float(chr(0x00A0)+str("42.75")+chr(0x2000))

From pypy.commits at gmail.com  Sat May 26 09:28:52 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 26 May 2018 06:28:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: calling subtypes of type with one arg is
 not supported any more
Message-ID: <5b096114.1c69fb81.238c1.e014@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94690:cbea6ffdd672
Date: 2018-05-26 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/cbea6ffdd672/

Log:	calling subtypes of type with one arg is not supported any more

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -95,8 +95,6 @@
         class A(type):
             pass
 
-        assert A("hello") is str
-
         # Make sure type(x) doesn't call x.__class__.__init__
         class T(type):
             counter = 0

From pypy.commits at gmail.com  Sun May 27 11:29:40 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 27 May 2018 08:29:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix issue #2838: don't swallow exceptions
 in BUILD_SET_UNPACK
Message-ID: <5b0acee4.1c69fb81.4b098.4152@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94691:26a7e1a30293
Date: 2018-05-27 17:27 +0200
http://bitbucket.org/pypy/pypy/changeset/26a7e1a30293/

Log:	fix issue #2838: don't swallow exceptions in BUILD_SET_UNPACK

	(the bytecode was quite badly broken in other ways too: the JIT
	would unroll the set generation completely}

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1433,21 +1433,12 @@
     @jit.unroll_safe
     def BUILD_SET_UNPACK(self, itemcount, next_instr):
         space = self.space
-        w_sum = space.newset()
+        w_set = space.newset()
         for i in range(itemcount, 0, -1):
             w_item = self.peekvalue(i-1)
-            # cannot use w_sum.update, w_item might not be a set
-            iterator = space.iter(w_item)
-            while True:
-                try:
-                    w_value = space.next(iterator)
-                except OperationError:
-                    break
-                w_sum.add(w_value)
-        while itemcount != 0:
-            self.popvalue()
-            itemcount -= 1
-        self.pushvalue(w_sum)
+            space.call_method(w_set, "update", w_item)
+        self.popvalues(itemcount)
+        self.pushvalue(w_set)
 
     @jit.unroll_safe
     def list_unpack_helper(frame, itemcount):
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -265,7 +265,7 @@
             return a, b, c, d
         """
         assert self.codetest(code, "f", [1, 2], {"d" : 4, "c" : 3}) == (1, 2, 3, 4)
-    
+
     def test_build_set_unpack(self):
         code = """ def f():
             return {*range(4), 4, *(5, 6, 7)}
@@ -274,13 +274,28 @@
         res = self.codetest(code, "f", [])
         l_res = space.call_function(space.w_list, res)
         assert space.unwrap(l_res) == [0, 1, 2, 3, 4, 5, 6, 7]
-        
+
+    def test_build_set_unpack_exception(self):
+        code = """ if 1:
+        def g():
+            yield 1
+            yield 2
+            raise TypeError
+        def f():
+            try:
+                {*g(), 1, 2}
+            except TypeError:
+                return True
+            return False
+        """
+        assert self.codetest(code, "f", [])
+
     def test_build_tuple_unpack(self):
         code = """ def f():
             return (*range(4), 4)
         """
         assert self.codetest(code, "f", []) == (0, 1, 2, 3, 4)
-    
+
     def test_build_list_unpack(self):
         code = """ def f():
             return [*range(4), 4]

From pypy.commits at gmail.com  Sun May 27 11:44:15 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 27 May 2018 08:44:15 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: use popvalues
Message-ID: <5b0ad24f.1c69fb81.6a773.7cb5@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r94692:a7e4b0731746
Date: 2018-05-27 17:37 +0200
http://bitbucket.org/pypy/pypy/changeset/a7e4b0731746/

Log:	use popvalues

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1447,9 +1447,7 @@
         for i in range(itemcount, 0, -1):
             w_item = frame.peekvalue(i-1)
             w_sum.extend(w_item)
-        while itemcount != 0:
-            frame.popvalue()
-            itemcount -= 1
+        self.popalues(itemcount)
         return w_sum
 
     @jit.unroll_safe
@@ -1489,9 +1487,7 @@
             space.call_method(w_dict, 'update', w_item)
         if with_call and space.len_w(w_dict) < expected_length:
             self._build_map_unpack_error(itemcount)
-        while itemcount > 0:
-            self.popvalue()
-            itemcount -= 1
+        self.popvalues(itemcount)
         self.pushvalue(w_dict)
 
     @jit.dont_look_inside

From pypy.commits at gmail.com  Sun May 27 15:45:46 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 27 May 2018 12:45:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: support for '_' format modifiert for
 formatting strings
Message-ID: <5b0b0aea.1c69fb81.a0778.5ae5@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94693:846993741acb
Date: 2018-05-27 21:44 +0200
http://bitbucket.org/pypy/pypy/changeset/846993741acb/

Log:	support for '_' format modifiert for formatting strings

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -455,7 +455,7 @@
             self._align = default_align
             self._alternate = False
             self._sign = "\0"
-            self._thousands_sep = False
+            self._thousands_sep = "\0"
             self._precision = -1
             the_type = default_type
             spec = self.spec
@@ -488,8 +488,17 @@
                 i += 1
             self._width, i = _parse_int(self.space, spec, i, length)
             if length != i and spec[i] == ",":
-                self._thousands_sep = True
+                self._thousands_sep = ","
                 i += 1
+            if length != i and spec[i] == "_":
+                if self._thousands_sep != "\0":
+                    raise oefmt(
+                        space.w_ValueError, "Cannot specify both ',' and '_'.")
+                self._thousands_sep = "_"
+                i += 1
+                if length != i and spec[i] == ",":
+                    raise oefmt(
+                        space.w_ValueError, "Cannot specify both ',' and '_'.")
             if length != i and spec[i] == ".":
                 i += 1
                 self._precision, i = _parse_int(self.space, spec, i, length)
@@ -509,7 +518,7 @@
                     the_type = presentation_type
                 i += 1
             self._type = the_type
-            if self._thousands_sep:
+            if self._thousands_sep != "\0":
                 tp = self._type
                 if (tp == "d" or
                     tp == "e" or
@@ -522,8 +531,15 @@
                     tp == "\0"):
                     # ok
                     pass
+                elif self._thousands_sep == "_" and (
+                        tp == "b" or
+                        tp == "o" or
+                        tp == "x" or
+                        tp == "X"):
+                    pass # ok
                 else:
-                    raise oefmt(space.w_ValueError, "invalid type with ','")
+                    raise oefmt(space.w_ValueError,
+                                "invalid type with ',' or '_'")
             return False
 
         def _calc_padding(self, string, length):
@@ -601,10 +617,13 @@
         def _get_locale(self, tp):
             if tp == "n":
                 dec, thousands, grouping = rlocale.numeric_formatting()
-            elif self._thousands_sep:
+            elif self._thousands_sep != "\0":
+                thousands = self._thousands_sep
                 dec = "."
-                thousands = ","
                 grouping = "\3"
+                if tp in "boxX":
+                    assert self._thousands_sep == "_"
+                    grouping = "\4"
             else:
                 dec = "."
                 thousands = ""
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -246,12 +246,18 @@
     def test_invalid(self):
         raises(ValueError, format, self.i(8), "s")
         raises(ValueError, format, self.i(8), ".3")
+        raises(ValueError, format, self.i(3), '_,')
+        raises(ValueError, format, self.i(3), ',_')
+        raises(ValueError, format, self.i(3), '_,d')
+        raises(ValueError, format, self.i(3), ',_d')
+
 
     def test_c(self):
         a = self.i(ord("a"))
         assert format(a, "c") == "a"
         raises(ValueError, format, a, "-c")
         raises(ValueError, format, a, ",c")
+        raises(ValueError, format, a, "_c")
         raises(ValueError, format, a, "#c")
         assert format(a, "3c") == "  a"
         assert format(a, "<3c") == "a  "
@@ -262,6 +268,8 @@
     def test_binary(self):
         assert format(self.i(2), "b") == "10"
         assert format(self.i(2), "#b") == "0b10"
+        assert format(12345, '_b') == '11_0000_0011_1001'
+        raises(ValueError, format, self.i(1234567890), ',b')
 
     def test_octal(self):
         assert format(self.i(8), "o") == "10"
@@ -270,6 +278,8 @@
         assert format(self.i(-8), "#o") == "-0o10"
         assert format(self.i(8), "+o") == "+10"
         assert format(self.i(8), "+#o") == "+0o10"
+        raises(ValueError, format, self.i(1234567890), ',o')
+        assert format(self.i(1234567890), '_o'), '111_4540_1322'
 
     def test_hex(self):
         assert format(self.i(16), "x") == "10"
@@ -278,6 +288,10 @@
         assert format(self.i(10), "#x") == "0xa"
         assert format(self.i(10), "X") == "A"
         assert format(self.i(10), "#X") == "0XA"
+        raises(ValueError, format, 1234567890, ',x')
+        assert format(1234567890, '_x') == '4996_02d2'
+        assert format(1234567890, '_X') == '4996_02D2'
+
 
     def test_padding(self):
         assert format(self.i(6), "3") == "  6"
@@ -310,6 +324,14 @@
         assert format(self.i(1234), "0=10,") == "00,001,234"
         assert format(self.i(1234), "010,") == "00,001,234"
 
+    def test_thousands_separator_underscore(self):
+        assert format(self.i(123), "_") == "123"
+        assert format(self.i(12345), "_") == "12_345"
+        assert format(self.i(123456789), "_") == "123_456_789"
+        assert format(self.i(12345), "7_") == " 12_345"
+        assert format(self.i(12345), "<7_") == "12_345 "
+        assert format(self.i(1234), "0=10_") == "00_001_234"
+        assert format(self.i(1234), "010_") == "00_001_234"
 
 class AppTestIntFormatting(BaseIntegralFormattingTest):
     def setup_class(cls):
@@ -342,6 +364,9 @@
     def test_digit_separator(self):
         assert format(-1234., "012,f") == "-1,234.000000"
 
+    def test_digit_separator_underscore(self):
+        assert format(-1234., "012_f") == "-1_234.000000"
+
     def test_locale(self):
         import locale
         for name in ['en_US.UTF8', 'en_US', 'en']:

From pypy.commits at gmail.com  Mon May 28 15:34:46 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 28 May 2018 12:34:46 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix error messages in complex number
 handling a bit
Message-ID: <5b0c59d6.1c69fb81.4b42.e295@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r94694:cadf86879441
Date: 2018-05-28 21:33 +0200
http://bitbucket.org/pypy/pypy/changeset/cadf86879441/

Log:	fix error messages in complex number handling a bit

diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py
--- a/pypy/objspace/std/complexobject.py
+++ b/pypy/objspace/std/complexobject.py
@@ -128,7 +128,7 @@
     return format_float(x, 'g', DTSF_STR_PRECISION)
 
 
-def unpackcomplex(space, w_complex, strict_typing=True):
+def unpackcomplex(space, w_complex, strict_typing=True, firstarg=True):
     """
     convert w_complex into a complex and return the unwrapped (real, imag)
     tuple. If strict_typing==True, we also typecheck the value returned by
@@ -180,7 +180,20 @@
         raise oefmt(space.w_TypeError,
                     "complex number expected, got '%T'", w_complex)
     #
-    return (space.float_w(space.float(w_complex)), 0.0)
+    try:
+        return (space.float_w(space.float(w_complex)), 0.0)
+    except OperationError as e:
+        if not e.match(space, space.w_TypeError):
+            raise
+        if firstarg:
+            raise oefmt(space.w_TypeError,
+                        "complex() first argument must be a string or a number, not '%T'",
+                         w_complex)
+        else:
+            raise oefmt(space.w_TypeError,
+                        "complex() second argument must be a number, not '%T'",
+                         w_complex)
+
 
 
 class W_ComplexObject(W_Root):
@@ -326,7 +339,8 @@
             if not noarg2:
                 # complex(x, y) == x+y*j, even if 'y' is already a complex.
                 realval2, imagval2 = unpackcomplex(space, w_imag,
-                                                   strict_typing=False)
+                                                   strict_typing=False,
+                                                   firstarg=False)
 
                 # try to preserve the signs of zeroes of realval and realval2
                 if imagval2 != 0.0:
@@ -476,7 +490,7 @@
         try:
             return self.div(w_rhs)
         except ZeroDivisionError as e:
-            raise OperationError(space.w_ZeroDivisionError, space.newtext(str(e)))
+            raise oefmt(space.w_ZeroDivisionError, "complex division by zero")
 
     def descr_rtruediv(self, space, w_lhs):
         w_lhs = self._to_complex(space, w_lhs)
@@ -485,7 +499,7 @@
         try:
             return w_lhs.div(self)
         except ZeroDivisionError as e:
-            raise OperationError(space.w_ZeroDivisionError, space.newtext(str(e)))
+            raise oefmt(space.w_ZeroDivisionError, "complex division by zero")
 
     def descr_floordiv(self, space, w_rhs):
         w_rhs = self._to_complex(space, w_rhs)
@@ -495,7 +509,7 @@
         try:
             return self.divmod(space, w_rhs)[0]
         except ZeroDivisionError as e:
-            raise OperationError(space.w_ZeroDivisionError, space.newtext(str(e)))
+            raise oefmt(space.w_ZeroDivisionError, "complex division by zero")
 
     def descr_rfloordiv(self, space, w_lhs):
         w_lhs = self._to_complex(space, w_lhs)
@@ -505,7 +519,7 @@
         try:
             return w_lhs.divmod(space, self)[0]
         except ZeroDivisionError as e:
-            raise OperationError(space.w_ZeroDivisionError, space.newtext(str(e)))
+            raise oefmt(space.w_ZeroDivisionError, "complex division by zero")
 
     def descr_mod(self, space, w_rhs):
         w_rhs = self._to_complex(space, w_rhs)
@@ -514,7 +528,7 @@
         try:
             return self.divmod(space, w_rhs)[1]
         except ZeroDivisionError as e:
-            raise OperationError(space.w_ZeroDivisionError, space.newtext(str(e)))
+            raise oefmt(space.w_ZeroDivisionError, "complex modulo by zero")
 
     def descr_rmod(self, space, w_lhs):
         w_lhs = self._to_complex(space, w_lhs)
@@ -523,7 +537,7 @@
         try:
             return w_lhs.divmod(space, self)[1]
         except ZeroDivisionError as e:
-            raise OperationError(space.w_ZeroDivisionError, space.newtext(str(e)))
+            raise oefmt(space.w_ZeroDivisionError, "complex modulo by zero")
 
     def descr_divmod(self, space, w_rhs):
         w_rhs = self._to_complex(space, w_rhs)
@@ -532,7 +546,7 @@
         try:
             div, mod = self.divmod(space, w_rhs)
         except ZeroDivisionError as e:
-            raise OperationError(space.w_ZeroDivisionError, space.newtext(str(e)))
+            raise oefmt(space.w_ZeroDivisionError, "complex divmod by zero")
         return space.newtuple([div, mod])
 
     def descr_rdivmod(self, space, w_lhs):
@@ -542,7 +556,7 @@
         try:
             div, mod = w_lhs.divmod(space, self)
         except ZeroDivisionError as e:
-            raise OperationError(space.w_ZeroDivisionError, space.newtext(str(e)))
+            raise oefmt(space.w_ZeroDivisionError, "complex divmod by zero")
         return space.newtuple([div, mod])
 
     @unwrap_spec(w_third_arg=WrappedDefault(None))
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -375,6 +375,21 @@
         assert self.almost_equal(complex(real=float2(17.), imag=float2(23.)), 17+23j)
         raises(TypeError, complex, float2(None))
 
+
+    def test_constructor_bad_error_message(self):
+        err = raises(TypeError, complex, {}).value
+        assert "float" not in str(err)
+        assert str(err) == "complex() first argument must be a string or a number, not 'dict'"
+        err = raises(TypeError, complex, 1, {}).value
+        assert "float" not in str(err)
+        assert str(err) == "complex() second argument must be a number, not 'dict'"
+
+    def test_error_messages(self):
+        err = raises(ZeroDivisionError, "1+1j / 0").value
+        assert str(err) == "complex division by zero"
+        err = raises(ZeroDivisionError, "1+1j // 0").value
+        assert str(err) == "complex division by zero"
+
     def test___complex___returning_non_complex(self):
         import cmath
         class Obj(object):

From pypy.commits at gmail.com  Mon May 28 15:37:25 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 28 May 2018 12:37:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: skip test_dict_version.py,
 pypy does this very differently
Message-ID: <5b0c5a75.1c69fb81.5df1d.d20c@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94695:49f42c75da0e
Date: 2018-05-28 21:36 +0200
http://bitbucket.org/pypy/pypy/changeset/49f42c75da0e/

Log:	skip test_dict_version.py, pypy does this very differently

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -189,7 +189,7 @@
     RegrTest('test_descrtut.py', core=True),
     RegrTest('test_devpoll.py'),
     RegrTest('test_dict.py', core=True),
-    RegrTest('test_dict_version.py'),
+    RegrTest('test_dict_version.py', skip="implementation detail"),
     RegrTest('test_dictcomps.py', core=True),
     RegrTest('test_dictviews.py', core=True),
     RegrTest('test_difflib.py'),

From pypy.commits at gmail.com  Tue May 29 02:08:20 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 28 May 2018 23:08:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Merged in
 toumorokoshi/pypy/fix_test_codecs (pull request #612)
Message-ID: <5b0cee54.1c69fb81.e7688.5cdc@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94705:24d343241901
Date: 2018-05-29 06:07 +0000
http://bitbucket.org/pypy/pypy/changeset/24d343241901/

Log:	Merged in toumorokoshi/pypy/fix_test_codecs (pull request #612)

	Fix test codecs

diff --git a/lib-python/3/test/test_codecs.py b/lib-python/3/test/test_codecs.py
--- a/lib-python/3/test/test_codecs.py
+++ b/lib-python/3/test/test_codecs.py
@@ -2468,7 +2468,8 @@
                 with self.assertWarns(DeprecationWarning):
                     check(b"\\" + b, "\\" + chr(i))
             if b.upper() not in b'UN':
-                with self.assertWarns(DeprecationWarning):
+                with self.assertWarns(DeprecationWarning,
+                                      msg="character {} did not raise an exception".format(i)):
                     check(b"\\" + b.upper(), "\\" + chr(i-32))
         with self.assertWarns(DeprecationWarning):
             check(br"\8", "\\8")
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -117,12 +117,6 @@
     v, first_escape_error_char = PyString_DecodeEscape(
         space, substr, 'strict', encoding)
 
-    if first_escape_error_char != '':
-        space.warn(
-            space.newtext("invalid escape sequence '\\%s'"
-                          % first_escape_error_char),
-            space.w_DeprecationWarning)
-
     return space.newbytes(v)
 
 def decode_unicode_utf8(space, s, ps, q):
@@ -252,6 +246,13 @@
             # an arbitry number of unescaped UTF-8 bytes may follow.
 
     buf = builder.build()
+
+    if first_escape_error_char != '':
+        space.warn(
+            space.newtext("invalid escape sequence '\\%s'"
+                          % first_escape_error_char),
+            space.w_DeprecationWarning)
+
     return buf, first_escape_error_char
 
 
diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py
--- a/pypy/interpreter/test/test_unicodehelper.py
+++ b/pypy/interpreter/test/test_unicodehelper.py
@@ -3,7 +3,10 @@
 import struct
 import sys
 from pypy.interpreter.unicodehelper import (
-    encode_utf8, decode_utf8, unicode_encode_utf_32_be, str_decode_utf_32_be)
+    encode_utf8, decode_utf8,
+    unicode_encode_utf_8,
+    unicode_encode_utf_32_be, str_decode_utf_32_be
+)
 from pypy.interpreter.unicodehelper import encode_utf8sp, decode_utf8sp
 
 
@@ -28,6 +31,35 @@
     c = u"\udc00"
     py.test.raises(Hit, encode_utf8, space, u"\ud800" + c)
 
+
+def test_encode_utf_8_combine_surrogates():
+    """
+    In the case of a surrogate pair, the error handler should
+    return back a start and stop position of the full surrogate
+    pair (new behavior inherited from python3.6)
+    """
+    u = u"\udc80\ud800\udfff"
+
+    handler_num = 0
+
+    def errorhandler(errors, encoding, msg, s, start, end):
+        """
+        This handler will be called twice, so asserting both times:
+
+        1. the first time, 0xDC80 will be handled as a single surrogate,
+           since it is a standalone character and an invalid surrogate.
+        2. the second time, the characters will be 0xD800 and 0xDFFF, since
+           that is a valid surrogate pair.
+        """
+        assert s[start:end] in [u'\udc80', u'\uD800\uDFFF']
+        return [], None, end
+
+    unicode_encode_utf_8(
+        u, len(u), True,
+        errorhandler=errorhandler,
+        allow_surrogates=False
+    )
+
 def test_encode_utf8_allow_surrogates():
     sp = FakeSpace()
     assert encode_utf8(sp, u"\ud800", allow_surrogates=True) == "\xed\xa0\x80"
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -1,12 +1,13 @@
 import sys
 from pypy.interpreter.error import OperationError, oefmt
-from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
-from rpython.rlib import runicode
+from rpython.rlib import runicode, jit, nonconst
 from rpython.rlib.runicode import (
     default_unicode_error_encode, default_unicode_error_decode,
     MAXUNICODE, BYTEORDER, BYTEORDER2, UNICHR)
+from rpython.tool.sourcetools import func_with_new_name
 
 _WIN32 = sys.platform == 'win32'
 _MACOSX = sys.platform == 'darwin'
@@ -85,7 +86,7 @@
                                     force_replace=False)
     elif _MACOSX:
         uni = space.unicode_w(w_uni)
-        bytes = runicode.unicode_encode_utf_8_impl(
+        bytes = unicode_encode_utf_8_impl(
             uni, len(uni), 'surrogateescape',
             errorhandler=state.encode_error_handler,
             allow_surrogates=False)
@@ -117,12 +118,176 @@
     from pypy.module._codecs import interp_codecs
     state = space.fromcache(interp_codecs.CodecState)
     unicodedata_handler = state.get_unicodedata_handler(space)
-    result, consumed = runicode.str_decode_unicode_escape(
+    result, consumed, first_escape_error_char = str_decode_unicode_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space),
         unicodedata_handler=unicodedata_handler)
     return result
 
+
+hexdigits = "0123456789ABCDEFabcdef"
+
+
+def hexescape(builder, s, pos, digits,
+              encoding, errorhandler, message, errors):
+    chr = 0
+    if pos + digits > len(s):
+        endinpos = pos
+        while endinpos < len(s) and s[endinpos] in hexdigits:
+            endinpos += 1
+        res, pos = errorhandler(errors, encoding,
+                                message, s, pos-2, endinpos)
+        builder.append(res)
+    else:
+        try:
+            chr = r_uint(int(s[pos:pos+digits], 16))
+        except ValueError:
+            endinpos = pos
+            while s[endinpos] in hexdigits:
+                endinpos += 1
+            res, pos = errorhandler(errors, encoding,
+                                    message, s, pos-2, endinpos)
+            builder.append(res)
+        else:
+            # when we get here, chr is a 32-bit unicode character
+            if chr <= MAXUNICODE:
+                builder.append(UNICHR(chr))
+                pos += digits
+
+            elif chr <= 0x10ffff:
+                chr -= 0x10000L
+                builder.append(unichr(0xD800 + (chr >> 10)))
+                builder.append(unichr(0xDC00 + (chr & 0x03FF)))
+                pos += digits
+            else:
+                message = "illegal Unicode character"
+                res, pos = errorhandler(errors, encoding,
+                                        message, s, pos-2, pos+digits)
+                builder.append(res)
+    return pos
+
+
+def str_decode_unicode_escape(s, size, errors, final=False,
+                              errorhandler=None,
+                              unicodedata_handler=None):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+
+    if size == 0:
+        return u'', 0, None
+
+    builder = UnicodeBuilder(size)
+    pos = 0
+    first_escape_error_char = None
+    while pos < size:
+        ch = s[pos]
+
+        # Non-escape characters are interpreted as Unicode ordinals
+        if ch != '\\':
+            builder.append(unichr(ord(ch)))
+            pos += 1
+            continue
+
+        # - Escapes
+        pos += 1
+        if pos >= size:
+            message = "\\ at end of string"
+            res, pos = errorhandler(errors, "unicodeescape",
+                                    message, s, pos-1, size)
+            builder.append(res)
+            continue
+
+        ch = s[pos]
+        pos += 1
+        # \x escapes
+        if ch == '\n': pass
+        elif ch == '\\': builder.append(u'\\')
+        elif ch == '\'': builder.append(u'\'')
+        elif ch == '\"': builder.append(u'\"')
+        elif ch == 'b' : builder.append(u'\b')
+        elif ch == 'f' : builder.append(u'\f')
+        elif ch == 't' : builder.append(u'\t')
+        elif ch == 'n' : builder.append(u'\n')
+        elif ch == 'r' : builder.append(u'\r')
+        elif ch == 'v' : builder.append(u'\v')
+        elif ch == 'a' : builder.append(u'\a')
+        elif '0' <= ch <= '7':
+            x = ord(ch) - ord('0')
+            if pos < size:
+                ch = s[pos]
+                if '0' <= ch <= '7':
+                    pos += 1
+                    x = (x<<3) + ord(ch) - ord('0')
+                    if pos < size:
+                        ch = s[pos]
+                        if '0' <= ch <= '7':
+                            pos += 1
+                            x = (x<<3) + ord(ch) - ord('0')
+            builder.append(unichr(x))
+        # hex escapes
+        # \xXX
+        elif ch == 'x':
+            digits = 2
+            message = "truncated \\xXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        # \uXXXX
+        elif ch == 'u':
+            digits = 4
+            message = "truncated \\uXXXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        #  \UXXXXXXXX
+        elif ch == 'U':
+            digits = 8
+            message = "truncated \\UXXXXXXXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        # \N{name}
+        elif ch == 'N' and unicodedata_handler is not None:
+            message = "malformed \\N character escape"
+            look = pos
+
+            if look < size and s[look] == '{':
+                # look for the closing brace
+                while look < size and s[look] != '}':
+                    look += 1
+                if look < size and s[look] == '}':
+                    # found a name.  look it up in the unicode database
+                    message = "unknown Unicode character name"
+                    name = s[pos+1:look]
+                    code = unicodedata_handler.call(name)
+                    if code < 0:
+                        res, pos = errorhandler(errors, "unicodeescape",
+                                                message, s, pos-1, look+1)
+                        builder.append(res)
+                        continue
+                    pos = look + 1
+                    if code <= MAXUNICODE:
+                        builder.append(UNICHR(code))
+                    else:
+                        code -= 0x10000L
+                        builder.append(unichr(0xD800 + (code >> 10)))
+                        builder.append(unichr(0xDC00 + (code & 0x03FF)))
+                else:
+                    res, pos = errorhandler(errors, "unicodeescape",
+                                            message, s, pos-1, look+1)
+                    builder.append(res)
+            else:
+                res, pos = errorhandler(errors, "unicodeescape",
+                                        message, s, pos-1, look+1)
+                builder.append(res)
+        else:
+            first_escape_error_char = unichr(ord(ch))
+            builder.append(u'\\')
+            builder.append(unichr(ord(ch)))
+
+    return builder.build(), pos, first_escape_error_char
+
+
 def decode_raw_unicode_escape(space, string):
     result, consumed = runicode.str_decode_raw_unicode_escape(
         string, len(string), "strict",
@@ -149,11 +314,109 @@
     # allowed, either paired or lone.  A paired surrogate is considered
     # like the non-BMP character it stands for.  See also *_utf8sp().
     assert isinstance(uni, unicode)
-    return runicode.unicode_encode_utf_8(
+    return unicode_encode_utf_8(
         uni, len(uni), "strict",
         errorhandler=encode_error_handler(space),
         allow_surrogates=allow_surrogates)
 
+ at jit.elidable
+def unicode_encode_utf_8(s, size, errors, errorhandler=None,
+                         allow_surrogates=False):
+    # In this function, allow_surrogates can be:
+    #
+    #  * True:  surrogates are always allowed.  A valid surrogate pair
+    #           is replaced with the non-BMP unicode char it stands for,
+    #           which is then encoded as 4 bytes.
+    #
+    #  * False: surrogates are always forbidden.
+    #
+    # See also unicode_encode_utf8sp().
+    #
+    if errorhandler is None:
+        errorhandler = default_unicode_error_encode
+    return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
+                                         allow_surrogates=allow_surrogates)
+
+def unicode_encode_utf_8_impl(s, size, errors, errorhandler,
+                              allow_surrogates=False):
+    assert(size >= 0)
+    result = StringBuilder(size)
+    pos = 0
+    while pos < size:
+        ch = ord(s[pos])
+        pos += 1
+        if ch < 0x80:
+            # Encode ASCII
+            result.append(chr(ch))
+        elif ch < 0x0800:
+            # Encode Latin-1
+            result.append(chr((0xc0 | (ch >> 6))))
+            result.append(chr((0x80 | (ch & 0x3f))))
+        else:
+            # Encode UCS2 Unicode ordinals
+            if ch < 0x10000:
+                # Special case: check for surrogates
+                if 0xD800 <= ch <= 0xDFFF:
+                    error_start_pos = pos - 1
+                    if pos != size:
+                        ch2 = ord(s[pos])
+                        # check if the first character is a high surrogate,
+                        # and the second character is a low surrogate. If so,
+                        # they should be handled collectively.
+                        if ch <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFFF:
+                            # pos should be incremented regardless.
+                            # by doing so, it ensures the lower surrogate
+                            # is also included in the characters considered
+                            # in the errorhandler.
+                            pos += 1
+                            # if we allow surrogates, we should combine
+                            # the two and form a UCS4 value
+                            if allow_surrogates or MAXUNICODE < 65535 or is_narrow_host():
+                                ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
+                                assert ch3 >= 0
+                                _encodeUCS4(result, ch3)
+                                continue
+                    # note: if the program only ever calls this with
+                    # allow_surrogates=True, then we'll never annotate
+                    # the following block of code, and errorhandler()
+                    # will never be called.  This causes RPython
+                    # problems.  Avoid it with the nonconst hack.
+                    if not allow_surrogates or nonconst.NonConstant(False):
+                        ru, rs, pos = errorhandler(errors, 'utf8',
+                                                   'surrogates not allowed',
+                                                   s, error_start_pos, pos)
+                        if rs is not None:
+                            # py3k only
+                            result.append(rs)
+                            continue
+                        for ch in ru:
+                            if ord(ch) < 0x80:
+                                result.append(chr(ord(ch)))
+                            else:
+                                errorhandler('strict', 'utf8',
+                                             'surrogates not allowed',
+                                             s, pos - 1 , pos)
+                        continue
+                    # else: Fall through and handles isolated high surrogates
+                result.append((chr((0xe0 | (ch >> 12)))))
+                result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
+                result.append((chr((0x80 | (ch & 0x3f)))))
+            else:
+                _encodeUCS4(result, ch)
+    return result.build()
+unicode_encode_utf_8_elidable = jit.elidable(
+    func_with_new_name(unicode_encode_utf_8_impl,
+                       "unicode_encode_utf_8_elidable"))
+
+
+def _encodeUCS4(result, ch):
+    # Encode UCS4 Unicode ordinals
+    result.append((chr((0xf0 | (ch >> 18)))))
+    result.append((chr((0x80 | ((ch >> 12) & 0x3f)))))
+    result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
+    result.append((chr((0x80 | (ch & 0x3f)))))
+
+
 def encode_utf8sp(space, uni):
     # Surrogate-preserving utf-8 encoding.  Any surrogate character
     # turns into its 3-bytes encoding, whether it is paired or not.
@@ -586,3 +849,7 @@
     return unicode_encode_utf_32_helper(s, size, errors, errorhandler,
                                         allow_surrogates, "little",
                                         'utf-32-le')
+
+
+def is_narrow_host():
+    return not we_are_translated() and sys.maxunicode == 0xFFFF
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -10,6 +10,9 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter import unicodehelper
+from pypy.interpreter.unicodehelper import (
+    unicode_encode_utf_8_impl,
+    str_decode_unicode_escape)
 from pypy.module.unicodedata import unicodedb
 
 
@@ -735,7 +738,7 @@
     # NB. can't call unicode_encode_utf_8() directly because that's
     # an @elidable function nowadays.  Instead, we need the _impl().
     # (The problem is the errorhandler, which calls arbitrary Python.)
-    result = runicode.unicode_encode_utf_8_impl(
+    result = unicode_encode_utf_8_impl(
         uni, len(uni), errors, state.encode_error_handler,
         allow_surrogates=False)
     return space.newtuple([space.newbytes(result), space.newint(len(uni))])
@@ -947,11 +950,18 @@
 
     unicode_name_handler = state.get_unicodedata_handler(space)
 
-    result, consumed = runicode.str_decode_unicode_escape(
+    result, consumed, first_escape_error_char = str_decode_unicode_escape(
         string, len(string), errors,
         final, state.decode_error_handler,
         unicode_name_handler)
 
+    if first_escape_error_char is not None:
+        space.warn(
+            space.newtext("invalid escape sequence '\\%s'"
+                          % str(first_escape_error_char)),
+            space.w_DeprecationWarning
+        )
+
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 # ____________________________________________________________
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -796,6 +796,15 @@
             test_sequence = before_sequence + ill_surrogate + after_sequence
             raises(UnicodeDecodeError, test_sequence.decode, encoding)
 
+    def test_lone_surrogates_utf_8(self):
+        """
+        utf-8 should not longer allow surrogates,
+        and should return back full surrogate pairs.
+        """
+        e = raises(UnicodeEncodeError, u"\udc80\ud800\udfff".encode, "utf-8",
+                   "surrogateescape").value
+        assert e.object[e.start:e.end] == u'\ud800\udfff'
+
     def test_charmap_encode(self):
         assert 'xxx'.encode('charmap') == b'xxx'
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -6,7 +6,7 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
-    unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii,
+    unicode_encode_ascii, fast_str_decode_ascii,
     unicode_encode_utf8_forbid_surrogates, SurrogateError)
 from rpython.rlib import jit
 
@@ -564,7 +564,7 @@
         if encoding is None or encoding == 'utf-8':
             u = space.unicode_w(w_object)
             eh = unicodehelper.encode_error_handler(space)
-            return space.newbytes(unicode_encode_utf_8(
+            return space.newbytes(unicodehelper.unicode_encode_utf_8(
                     u, len(u), errors, errorhandler=eh))
         elif encoding == 'ascii':
             u = space.unicode_w(w_object)

From pypy.commits at gmail.com  Tue May 29 02:08:26 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Mon, 28 May 2018 23:08:26 -0700 (PDT)
Subject: [pypy-commit] pypy fix_test_codecs: Adding branch fix_test_codecs
Message-ID: <5b0cee5a.1c69fb81.f1f2.291b@mx.google.com>

Author: Yusuke Tsutsumi <yusuket at tsutsumi.io>
Branch: fix_test_codecs
Changeset: r94697:4020e294a7ba
Date: 2018-05-13 21:48 -0700
http://bitbucket.org/pypy/pypy/changeset/4020e294a7ba/

Log:	Adding branch fix_test_codecs


From pypy.commits at gmail.com  Tue May 29 02:08:24 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Mon, 28 May 2018 23:08:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Resolving failing deprecation tests by
 moving the warning code
Message-ID: <5b0cee58.1c69fb81.45516.c9f1@mx.google.com>

Author: Yusuke Tsutsumi <yusuket at tsutsumi.io>
Branch: py3.6
Changeset: r94696:f9930d8ca5da
Date: 2018-05-13 21:41 -0700
http://bitbucket.org/pypy/pypy/changeset/f9930d8ca5da/

Log:	Resolving failing deprecation tests by moving the warning code

	Moving the warning code for unicode errors higher up the chain,
	thereby ensuring the warnings are emitted in those calls.

diff --git a/lib-python/3/test/test_codecs.py b/lib-python/3/test/test_codecs.py
--- a/lib-python/3/test/test_codecs.py
+++ b/lib-python/3/test/test_codecs.py
@@ -2468,7 +2468,8 @@
                 with self.assertWarns(DeprecationWarning):
                     check(b"\\" + b, "\\" + chr(i))
             if b.upper() not in b'UN':
-                with self.assertWarns(DeprecationWarning):
+                with self.assertWarns(DeprecationWarning,
+                                      msg="character {} did not raise an exception".format(i)):
                     check(b"\\" + b.upper(), "\\" + chr(i-32))
         with self.assertWarns(DeprecationWarning):
             check(br"\8", "\\8")
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -117,12 +117,6 @@
     v, first_escape_error_char = PyString_DecodeEscape(
         space, substr, 'strict', encoding)
 
-    if first_escape_error_char != '':
-        space.warn(
-            space.newtext("invalid escape sequence '\\%s'"
-                          % first_escape_error_char),
-            space.w_DeprecationWarning)
-
     return space.newbytes(v)
 
 def decode_unicode_utf8(space, s, ps, q):
@@ -252,6 +246,13 @@
             # an arbitry number of unescaped UTF-8 bytes may follow.
 
     buf = builder.build()
+
+    if first_escape_error_char != '':
+        space.warn(
+            space.newtext("invalid escape sequence '\\%s'"
+                          % first_escape_error_char),
+            space.w_DeprecationWarning)
+
     return buf, first_escape_error_char
 
 

From pypy.commits at gmail.com  Tue May 29 02:08:33 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Mon, 28 May 2018 23:08:33 -0700 (PDT)
Subject: [pypy-commit] pypy fix_test_codecs: Copying code out of runicode
 into unicodehelper, further isolating the pypy code from rpython
Message-ID: <5b0cee61.1c69fb81.91a7e.e14d@mx.google.com>

Author: Yusuke Tsutsumi <yusuke at tsutsumi.io>
Branch: fix_test_codecs
Changeset: r94700:8f5146e6c44f
Date: 2018-05-24 20:01 -0700
http://bitbucket.org/pypy/pypy/changeset/8f5146e6c44f/

Log:	Copying code out of runicode into unicodehelper, further isolating
	the pypy code from rpython

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -1,12 +1,13 @@
 import sys
 from pypy.interpreter.error import OperationError, oefmt
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
-from rpython.rlib import runicode
+from rpython.rlib import runicode, jit, nonconst
 from rpython.rlib.runicode import (
     default_unicode_error_encode, default_unicode_error_decode,
     MAXUNICODE, BYTEORDER, BYTEORDER2, UNICHR)
+from rpython.tool.sourcetools import func_with_new_name
 
 _WIN32 = sys.platform == 'win32'
 _MACOSX = sys.platform == 'darwin'
@@ -85,7 +86,7 @@
                                     force_replace=False)
     elif _MACOSX:
         uni = space.unicode_w(w_uni)
-        bytes = runicode.unicode_encode_utf_8_impl(
+        bytes = unicode_encode_utf_8_impl(
             uni, len(uni), 'surrogateescape',
             errorhandler=state.encode_error_handler,
             allow_surrogates=False)
@@ -149,11 +150,102 @@
     # allowed, either paired or lone.  A paired surrogate is considered
     # like the non-BMP character it stands for.  See also *_utf8sp().
     assert isinstance(uni, unicode)
-    return runicode.unicode_encode_utf_8(
+    return unicode_encode_utf_8(
         uni, len(uni), "strict",
         errorhandler=encode_error_handler(space),
         allow_surrogates=allow_surrogates)
 
+ at jit.elidable
+def unicode_encode_utf_8(s, size, errors, errorhandler=None,
+                         allow_surrogates=False):
+    # In this function, allow_surrogates can be:
+    #
+    #  * True:  surrogates are always allowed.  A valid surrogate pair
+    #           is replaced with the non-BMP unicode char it stands for,
+    #           which is then encoded as 4 bytes.
+    #
+    #  * False: surrogates are always forbidden.
+    #
+    # See also unicode_encode_utf8sp().
+    #
+    if errorhandler is None:
+        errorhandler = default_unicode_error_encode
+    return unicode_encode_utf_8_elidable(s, size, errors, errorhandler,
+                                         allow_surrogates=allow_surrogates)
+
+def unicode_encode_utf_8_impl(s, size, errors, errorhandler,
+                              allow_surrogates=False):
+    assert(size >= 0)
+    result = StringBuilder(size)
+    pos = 0
+    while pos < size:
+        ch = ord(s[pos])
+        pos += 1
+        if ch < 0x80:
+            # Encode ASCII
+            result.append(chr(ch))
+        elif ch < 0x0800:
+            # Encode Latin-1
+            result.append(chr((0xc0 | (ch >> 6))))
+            result.append(chr((0x80 | (ch & 0x3f))))
+        else:
+            # Encode UCS2 Unicode ordinals
+            if ch < 0x10000:
+                # Special case: check for high surrogate
+                if 0xD800 <= ch <= 0xDFFF:
+                    if pos != size:
+                        ch2 = ord(s[pos])
+                        # Check for low surrogate and combine the two to
+                        # form a UCS4 value
+                        if ((allow_surrogates or MAXUNICODE < 65536
+                             or is_narrow_host()) and
+                            ch <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF):
+                            ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
+                            assert ch3 >= 0
+                            pos += 1
+                            _encodeUCS4(result, ch3)
+                            continue
+                    # note: if the program only ever calls this with
+                    # allow_surrogates=True, then we'll never annotate
+                    # the following block of code, and errorhandler()
+                    # will never be called.  This causes RPython
+                    # problems.  Avoid it with the nonconst hack.
+                    if not allow_surrogates or nonconst.NonConstant(False):
+                        ru, rs, pos = errorhandler(errors, 'utf8',
+                                                   'surrogates not allowed',
+                                                   s, pos-1, pos)
+                        if rs is not None:
+                            # py3k only
+                            result.append(rs)
+                            continue
+                        for ch in ru:
+                            if ord(ch) < 0x80:
+                                result.append(chr(ord(ch)))
+                            else:
+                                errorhandler('strict', 'utf8',
+                                             'surrogates not allowed',
+                                             s, pos-1, pos)
+                        continue
+                    # else: Fall through and handles isolated high surrogates
+                result.append((chr((0xe0 | (ch >> 12)))))
+                result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
+                result.append((chr((0x80 | (ch & 0x3f)))))
+            else:
+                _encodeUCS4(result, ch)
+    return result.build()
+unicode_encode_utf_8_elidable = jit.elidable(
+    func_with_new_name(unicode_encode_utf_8_impl,
+                       "unicode_encode_utf_8_elidable"))
+
+
+def _encodeUCS4(result, ch):
+    # Encode UCS4 Unicode ordinals
+    result.append((chr((0xf0 | (ch >> 18)))))
+    result.append((chr((0x80 | ((ch >> 12) & 0x3f)))))
+    result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
+    result.append((chr((0x80 | (ch & 0x3f)))))
+
+
 def encode_utf8sp(space, uni):
     # Surrogate-preserving utf-8 encoding.  Any surrogate character
     # turns into its 3-bytes encoding, whether it is paired or not.
@@ -370,3 +462,7 @@
     return unicode_encode_utf_32_helper(s, size, errors, errorhandler,
                                         allow_surrogates, "little",
                                         'utf-32-le')
+
+
+def is_narrow_host():
+    return not we_are_translated() and sys.maxunicode == 0xFFFF
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -10,6 +10,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter import unicodehelper
+from pypy.interpreter.unicodehelper import unicode_encode_utf_8_impl
 from pypy.module.unicodedata import unicodedb
 
 
@@ -737,7 +738,7 @@
     # NB. can't call unicode_encode_utf_8() directly because that's
     # an @elidable function nowadays.  Instead, we need the _impl().
     # (The problem is the errorhandler, which calls arbitrary Python.)
-    result = runicode.unicode_encode_utf_8_impl(
+    result = unicode_encode_utf_8_impl(
         uni, len(uni), errors, state.encode_error_handler,
         allow_surrogates=False)
     return space.newtuple([space.newbytes(result), space.newint(len(uni))])
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -6,7 +6,7 @@
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
-    unicode_encode_ascii, unicode_encode_utf_8, fast_str_decode_ascii,
+    unicode_encode_ascii, fast_str_decode_ascii,
     unicode_encode_utf8_forbid_surrogates, SurrogateError)
 from rpython.rlib import jit
 
@@ -564,7 +564,7 @@
         if encoding is None or encoding == 'utf-8':
             u = space.unicode_w(w_object)
             eh = unicodehelper.encode_error_handler(space)
-            return space.newbytes(unicode_encode_utf_8(
+            return space.newbytes(unicodehelper.unicode_encode_utf_8(
                     u, len(u), errors, errorhandler=eh))
         elif encoding == 'ascii':
             u = space.unicode_w(w_object)

From pypy.commits at gmail.com  Tue May 29 02:08:29 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Mon, 28 May 2018 23:08:29 -0700 (PDT)
Subject: [pypy-commit] pypy fix_test_codecs: Adding warning with escaping
 invalid unicode escape characters.
Message-ID: <5b0cee5d.1c69fb81.ba831.d4b9@mx.google.com>

Author: Yusuke Tsutsumi <yusuke at tsutsumi.io>
Branch: fix_test_codecs
Changeset: r94698:2fd74c4c379d
Date: 2018-05-16 07:13 -0700
http://bitbucket.org/pypy/pypy/changeset/2fd74c4c379d/

Log:	Adding warning with escaping invalid unicode escape characters.

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -117,7 +117,7 @@
     from pypy.module._codecs import interp_codecs
     state = space.fromcache(interp_codecs.CodecState)
     unicodedata_handler = state.get_unicodedata_handler(space)
-    result, consumed = runicode.str_decode_unicode_escape(
+    result, consumed, first_escape_error_char = runicode.str_decode_unicode_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space),
         unicodedata_handler=unicodedata_handler)
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -949,11 +949,18 @@
 
     unicode_name_handler = state.get_unicodedata_handler(space)
 
-    result, consumed = runicode.str_decode_unicode_escape(
+    result, consumed, first_escape_error_char = runicode.str_decode_unicode_escape(
         string, len(string), errors,
         final, state.decode_error_handler,
         unicode_name_handler)
 
+    if first_escape_error_char is not None:
+        space.warn(
+            space.newtext("invalid escape sequence '\\%s'"
+                          % str(first_escape_error_char)),
+            space.w_DeprecationWarning
+        )
+
     return space.newtuple([space.newunicode(result), space.newint(consumed)])
 
 # ____________________________________________________________
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -1435,10 +1435,11 @@
         errorhandler = default_unicode_error_decode
 
     if size == 0:
-        return u'', 0
+        return u'', 0, None
 
     builder = UnicodeBuilder(size)
     pos = 0
+    first_escape_error_char = None
     while pos < size:
         ch = s[pos]
 
@@ -1541,10 +1542,11 @@
                                         message, s, pos-1, look+1)
                 builder.append(res)
         else:
+            first_escape_error_char = unichr(ord(ch))
             builder.append(u'\\')
             builder.append(unichr(ord(ch)))
 
-    return builder.build(), pos
+    return builder.build(), pos, first_escape_error_char
 
 def make_unicode_escape_function(pass_printable=False, unicode_output=False,
                                  quotes=False, prefix=None):

From pypy.commits at gmail.com  Tue May 29 02:08:35 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Mon, 28 May 2018 23:08:35 -0700 (PDT)
Subject: [pypy-commit] pypy fix_test_codecs: Ported fix for surrogate
 handling to copied unicodehelper
Message-ID: <5b0cee63.1c69fb81.db5a3.01b9@mx.google.com>

Author: Yusuke Tsutsumi <yusuke at tsutsumi.io>
Branch: fix_test_codecs
Changeset: r94701:f9ba6a5a05cc
Date: 2018-05-25 07:54 -0700
http://bitbucket.org/pypy/pypy/changeset/f9ba6a5a05cc/

Log:	Ported fix for surrogate handling to copied unicodehelper

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -191,20 +191,27 @@
         else:
             # Encode UCS2 Unicode ordinals
             if ch < 0x10000:
-                # Special case: check for high surrogate
+                # Special case: check for surrogates
                 if 0xD800 <= ch <= 0xDFFF:
+                    error_start_pos = pos - 1
                     if pos != size:
                         ch2 = ord(s[pos])
-                        # Check for low surrogate and combine the two to
-                        # form a UCS4 value
-                        if ((allow_surrogates or MAXUNICODE < 65536
-                             or is_narrow_host()) and
-                            ch <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF):
-                            ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
-                            assert ch3 >= 0
+                        # check if the first character is a high surrogate,
+                        # and the second character is a low surrogate. If so,
+                        # they should be handled collectively.
+                        if ch <= 0xDBFF and 0xDC80 <= ch2 <= 0xDFFFF:
+                            # pos should be incremented regardless.
+                            # by doing so, it ensures the lower surrogate
+                            # is also included in the characters considered
+                            # in the errorhandler.
                             pos += 1
-                            _encodeUCS4(result, ch3)
-                            continue
+                            # if we allow surrogates, we should combine
+                            # the two and form a UCS4 value
+                            if allow_surrogates or MAXUNICODE < 65535 or is_narrow_host():
+                                ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
+                                assert ch3 >= 0
+                                _encodeUCS4(result, ch3)
+                                continue
                     # note: if the program only ever calls this with
                     # allow_surrogates=True, then we'll never annotate
                     # the following block of code, and errorhandler()
@@ -213,7 +220,7 @@
                     if not allow_surrogates or nonconst.NonConstant(False):
                         ru, rs, pos = errorhandler(errors, 'utf8',
                                                    'surrogates not allowed',
-                                                   s, pos-1, pos)
+                                                   s, error_start_pos, pos)
                         if rs is not None:
                             # py3k only
                             result.append(rs)
@@ -224,7 +231,7 @@
                             else:
                                 errorhandler('strict', 'utf8',
                                              'surrogates not allowed',
-                                             s, pos-1, pos)
+                                             s, pos - 1 , pos)
                         continue
                     # else: Fall through and handles isolated high surrogates
                 result.append((chr((0xe0 | (ch >> 12)))))

From pypy.commits at gmail.com  Tue May 29 02:08:31 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Mon, 28 May 2018 23:08:31 -0700 (PDT)
Subject: [pypy-commit] pypy fix_test_codecs: Adding low surrogates into
 errorhandlers for utf-8encoding
Message-ID: <5b0cee5f.1c69fb81.eefda.0a54@mx.google.com>

Author: Yusuke Tsutsumi <yusuke at tsutsumi.io>
Branch: fix_test_codecs
Changeset: r94699:6d4fc7830371
Date: 2018-05-20 06:34 -0700
http://bitbucket.org/pypy/pypy/changeset/6d4fc7830371/

Log:	Adding low surrogates into errorhandlers for utf-8encoding

	In CPython 3.6, Behavior was defined that included low surrogates
	when handling unicode encoding errors. This change adds that
	behavior.

	References:

	https://hg.python.org/cpython/rev/2b5357b38366
	https://bugs.python.org/issue25267

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -361,20 +361,27 @@
         else:
             # Encode UCS2 Unicode ordinals
             if ch < 0x10000:
-                # Special case: check for high surrogate
+                # Special case: check for surrogates
                 if 0xD800 <= ch <= 0xDFFF:
+                    error_start_pos = pos - 1
                     if pos != size:
                         ch2 = ord(s[pos])
-                        # Check for low surrogate and combine the two to
-                        # form a UCS4 value
-                        if ((allow_surrogates or MAXUNICODE < 65536
-                             or is_narrow_host()) and
-                            ch <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF):
-                            ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
-                            assert ch3 >= 0
+                        # check if the first character is a high surrogate,
+                        # and the second character is a low surrogate. If so,
+                        # they should be handled collectively.
+                        if ch <= 0xDBFF and 0xDC80 <= ch2 <= 0xDFFFF:
+                            # pos should be incremented regardless.
+                            # by doing so, it ensures the lower surrogate
+                            # is also included in the characters considered
+                            # in the errorhandler.
                             pos += 1
-                            _encodeUCS4(result, ch3)
-                            continue
+                            # if we allow surrogates, we should combine
+                            # the two and form a UCS4 value
+                            if allow_surrogates or MAXUNICODE < 65535 or is_narrow_host():
+                                ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
+                                assert ch3 >= 0
+                                _encodeUCS4(result, ch3)
+                                continue
                     # note: if the program only ever calls this with
                     # allow_surrogates=True, then we'll never annotate
                     # the following block of code, and errorhandler()
@@ -383,7 +390,7 @@
                     if not allow_surrogates or nonconst.NonConstant(False):
                         ru, rs, pos = errorhandler(errors, 'utf8',
                                                    'surrogates not allowed',
-                                                   s, pos-1, pos)
+                                                   s, error_start_pos, pos)
                         if rs is not None:
                             # py3k only
                             result.append(rs)
@@ -394,7 +401,7 @@
                             else:
                                 errorhandler('strict', 'utf8',
                                              'surrogates not allowed',
-                                             s, pos-1, pos)
+                                             s, pos - 1 , pos)
                         continue
                     # else: Fall through and handles isolated high surrogates
                 result.append((chr((0xe0 | (ch >> 12)))))

From pypy.commits at gmail.com  Tue May 29 02:08:37 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Mon, 28 May 2018 23:08:37 -0700 (PDT)
Subject: [pypy-commit] pypy fix_test_codecs: Adding unit test for new
 behavior in runicode. Fixing bug
Message-ID: <5b0cee65.1c69fb81.c28bd.ea3e@mx.google.com>

Author: Yusuke Tsutsumi <yusuke at tsutsumi.io>
Branch: fix_test_codecs
Changeset: r94702:c2a3d03741e2
Date: 2018-05-25 23:09 -0700
http://bitbucket.org/pypy/pypy/changeset/c2a3d03741e2/

Log:	Adding unit test for new behavior in runicode. Fixing bug

	Fixing a bug in the utf 8 handling which did not consider lower
	surrogates below 0xDC80.

	Adding unit tests for the new behavior in runicode, which combines
	high and low surrogates into a single errorhandler call.

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -199,7 +199,7 @@
                         # check if the first character is a high surrogate,
                         # and the second character is a low surrogate. If so,
                         # they should be handled collectively.
-                        if ch <= 0xDBFF and 0xDC80 <= ch2 <= 0xDFFFF:
+                        if ch <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFFF:
                             # pos should be incremented regardless.
                             # by doing so, it ensures the lower surrogate
                             # is also included in the characters considered
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -369,7 +369,7 @@
                         # check if the first character is a high surrogate,
                         # and the second character is a low surrogate. If so,
                         # they should be handled collectively.
-                        if ch <= 0xDBFF and 0xDC80 <= ch2 <= 0xDFFFF:
+                        if ch <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFFF:
                             # pos should be incremented regardless.
                             # by doing so, it ensures the lower surrogate
                             # is also included in the characters considered
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -963,3 +963,32 @@
             py.test.raises(
                 UnicodeEncodeError, runicode.unicode_encode_utf_8,
                 u, len(u), True, allow_surrogates=False)
+
+    def test_encode_utf_8_combine_surrogates(self):
+        """
+        In the case of a surrogate pair, the error handler should
+        return back a start and stop position of the full surrogate
+        pair (new behavior inherited from python3.6)
+        """
+        u = runicode.UNICHR(0xDC80) + runicode.UNICHR(0xD800) + \
+            runicode.UNICHR(0xDFFF)
+
+        handler_num = 0
+
+        def errorhandler(errors, encoding, msg, s, start, end):
+            """
+            This handler will be called twice, so asserting both times:
+
+            1. the first time, 0xDC80 will be handled as a single surrogate,
+               since it is a standalone character and an invalid surrogate.
+            2. the second time, the characters will be 0xD800 and 0xDFFF, since
+               that is a valid surrogate pair.
+            """
+            assert s[start:end] in [u'\udc80', u'\uD800\uDFFF']
+            return [], None, end
+
+        runicode.unicode_encode_utf_8(
+            u, len(u), True,
+            errorhandler=errorhandler,
+            allow_surrogates=False
+        )

From pypy.commits at gmail.com  Tue May 29 02:08:39 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Mon, 28 May 2018 23:08:39 -0700 (PDT)
Subject: [pypy-commit] pypy fix_test_codecs: Addressing code review feedback
 on #612
Message-ID: <5b0cee67.1c69fb81.4c3ee.16f9@mx.google.com>

Author: Yusuke Tsutsumi <yusuke at tsutsumi.io>
Branch: fix_test_codecs
Changeset: r94703:c6a20c1af5c0
Date: 2018-05-26 21:56 -0700
http://bitbucket.org/pypy/pypy/changeset/c6a20c1af5c0/

Log:	Addressing code review feedback on #612

	* removing all changes to rpython, as nothing needs to change there
	to ensure pypy3 is python3.6 compliant.
	* adding tests for new behavior introduced in pypy3, to satsify
	pyhton3.6 behavior

diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py
--- a/pypy/interpreter/test/test_unicodehelper.py
+++ b/pypy/interpreter/test/test_unicodehelper.py
@@ -3,7 +3,10 @@
 import struct
 import sys
 from pypy.interpreter.unicodehelper import (
-    encode_utf8, decode_utf8, unicode_encode_utf_32_be, str_decode_utf_32_be)
+    encode_utf8, decode_utf8,
+    unicode_encode_utf_8,
+    unicode_encode_utf_32_be, str_decode_utf_32_be
+)
 from pypy.interpreter.unicodehelper import encode_utf8sp, decode_utf8sp
 
 
@@ -28,6 +31,35 @@
     c = u"\udc00"
     py.test.raises(Hit, encode_utf8, space, u"\ud800" + c)
 
+
+def test_encode_utf_8_combine_surrogates():
+    """
+    In the case of a surrogate pair, the error handler should
+    return back a start and stop position of the full surrogate
+    pair (new behavior inherited from python3.6)
+    """
+    u = u"\udc80\ud800\udfff"
+
+    handler_num = 0
+
+    def errorhandler(errors, encoding, msg, s, start, end):
+        """
+        This handler will be called twice, so asserting both times:
+
+        1. the first time, 0xDC80 will be handled as a single surrogate,
+           since it is a standalone character and an invalid surrogate.
+        2. the second time, the characters will be 0xD800 and 0xDFFF, since
+           that is a valid surrogate pair.
+        """
+        assert s[start:end] in [u'\udc80', u'\uD800\uDFFF']
+        return [], None, end
+
+    unicode_encode_utf_8(
+        u, len(u), True,
+        errorhandler=errorhandler,
+        allow_surrogates=False
+    )
+
 def test_encode_utf8_allow_surrogates():
     sp = FakeSpace()
     assert encode_utf8(sp, u"\ud800", allow_surrogates=True) == "\xed\xa0\x80"
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -796,6 +796,14 @@
             test_sequence = before_sequence + ill_surrogate + after_sequence
             raises(UnicodeDecodeError, test_sequence.decode, encoding)
 
+    def test_lone_surrogates_utf_8(self):
+        """
+        utf-8 should not longer allow surrogates,
+        and should return back full surrogate pairs.
+        """
+        e = raises(UnicodeEncodeError, u"\udc80\ud800\udfff".encode, "utf-8")
+        assert e.object[e.start:e.end] == u'\ud800\udfff'
+
     def test_charmap_encode(self):
         assert 'xxx'.encode('charmap') == b'xxx'
 
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -361,27 +361,20 @@
         else:
             # Encode UCS2 Unicode ordinals
             if ch < 0x10000:
-                # Special case: check for surrogates
+                # Special case: check for high surrogate
                 if 0xD800 <= ch <= 0xDFFF:
-                    error_start_pos = pos - 1
                     if pos != size:
                         ch2 = ord(s[pos])
-                        # check if the first character is a high surrogate,
-                        # and the second character is a low surrogate. If so,
-                        # they should be handled collectively.
-                        if ch <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFFF:
-                            # pos should be incremented regardless.
-                            # by doing so, it ensures the lower surrogate
-                            # is also included in the characters considered
-                            # in the errorhandler.
+                        # Check for low surrogate and combine the two to
+                        # form a UCS4 value
+                        if ((allow_surrogates or MAXUNICODE < 65536
+                             or is_narrow_host()) and
+                            ch <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF):
+                            ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
+                            assert ch3 >= 0
                             pos += 1
-                            # if we allow surrogates, we should combine
-                            # the two and form a UCS4 value
-                            if allow_surrogates or MAXUNICODE < 65535 or is_narrow_host():
-                                ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
-                                assert ch3 >= 0
-                                _encodeUCS4(result, ch3)
-                                continue
+                            _encodeUCS4(result, ch3)
+                            continue
                     # note: if the program only ever calls this with
                     # allow_surrogates=True, then we'll never annotate
                     # the following block of code, and errorhandler()
@@ -390,7 +383,7 @@
                     if not allow_surrogates or nonconst.NonConstant(False):
                         ru, rs, pos = errorhandler(errors, 'utf8',
                                                    'surrogates not allowed',
-                                                   s, error_start_pos, pos)
+                                                   s, pos-1, pos)
                         if rs is not None:
                             # py3k only
                             result.append(rs)
@@ -401,7 +394,7 @@
                             else:
                                 errorhandler('strict', 'utf8',
                                              'surrogates not allowed',
-                                             s, pos - 1 , pos)
+                                             s, pos-1, pos)
                         continue
                     # else: Fall through and handles isolated high surrogates
                 result.append((chr((0xe0 | (ch >> 12)))))
@@ -1442,11 +1435,10 @@
         errorhandler = default_unicode_error_decode
 
     if size == 0:
-        return u'', 0, None
+        return u'', 0
 
     builder = UnicodeBuilder(size)
     pos = 0
-    first_escape_error_char = None
     while pos < size:
         ch = s[pos]
 
@@ -1549,11 +1541,10 @@
                                         message, s, pos-1, look+1)
                 builder.append(res)
         else:
-            first_escape_error_char = unichr(ord(ch))
             builder.append(u'\\')
             builder.append(unichr(ord(ch)))
 
-    return builder.build(), pos, first_escape_error_char
+    return builder.build(), pos
 
 def make_unicode_escape_function(pass_printable=False, unicode_output=False,
                                  quotes=False, prefix=None):
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -963,32 +963,3 @@
             py.test.raises(
                 UnicodeEncodeError, runicode.unicode_encode_utf_8,
                 u, len(u), True, allow_surrogates=False)
-
-    def test_encode_utf_8_combine_surrogates(self):
-        """
-        In the case of a surrogate pair, the error handler should
-        return back a start and stop position of the full surrogate
-        pair (new behavior inherited from python3.6)
-        """
-        u = runicode.UNICHR(0xDC80) + runicode.UNICHR(0xD800) + \
-            runicode.UNICHR(0xDFFF)
-
-        handler_num = 0
-
-        def errorhandler(errors, encoding, msg, s, start, end):
-            """
-            This handler will be called twice, so asserting both times:
-
-            1. the first time, 0xDC80 will be handled as a single surrogate,
-               since it is a standalone character and an invalid surrogate.
-            2. the second time, the characters will be 0xD800 and 0xDFFF, since
-               that is a valid surrogate pair.
-            """
-            assert s[start:end] in [u'\udc80', u'\uD800\uDFFF']
-            return [], None, end
-
-        runicode.unicode_encode_utf_8(
-            u, len(u), True,
-            errorhandler=errorhandler,
-            allow_surrogates=False
-        )

From pypy.commits at gmail.com  Tue May 29 02:08:41 2018
From: pypy.commits at gmail.com (Yusuke Tsutsumi)
Date: Mon, 28 May 2018 23:08:41 -0700 (PDT)
Subject: [pypy-commit] pypy fix_test_codecs: Fixing tests,
 and re-adding fixed for unicode encoding
Message-ID: <5b0cee69.1c69fb81.f4eb0.132b@mx.google.com>

Author: Yusuke Tsutsumi <yusuke at tsutsumi.io>
Branch: fix_test_codecs
Changeset: r94704:40a452db0df6
Date: 2018-05-26 22:29 -0700
http://bitbucket.org/pypy/pypy/changeset/40a452db0df6/

Log:	Fixing tests, and re-adding fixed for unicode encoding

	This re-adds the fixes for the failing test_codecs tests in pypy3,
	for python3.6 compatibility.

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -1,7 +1,7 @@
 import sys
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib import runicode, jit, nonconst
 from rpython.rlib.runicode import (
@@ -118,12 +118,176 @@
     from pypy.module._codecs import interp_codecs
     state = space.fromcache(interp_codecs.CodecState)
     unicodedata_handler = state.get_unicodedata_handler(space)
-    result, consumed, first_escape_error_char = runicode.str_decode_unicode_escape(
+    result, consumed, first_escape_error_char = str_decode_unicode_escape(
         string, len(string), "strict",
         final=True, errorhandler=decode_error_handler(space),
         unicodedata_handler=unicodedata_handler)
     return result
 
+
+hexdigits = "0123456789ABCDEFabcdef"
+
+
+def hexescape(builder, s, pos, digits,
+              encoding, errorhandler, message, errors):
+    chr = 0
+    if pos + digits > len(s):
+        endinpos = pos
+        while endinpos < len(s) and s[endinpos] in hexdigits:
+            endinpos += 1
+        res, pos = errorhandler(errors, encoding,
+                                message, s, pos-2, endinpos)
+        builder.append(res)
+    else:
+        try:
+            chr = r_uint(int(s[pos:pos+digits], 16))
+        except ValueError:
+            endinpos = pos
+            while s[endinpos] in hexdigits:
+                endinpos += 1
+            res, pos = errorhandler(errors, encoding,
+                                    message, s, pos-2, endinpos)
+            builder.append(res)
+        else:
+            # when we get here, chr is a 32-bit unicode character
+            if chr <= MAXUNICODE:
+                builder.append(UNICHR(chr))
+                pos += digits
+
+            elif chr <= 0x10ffff:
+                chr -= 0x10000L
+                builder.append(unichr(0xD800 + (chr >> 10)))
+                builder.append(unichr(0xDC00 + (chr & 0x03FF)))
+                pos += digits
+            else:
+                message = "illegal Unicode character"
+                res, pos = errorhandler(errors, encoding,
+                                        message, s, pos-2, pos+digits)
+                builder.append(res)
+    return pos
+
+
+def str_decode_unicode_escape(s, size, errors, final=False,
+                              errorhandler=None,
+                              unicodedata_handler=None):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_decode
+
+    if size == 0:
+        return u'', 0, None
+
+    builder = UnicodeBuilder(size)
+    pos = 0
+    first_escape_error_char = None
+    while pos < size:
+        ch = s[pos]
+
+        # Non-escape characters are interpreted as Unicode ordinals
+        if ch != '\\':
+            builder.append(unichr(ord(ch)))
+            pos += 1
+            continue
+
+        # - Escapes
+        pos += 1
+        if pos >= size:
+            message = "\\ at end of string"
+            res, pos = errorhandler(errors, "unicodeescape",
+                                    message, s, pos-1, size)
+            builder.append(res)
+            continue
+
+        ch = s[pos]
+        pos += 1
+        # \x escapes
+        if ch == '\n': pass
+        elif ch == '\\': builder.append(u'\\')
+        elif ch == '\'': builder.append(u'\'')
+        elif ch == '\"': builder.append(u'\"')
+        elif ch == 'b' : builder.append(u'\b')
+        elif ch == 'f' : builder.append(u'\f')
+        elif ch == 't' : builder.append(u'\t')
+        elif ch == 'n' : builder.append(u'\n')
+        elif ch == 'r' : builder.append(u'\r')
+        elif ch == 'v' : builder.append(u'\v')
+        elif ch == 'a' : builder.append(u'\a')
+        elif '0' <= ch <= '7':
+            x = ord(ch) - ord('0')
+            if pos < size:
+                ch = s[pos]
+                if '0' <= ch <= '7':
+                    pos += 1
+                    x = (x<<3) + ord(ch) - ord('0')
+                    if pos < size:
+                        ch = s[pos]
+                        if '0' <= ch <= '7':
+                            pos += 1
+                            x = (x<<3) + ord(ch) - ord('0')
+            builder.append(unichr(x))
+        # hex escapes
+        # \xXX
+        elif ch == 'x':
+            digits = 2
+            message = "truncated \\xXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        # \uXXXX
+        elif ch == 'u':
+            digits = 4
+            message = "truncated \\uXXXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        #  \UXXXXXXXX
+        elif ch == 'U':
+            digits = 8
+            message = "truncated \\UXXXXXXXX escape"
+            pos = hexescape(builder, s, pos, digits,
+                            "unicodeescape", errorhandler, message, errors)
+
+        # \N{name}
+        elif ch == 'N' and unicodedata_handler is not None:
+            message = "malformed \\N character escape"
+            look = pos
+
+            if look < size and s[look] == '{':
+                # look for the closing brace
+                while look < size and s[look] != '}':
+                    look += 1
+                if look < size and s[look] == '}':
+                    # found a name.  look it up in the unicode database
+                    message = "unknown Unicode character name"
+                    name = s[pos+1:look]
+                    code = unicodedata_handler.call(name)
+                    if code < 0:
+                        res, pos = errorhandler(errors, "unicodeescape",
+                                                message, s, pos-1, look+1)
+                        builder.append(res)
+                        continue
+                    pos = look + 1
+                    if code <= MAXUNICODE:
+                        builder.append(UNICHR(code))
+                    else:
+                        code -= 0x10000L
+                        builder.append(unichr(0xD800 + (code >> 10)))
+                        builder.append(unichr(0xDC00 + (code & 0x03FF)))
+                else:
+                    res, pos = errorhandler(errors, "unicodeescape",
+                                            message, s, pos-1, look+1)
+                    builder.append(res)
+            else:
+                res, pos = errorhandler(errors, "unicodeescape",
+                                        message, s, pos-1, look+1)
+                builder.append(res)
+        else:
+            first_escape_error_char = unichr(ord(ch))
+            builder.append(u'\\')
+            builder.append(unichr(ord(ch)))
+
+    return builder.build(), pos, first_escape_error_char
+
+
 def decode_raw_unicode_escape(space, string):
     result, consumed = runicode.str_decode_raw_unicode_escape(
         string, len(string), "strict",
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -10,7 +10,9 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter import unicodehelper
-from pypy.interpreter.unicodehelper import unicode_encode_utf_8_impl
+from pypy.interpreter.unicodehelper import (
+    unicode_encode_utf_8_impl,
+    str_decode_unicode_escape)
 from pypy.module.unicodedata import unicodedb
 
 
@@ -950,7 +952,7 @@
 
     unicode_name_handler = state.get_unicodedata_handler(space)
 
-    result, consumed, first_escape_error_char = runicode.str_decode_unicode_escape(
+    result, consumed, first_escape_error_char = str_decode_unicode_escape(
         string, len(string), errors,
         final, state.decode_error_handler,
         unicode_name_handler)
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -801,7 +801,8 @@
         utf-8 should not longer allow surrogates,
         and should return back full surrogate pairs.
         """
-        e = raises(UnicodeEncodeError, u"\udc80\ud800\udfff".encode, "utf-8")
+        e = raises(UnicodeEncodeError, u"\udc80\ud800\udfff".encode, "utf-8",
+                   "surrogateescape").value
         assert e.object[e.start:e.end] == u'\ud800\udfff'
 
     def test_charmap_encode(self):

From pypy.commits at gmail.com  Tue May 29 02:16:12 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 28 May 2018 23:16:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: a test for the warning in
 unicode_escape_decode
Message-ID: <5b0cf02c.1c69fb81.11c8d.6a2a@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94706:990690db345d
Date: 2018-05-29 08:15 +0200
http://bitbucket.org/pypy/pypy/changeset/990690db345d/

Log:	a test for the warning in unicode_escape_decode

diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -2,7 +2,7 @@
 
 class AppTestCodecs:
     spaceconfig = {
-        "usemodules": ['unicodedata', 'struct', 'binascii'],
+        "usemodules": ['unicodedata', 'struct', 'binascii', '_warnings'],
     }
 
     def test_register_noncallable(self):
@@ -999,3 +999,16 @@
         errors = []
         assert sin.encode("iso-8859-15", "test.record") == b"\xac\xa4"
         assert errors == [u'\u1234\u1234', u'\u8000']
+
+    def test_warn_escape_decode(self):
+        import warnings
+        import codecs
+
+        with warnings.catch_warnings(record=True) as l:
+            warnings.simplefilter("always")
+            codecs.unicode_escape_decode(b'\\A')
+
+        assert isinstance(l[0].message, DeprecationWarning)
+
+
+

From pypy.commits at gmail.com  Tue May 29 07:14:49 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 May 2018 04:14:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: use the actual object types for the error
 message, and use CPython's arguably clearer message
Message-ID: <5b0d3629.1c69fb81.6f1c5.97d6@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94707:032703549e15
Date: 2018-05-29 13:14 +0200
http://bitbucket.org/pypy/pypy/changeset/032703549e15/

Log:	use the actual object types for the error message, and use CPython's
	arguably clearer message

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -581,9 +581,9 @@
             else:
                 return rstring.StringBuilder()
 
-        def _unknown_presentation(self, tp):
+        def _unknown_presentation(self, w_val):
             raise oefmt(self.space.w_ValueError,
-                        "unknown presentation for %s: '%s'", tp, self._type)
+                        "unknown format code %s for object of type '%T'", self._type, w_val)
 
         def format_string(self, w_string):
             space = self.space
@@ -593,7 +593,7 @@
             if self._parse_spec("s", "<"):
                 return self.wrap(string)
             if self._type != "s":
-                self._unknown_presentation("string")
+                self._unknown_presentation(w_string)
             if self._sign != "\0":
                 raise oefmt(space.w_ValueError,
                             "Sign not allowed in string format specifier")
@@ -955,7 +955,7 @@
                 w_float = space.float(w_num)
                 return self._format_float(w_float)
             else:
-                self._unknown_presentation("int" if kind == INT_KIND else "long")
+                self._unknown_presentation(w_num)
 
         def _parse_number(self, s, i):
             """Determine if s has a decimal point, and the index of the first #
@@ -1038,7 +1038,7 @@
                 tp == "n" or
                 tp == "%"):
                 return self._format_float(w_float)
-            self._unknown_presentation("float")
+            self._unknown_presentation(w_float)
 
         def _format_complex(self, w_complex):
             flags = 0
@@ -1197,7 +1197,7 @@
                 tp == "G" or
                 tp == "n"):
                 return self._format_complex(w_complex)
-            self._unknown_presentation("complex")
+            self._unknown_presentation(w_complex)
     return Formatter
 
 unicode_formatter = make_formatting_class(for_unicode=True)
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -480,3 +480,10 @@
         assert isinstance(first, str)
         for x, y in l:
             assert isinstance(y, str)
+
+    def test_unknown_presentation_error_message(self):
+        class x(int):
+            pass
+
+        excinfo = raises(ValueError, "{:j}".format, x(1))
+        assert str(excinfo.value) == "unknown format code j for object of type 'x'"

From pypy.commits at gmail.com  Tue May 29 07:29:20 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 May 2018 04:29:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6-wordcode: at least give a clear error
 message
Message-ID: <5b0d3990.1c69fb81.6f1c5.9bea@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6-wordcode
Changeset: r94708:95399d92b028
Date: 2018-05-26 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/95399d92b028/

Log:	at least give a clear error message

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -216,7 +216,7 @@
         """
         Hack to initialize the code object from a real (CPython) one.
         """
-        XXX
+        raise TypeError("assert reinterpretation for applevel tests is broken on PyPy3!")
         assert isinstance(code, types.CodeType)
         newconsts_w = [None] * len(code.co_consts)
         num = 0

From pypy.commits at gmail.com  Tue May 29 07:57:47 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 29 May 2018 04:57:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: differences in error message
Message-ID: <5b0d403b.1c69fb81.47a29.29cb@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.6
Changeset: r94709:88d3192d1645
Date: 2018-05-29 13:43 +0200
http://bitbucket.org/pypy/pypy/changeset/88d3192d1645/

Log:	differences in error message

diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py
--- a/lib-python/3/test/test_fstring.py
+++ b/lib-python/3/test/test_fstring.py
@@ -148,7 +148,7 @@
         # Inside of strings, don't interpret doubled brackets.
         self.assertEqual(f'{"{{}}"}', '{{}}')
 
-        self.assertAllRaise(TypeError, 'unhashable type',
+        self.assertAllRaise(TypeError, 'unhashable',
                             ["f'{ {{}} }'", # dict in a set
                              ])
 
@@ -411,7 +411,7 @@
 
         # lambda doesn't work without parens, because the colon
         #  makes the parser think it's a format_spec
-        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+        self.assertAllRaise(SyntaxError, '',
                             ["f'{lambda x:x}'",
                              ])
 

From pypy.commits at gmail.com  Wed May 30 12:43:36 2018
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 30 May 2018 09:43:36 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: this looks like a typo which was
 introduced by 490058ea54e6; add a test and fix
Message-ID: <5b0ed4b8.1c69fb81.e6331.61f8@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94710:f28af902fb66
Date: 2018-05-29 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/f28af902fb66/

Log:	this looks like a typo which was introduced by 490058ea54e6; add a
	test and fix

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -760,7 +760,7 @@
     def collect(self, gen=2):
         """Do a minor (gen=0), start a major (gen=1), or do a full
         major (gen>=2) collection."""
-        if gen < 0:
+        if gen <= 0:
             self._minor_collection()   # dangerous! no major GC cycle progress
         elif gen <= 1:
             self.minor_collection_with_major_progress(force_enabled=True)
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -771,6 +771,12 @@
                 assert elem.prev == lltype.nullptr(S)
                 assert elem.next == lltype.nullptr(S)
 
+    def test_collect_0(self, debuglog):
+        self.gc.collect(1) # start a major
+        debuglog.reset()
+        self.gc.collect(0) # do ONLY a minor
+        assert debuglog.summary() == {'gc-minor': 1}
+
     def test_enable_disable(self, debuglog):
         def large_malloc():
             # malloc an object which is large enough to trigger a major collection

From pypy.commits at gmail.com  Wed May 30 12:43:39 2018
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 30 May 2018 09:43:39 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: add a method to manually run a
 single gc-collect-step
Message-ID: <5b0ed4bb.1c69fb81.3e59b.e35a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94711:5a7307d2116c
Date: 2018-05-29 16:25 +0200
http://bitbucket.org/pypy/pypy/changeset/5a7307d2116c/

Log:	add a method to manually run a single gc-collect-step

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -770,6 +770,22 @@
             self.minor_and_major_collection()
         self.rrc_invoke_callback()
 
+    def collect_step(self):
+        """
+        Do a single major collection step. Return True when the major collection
+        is completed.
+
+        This is meant to be used together with gc.disable(), to have a
+        fine-grained control on when the GC runs.
+        """
+        in_progress = self.gc_state != STATE_SCANNING
+        self._minor_collection()
+        self.major_collection_step()
+        self.rrc_invoke_callback()
+        # if we were in the middle of a collection and we are back to
+        # STATE_SCANNING, it means we have just finished one
+        done = in_progress and self.gc_state == STATE_SCANNING
+        return done
 
     def minor_collection_with_major_progress(self, extrasize=0,
                                              force_enabled=False):
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -821,3 +821,16 @@
         assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor']
         # s is freed
         py.test.raises(RuntimeError, 's.x')
+
+    def test_collect_step(self, debuglog):
+        n = 0
+        while True:
+            debuglog.reset()
+            done = self.gc.collect_step()
+            summary = debuglog.summary()
+            assert summary == {'gc-minor': 1, 'gc-collect-step': 1}
+            if done:
+                break
+            n += 1
+            if n == 100:
+                assert False, 'this looks like an endless loop'

From pypy.commits at gmail.com  Wed May 30 12:43:41 2018
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 30 May 2018 09:43:41 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: implement rgc.collect_step() through
 all the layers
Message-ID: <5b0ed4bd.1c69fb81.3e781.f9ac@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94712:868c42b150e9
Date: 2018-05-29 18:58 +0200
http://bitbucket.org/pypy/pypy/changeset/868c42b150e9/

Log:	implement rgc.collect_step() through all the layers

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -159,6 +159,9 @@
     def isenabled(self):
         return True
 
+    def collect_step(self):
+        return True
+
     def malloc(self, typeid, length=0, zero=False):
         """NOT_RPYTHON
         For testing.  The interface used by the gctransformer is
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -309,6 +309,8 @@
 
         self.collect_ptr = getfn(GCClass.collect.im_func,
             [s_gc, annmodel.SomeInteger()], annmodel.s_None)
+        self.collect_step_ptr = getfn(GCClass.collect_step.im_func, [s_gc],
+                                      annmodel.s_Bool)
         self.enable_ptr = getfn(GCClass.enable.im_func, [s_gc], annmodel.s_None)
         self.disable_ptr = getfn(GCClass.disable.im_func, [s_gc], annmodel.s_None)
         self.isenabled_ptr = getfn(GCClass.isenabled.im_func, [s_gc],
@@ -888,6 +890,13 @@
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
+    def gct_gc__collect_step(self, hop):
+        op = hop.spaceop
+        livevars = self.push_roots(hop)
+        hop.genop("direct_call", [self.collect_step_ptr, self.c_const_gc],
+                  resultvar=op.result)
+        self.pop_roots(hop, livevars)
+
     def gct_gc__enable(self, hop):
         op = hop.spaceop
         hop.genop("direct_call", [self.enable_ptr, self.c_const_gc],
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -17,6 +17,14 @@
 disable = gc.disable
 isenabled = gc.isenabled
 
+def collect_step():
+    """
+    If the GC is incremental, run a single gc-collect-step. Return True when
+    the major collection is completed.
+    If the GC is not incremental, do nothing.
+    """
+    return True
+
 def set_max_heap_size(nbytes):
     """Limit the heap size to n bytes.
     """
@@ -153,6 +161,18 @@
         return hop.genop('gc__isenabled', hop.args_v, resulttype=hop.r_result)
 
 
+class CollectStepEntry(ExtRegistryEntry):
+    _about_ = collect_step
+
+    def compute_result_annotation(self):
+        from rpython.annotator import model as annmodel
+        return annmodel.s_Bool
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        return hop.genop('gc__collect_step', hop.args_v, resulttype=hop.r_result)
+
+
 class SetMaxHeapSizeEntry(ExtRegistryEntry):
     _about_ = set_max_heap_size
 
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -56,6 +56,19 @@
     res = interpret(f, [])
     assert res
 
+def test_collect_step():
+    def f():
+        return rgc.collect_step()
+
+    assert f()
+    t, typer, graph = gengraph(f, [])
+    blockops = list(graph.iterblockops())
+    opnames = [op.opname for block, op in blockops
+               if op.opname.startswith('gc__')]
+    assert opnames == ['gc__collect_step']
+    res = interpret(f, [])
+    assert res
+
 
 def test_can_move():
     T0 = lltype.GcStruct('T')
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -819,6 +819,9 @@
     def op_gc__collect(self, *gen):
         self.heap.collect(*gen)
 
+    def op_gc__collect_step(self):
+        return self.heap.collect_step()
+
     def op_gc__enable(self):
         self.heap.enable()
 
diff --git a/rpython/rtyper/lltypesystem/llheap.py b/rpython/rtyper/lltypesystem/llheap.py
--- a/rpython/rtyper/lltypesystem/llheap.py
+++ b/rpython/rtyper/lltypesystem/llheap.py
@@ -5,7 +5,7 @@
 
 setfield = setattr
 from operator import setitem as setarrayitem
-from rpython.rlib.rgc import can_move, collect, enable, disable, isenabled, add_memory_pressure
+from rpython.rlib.rgc import can_move, collect, enable, disable, isenabled, add_memory_pressure, collect_step
 
 def setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue,
                 offsets=None):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -450,6 +450,7 @@
     # __________ GC operations __________
 
     'gc__collect':          LLOp(canmallocgc=True),
+    'gc__collect_step':     LLOp(canmallocgc=True),
     'gc__enable':           LLOp(),
     'gc__disable':          LLOp(),
     'gc__isenabled':        LLOp(),
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1851,6 +1851,39 @@
         deleted = self.run("enable_disable", 1)
         assert deleted == 0
 
+    def define_collect_step(self):
+        class Counter(object):
+            val = 0
+        counter = Counter()
+        class X(object):
+            def __del__(self):
+                counter.val += 1
+        def f():
+            x1 = X()
+            rgc.collect() # make x1 old
+            assert not rgc.can_move(x1)
+            x1 = None
+            #
+            gc.disable()
+            n = 0
+            while True:
+                n += 1
+                if rgc.collect_step():
+                    break
+                if n == 100:
+                    print 'Endless loop!'
+                    assert False, 'this looks like an endless loop'
+                
+            if n < 4: # we expect at least 4 steps
+                print 'Too few steps! n =', n
+                assert False
+            return counter.val
+        return f
+
+    def test_collect_step(self):
+        deleted = self.run("collect_step")
+        assert deleted == 1
+
 
 # ____________________________________________________________________
 

From pypy.commits at gmail.com  Wed May 30 12:43:43 2018
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 30 May 2018 09:43:43 -0700 (PDT)
Subject: [pypy-commit] pypy gc-disable: implement the app-level version of
 gc.collect_step()
Message-ID: <5b0ed4bf.1c69fb81.6b2b4.f39d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: gc-disable
Changeset: r94713:8315bb2ed1ef
Date: 2018-05-30 18:42 +0200
http://bitbucket.org/pypy/pypy/changeset/8315bb2ed1ef/

Log:	implement the app-level version of gc.collect_step()

diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -4,6 +4,7 @@
 class Module(MixedModule):
     interpleveldefs = {
         'collect': 'interp_gc.collect',
+        'collect_step': 'interp_gc.collect_step',
         'enable': 'interp_gc.enable',
         'disable': 'interp_gc.disable',
         'isenabled': 'interp_gc.isenabled',
diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py
--- a/pypy/module/gc/interp_gc.py
+++ b/pypy/module/gc/interp_gc.py
@@ -16,7 +16,9 @@
     cache.clear()
 
     rgc.collect()
+    _run_finalizers(space)
 
+def _run_finalizers(space):
     # if we are running in gc.disable() mode but gc.collect() is called,
     # we should still call the finalizers now.  We do this as an attempt
     # to get closer to CPython's behavior: in Py3.5 some tests
@@ -79,6 +81,43 @@
     if uda.pending_with_disabled_del is None:
         uda.pending_with_disabled_del = []
 
+
+class StepCollector(object):
+    """
+    Invoke rgc.collect_step() until we are done, then run the app-level
+    finalizers as a separate step
+    """
+
+    def __init__(self, space):
+        self.space = space
+        self.finalizing = False
+
+    def do(self):
+        if self.finalizing:
+            self._run_finalizers()
+            self.finalizing = False
+            return True # everything done
+        else:
+            done = self._collect_step()
+            if done:
+                self.finalizing = True
+            return False # still something to do
+
+    def _collect_step(self):
+        return rgc.collect_step()
+
+    def _run_finalizers(self):
+        _run_finalizers(self.space)
+
+def collect_step(space):
+    """
+    If the GC is incremental, run a single gc-collect-step. Return True when
+    the major collection is completed.
+    If the GC is not incremental, do a full collection and return True.
+    """
+    sc = space.fromcache(StepCollector)
+    return space.newbool(sc.do())
+
 # ____________________________________________________________
 
 @unwrap_spec(filename='fsencode')
diff --git a/pypy/module/gc/test/test_gc.py b/pypy/module/gc/test/test_gc.py
--- a/pypy/module/gc/test/test_gc.py
+++ b/pypy/module/gc/test/test_gc.py
@@ -3,6 +3,7 @@
 from rpython.rlib import rgc
 from pypy.interpreter.baseobjspace import ObjSpace
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.module.gc.interp_gc import StepCollector
 
 class AppTestGC(object):
 
@@ -99,6 +100,24 @@
         assert deleted == [1]
         gc.enable()
 
+    def test_gc_collect_step(self):
+        import gc
+
+        class X(object):
+            deleted = 0
+            def __del__(self):
+                X.deleted += 1
+
+        gc.disable()
+        X(); X(); X();
+        n = 0
+        while True:
+            n += 1
+            if gc.collect_step():
+                break
+
+        assert n >= 2 # at least one step + 1 finalizing
+        assert X.deleted == 3
 
 class AppTestGcDumpHeap(object):
     pytestmark = py.test.mark.xfail(run=False)
@@ -172,3 +191,36 @@
         gc.collect()    # the classes C should all go away here
         for r in rlist:
             assert r() is None
+
+
+def test_StepCollector():
+    class MyStepCollector(StepCollector):
+        my_steps = 0
+        my_done = False
+        my_finalized = 0
+
+        def _collect_step(self):
+            self.my_steps += 1
+            return self.my_done
+
+        def _run_finalizers(self):
+            self.my_finalized += 1
+
+    sc = MyStepCollector(space=None)
+    assert not sc.do()
+    assert sc.my_steps == 1
+    assert not sc.do()
+    assert sc.my_steps == 2
+    sc.my_done = True
+    assert not sc.do()
+    assert sc.my_steps == 3
+    assert sc.my_finalized == 0
+    assert sc.finalizing
+    assert sc.do()
+    assert sc.my_steps == 3
+    assert sc.my_finalized == 1
+    assert not sc.finalizing
+    assert not sc.do()
+    assert sc.my_steps == 4
+    assert sc.my_finalized == 1
+
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -160,6 +160,7 @@
         return True
 
     def collect_step(self):
+        self.collect()
         return True
 
     def malloc(self, typeid, length=0, zero=False):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -21,8 +21,9 @@
     """
     If the GC is incremental, run a single gc-collect-step. Return True when
     the major collection is completed.
-    If the GC is not incremental, do nothing.
+    If the GC is not incremental, do a full collection and return True.
     """
+    gc.collect()
     return True
 
 def set_max_heap_size(nbytes):

From pypy.commits at gmail.com  Wed May 30 19:02:29 2018
From: pypy.commits at gmail.com (mjacob)
Date: Wed, 30 May 2018 16:02:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix NameError from last commit.
Message-ID: <5b0f2d85.1c69fb81.690c.0470@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r94714:1807915e1ede
Date: 2018-05-31 01:01 +0200
http://bitbucket.org/pypy/pypy/changeset/1807915e1ede/

Log:	Fix NameError from last commit.

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1447,7 +1447,7 @@
         for i in range(itemcount, 0, -1):
             w_item = frame.peekvalue(i-1)
             w_sum.extend(w_item)
-        self.popalues(itemcount)
+        frame.popvalues(itemcount)
         return w_sum
 
     @jit.unroll_safe

From pypy.commits at gmail.com  Thu May 31 13:05:09 2018
From: pypy.commits at gmail.com (arigo)
Date: Thu, 31 May 2018 10:05:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: Make 'grp' and the app-level 'pwd'
 thread-safe. Note that there is
Message-ID: <5b102b45.1c69fb81.a746b.68cf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r94715:ed27afd7ef3c
Date: 2018-05-31 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/ed27afd7ef3c/

Log:	Make 'grp' and the app-level 'pwd' thread-safe. Note that there is
	already a built-in 'pwd' module that is already thread-safe.

diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py
--- a/lib_pypy/grp.py
+++ b/lib_pypy/grp.py
@@ -4,6 +4,8 @@
 
 from _pwdgrp_cffi import ffi, lib
 import _structseq
+import thread
+_lock = thread.allocate_lock()
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
@@ -33,32 +35,35 @@
 
 @builtinify
 def getgrgid(gid):
-    res = lib.getgrgid(gid)
-    if not res:
-        # XXX maybe check error eventually
-        raise KeyError(gid)
-    return _group_from_gstruct(res)
+    with _lock:
+        res = lib.getgrgid(gid)
+        if not res:
+            # XXX maybe check error eventually
+            raise KeyError(gid)
+        return _group_from_gstruct(res)
 
 @builtinify
 def getgrnam(name):
     if not isinstance(name, basestring):
         raise TypeError("expected string")
     name = str(name)
-    res = lib.getgrnam(name)
-    if not res:
-        raise KeyError("'getgrnam(): name not found: %s'" % name)
-    return _group_from_gstruct(res)
+    with _lock:
+        res = lib.getgrnam(name)
+        if not res:
+            raise KeyError("'getgrnam(): name not found: %s'" % name)
+        return _group_from_gstruct(res)
 
 @builtinify
 def getgrall():
-    lib.setgrent()
     lst = []
-    while 1:
-        p = lib.getgrent()
-        if not p:
-            break
-        lst.append(_group_from_gstruct(p))
-    lib.endgrent()
+    with _lock:
+        lib.setgrent()
+        while 1:
+            p = lib.getgrent()
+            if not p:
+                break
+            lst.append(_group_from_gstruct(p))
+        lib.endgrent()
     return lst
 
 __all__ = ('struct_group', 'getgrgid', 'getgrnam', 'getgrall')
diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py
--- a/lib_pypy/pwd.py
+++ b/lib_pypy/pwd.py
@@ -12,6 +12,8 @@
 
 from _pwdgrp_cffi import ffi, lib
 import _structseq
+import thread
+_lock = thread.allocate_lock()
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
@@ -55,10 +57,11 @@
     Return the password database entry for the given numeric user ID.
     See pwd.__doc__ for more on password database entries.
     """
-    pw = lib.getpwuid(uid)
-    if not pw:
-        raise KeyError("getpwuid(): uid not found: %s" % uid)
-    return _mkpwent(pw)
+    with _lock:
+        pw = lib.getpwuid(uid)
+        if not pw:
+            raise KeyError("getpwuid(): uid not found: %s" % uid)
+        return _mkpwent(pw)
 
 @builtinify
 def getpwnam(name):
@@ -71,10 +74,11 @@
     if not isinstance(name, basestring):
         raise TypeError("expected string")
     name = str(name)
-    pw = lib.getpwnam(name)
-    if not pw:
-        raise KeyError("getpwname(): name not found: %s" % name)
-    return _mkpwent(pw)
+    with _lock:
+        pw = lib.getpwnam(name)
+        if not pw:
+            raise KeyError("getpwname(): name not found: %s" % name)
+        return _mkpwent(pw)
 
 @builtinify
 def getpwall():
@@ -84,13 +88,14 @@
     See pwd.__doc__ for more on password database entries.
     """
     users = []
-    lib.setpwent()
-    while True:
-        pw = lib.getpwent()
-        if not pw:
-            break
-        users.append(_mkpwent(pw))
-    lib.endpwent()
+    with _lock:
+        lib.setpwent()
+        while True:
+            pw = lib.getpwent()
+            if not pw:
+                break
+            users.append(_mkpwent(pw))
+        lib.endpwent()
     return users
 
 __all__ = ('struct_passwd', 'getpwuid', 'getpwnam', 'getpwall')